From 033b11912121ad2c1dbd4a93202eeac196124801 Mon Sep 17 00:00:00 2001 From: Jon Olav Hauglid Date: Tue, 16 Aug 2016 15:35:19 +0200 Subject: Bug#24388746: PRIVILEGE ESCALATION AND RACE CONDITION USING CREATE TABLE During REPAIR TABLE of a MyISAM table, a temporary data file (.TMD) is created. When repair finishes, this file is renamed to the original .MYD file. The problem was that during this rename, we copied the stats from the old file to the new file with chmod/chown. If a user managed to replace the temporary file before chmod/chown was executed, it was possible to get an arbitrary file with the privileges of the mysql user. This patch fixes the problem by not copying stats from the old file to the new file. This is not needed as the new file was created with the correct stats. This fix only changes server behavior - external utilities such as myisamchk still does chmod/chown. No test case provided since the problem involves synchronization with file system operations. --- storage/myisam/ha_myisam.cc | 26 +++++++++++++++++++++----- storage/myisam/mi_check.c | 41 +++++++++++++++++++++++++++++------------ storage/myisam/myisamchk.c | 16 ++++++++++------ 3 files changed, 60 insertions(+), 23 deletions(-) (limited to 'storage') diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 602a0ae6cc1..21cbef32188 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1091,24 +1091,36 @@ int ha_myisam::repair(THD *thd, MI_CHECK ¶m, bool do_optimize) /* TODO: respect myisam_repair_threads variable */ my_snprintf(buf, 40, "Repair with %d threads", my_count_bits(key_map)); thd_proc_info(thd, buf); + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ error = mi_repair_parallel(¶m, file, fixed_name, - param.testflag & T_QUICK); + param.testflag & T_QUICK, TRUE); thd_proc_info(thd, "Repair done"); // to reset proc_info, as // it was pointing to local buffer } else { thd_proc_info(thd, "Repair by sorting"); + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ error = mi_repair_by_sort(¶m, file, fixed_name, - param.testflag & T_QUICK); + param.testflag & T_QUICK, TRUE); } } else { thd_proc_info(thd, "Repair with keycache"); param.testflag &= ~T_REP_BY_SORT; + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ error= mi_repair(¶m, file, fixed_name, - param.testflag & T_QUICK); + param.testflag & T_QUICK, TRUE); } #ifdef HAVE_MMAP if (remap) @@ -1124,7 +1136,11 @@ int ha_myisam::repair(THD *thd, MI_CHECK ¶m, bool do_optimize) { optimize_done=1; thd_proc_info(thd, "Sorting index"); - error=mi_sort_index(¶m,file,fixed_name); + /* + The new file is created with the right stats, so we can skip + copying file stats from old to new. + */ + error=mi_sort_index(¶m,file,fixed_name, TRUE); } if (!statistics_done && (local_testflag & T_STATISTICS)) { diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index ba1f975549a..fe0d4c9c30b 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -1,5 +1,5 @@ /* - Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1512,7 +1512,7 @@ static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force) /* Save new datafile-name in temp_filename */ int mi_repair(MI_CHECK *param, register MI_INFO *info, - char * name, int rep_quick) + char * name, int rep_quick, my_bool no_copy_stat) { int error,got_error; ha_rows start_records,new_header_length; @@ -1726,6 +1726,11 @@ err: /* Replace the actual file with the temporary file */ if (new_file >= 0) { + myf flags= 0; + if (param->testflag & T_BACKUP_DATA) + flags |= MY_REDEL_MAKE_BACKUP; + if (no_copy_stat) + flags |= MY_REDEL_NO_COPY_STAT; mysql_file_close(new_file, MYF(0)); info->dfile=new_file= -1; /* @@ -1744,8 +1749,7 @@ err: info->s->file_map= NULL; } if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT, - (param->testflag & T_BACKUP_DATA ? - MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + flags) || mi_open_datafile(info,share,name,-1)) got_error=1; @@ -1933,7 +1937,8 @@ int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file) /* Sort index for more efficent reads */ -int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name) +int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name, + my_bool no_copy_stat) { reg2 uint key; reg1 MI_KEYDEF *keyinfo; @@ -2004,7 +2009,7 @@ int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name) share->kfile = -1; (void) mysql_file_close(new_file, MYF(MY_WME)); if (change_to_newfile(share->index_file_name, MI_NAME_IEXT, INDEX_TMP_EXT, - MYF(0)) || + no_copy_stat ? MYF(MY_REDEL_NO_COPY_STAT) : MYF(0)) || mi_open_keyfile(share)) goto err2; info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */ @@ -2209,6 +2214,8 @@ err: info MyISAM handler to repair name Name of table (for warnings) rep_quick set to <> 0 if we should not change data file + no_copy_stat Don't copy file stats from old to new file, + assume that new file was created with correct stats RESULT 0 ok @@ -2216,7 +2223,7 @@ err: */ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, - const char * name, int rep_quick) + const char * name, int rep_quick, my_bool no_copy_stat) { int got_error; uint i; @@ -2543,11 +2550,15 @@ err: /* Replace the actual file with the temporary file */ if (new_file >= 0) { + myf flags= 0; + if (param->testflag & T_BACKUP_DATA) + flags |= MY_REDEL_MAKE_BACKUP; + if (no_copy_stat) + flags |= MY_REDEL_NO_COPY_STAT; mysql_file_close(new_file, MYF(0)); info->dfile=new_file= -1; if (change_to_newfile(share->data_file_name,MI_NAME_DEXT, DATA_TMP_EXT, - (param->testflag & T_BACKUP_DATA ? - MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + flags) || mi_open_datafile(info,share,name,-1)) got_error=1; } @@ -2595,6 +2606,8 @@ err: info MyISAM handler to repair name Name of table (for warnings) rep_quick set to <> 0 if we should not change data file + no_copy_stat Don't copy file stats from old to new file, + assume that new file was created with correct stats DESCRIPTION Same as mi_repair_by_sort but do it multithreaded @@ -2629,7 +2642,7 @@ err: */ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, - const char * name, int rep_quick) + const char * name, int rep_quick, my_bool no_copy_stat) { int got_error; uint i,key, total_key_length, istep; @@ -3076,11 +3089,15 @@ err: /* Replace the actual file with the temporary file */ if (new_file >= 0) { + myf flags= 0; + if (param->testflag & T_BACKUP_DATA) + flags |= MY_REDEL_MAKE_BACKUP; + if (no_copy_stat) + flags |= MY_REDEL_NO_COPY_STAT; mysql_file_close(new_file, MYF(0)); info->dfile=new_file= -1; if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT, - (param->testflag & T_BACKUP_DATA ? - MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || + flags) || mi_open_datafile(info,share,name,-1)) got_error=1; } diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c index 8606bd7c748..9360a054872 100644 --- a/storage/myisam/myisamchk.c +++ b/storage/myisam/myisamchk.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -993,14 +993,18 @@ static int myisamchk(MI_CHECK *param, char * filename) info->s->state.key_map, param->force_sort)) { + /* + The new file might not be created with the right stats depending + on how myisamchk is run, so we must copy file stats from old to new. + */ if (param->testflag & T_REP_BY_SORT) - error=mi_repair_by_sort(param,info,filename,rep_quick); + error= mi_repair_by_sort(param, info, filename, rep_quick, FALSE); else - error=mi_repair_parallel(param,info,filename,rep_quick); + error= mi_repair_parallel(param, info, filename, rep_quick, FALSE); state_updated=1; } else if (param->testflag & T_REP_ANY) - error=mi_repair(param, info,filename,rep_quick); + error= mi_repair(param, info, filename, rep_quick, FALSE); } if (!error && param->testflag & T_SORT_RECORDS) { @@ -1040,12 +1044,12 @@ static int myisamchk(MI_CHECK *param, char * filename) { if (param->verbose) puts("Table had a compressed index; We must now recreate the index"); - error=mi_repair_by_sort(param,info,filename,1); + error= mi_repair_by_sort(param, info, filename, 1, FALSE); } } } if (!error && param->testflag & T_SORT_INDEX) - error=mi_sort_index(param,info,filename); + error= mi_sort_index(param, info, filename, FALSE); if (!error) share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR); -- cgit v1.2.1 From 65febcce97ebe2da0c9723b76a041e249b053a98 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 27 Sep 2016 14:09:54 +0300 Subject: Fix Bug#24707869 GCC 5 AND 6 MISCOMPILE MACH_PARSE_COMPRESSED Prevent GCC from moving a mach_read_from_4() before we have checked that we have 4 bytes to read. The pointer may only point to a 1, 2 or 3 bytes in which case the code should not read 4 bytes. This is a workaround to a GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673 Patch submitted by: Laurynas Biveinis RB: 14135 Reviewed by: Pawel Olchawa --- storage/innobase/mach/mach0data.c | 53 +++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 11 deletions(-) (limited to 'storage') diff --git a/storage/innobase/mach/mach0data.c b/storage/innobase/mach/mach0data.c index 95b135b0954..9669516244d 100644 --- a/storage/innobase/mach/mach0data.c +++ b/storage/innobase/mach/mach0data.c @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,8 +55,22 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xC0UL) { + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } @@ -64,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xE0UL) { + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -73,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -81,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } -- cgit v1.2.1 From 5884aa15d40b4dcc6de5cbcf276200c5fcbac938 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 6 Nov 2016 14:57:27 +0100 Subject: - Fix MDEV-11234. Escape quoting character. Should be doubled. Now it is also possible to escape it by a backslash. modified: storage/connect/tabfmt.cpp - Prepare making VEC table type support conditional. VEC tables might be unsupported in future versions modified: storage/connect/CMakeLists.txt modified: storage/connect/mycat.cc modified: storage/connect/reldef.cpp modified: storage/connect/xindex.cpp - MDEV-11067 suggested to add configuration support to the Apache wrapper. Was added but commented out until prooved it is really useful. modified: storage/connect/ApacheInterface.java modified: storage/connect/ha_connect.cc modified: storage/connect/jdbccat.h modified: storage/connect/jdbconn.cpp modified: storage/connect/jdbconn.h modified: storage/connect/tabjdbc.cpp modified: storage/connect/tabjdbc.h - Remove useless members. modified: storage/connect/jdbconn.cpp modified: storage/connect/jdbconn.h - New UDF countin. modified: storage/connect/jsonudf.cpp modified: storage/connect/jsonudf.h --- storage/connect/ApacheInterface.java | 5 ++- storage/connect/CMakeLists.txt | 2 +- storage/connect/ha_connect.cc | 7 +++-- storage/connect/jdbccat.h | 1 + storage/connect/jdbconn.cpp | 60 +++++++++++++++++++++--------------- storage/connect/jdbconn.h | 6 ++-- storage/connect/jsonudf.cpp | 47 ++++++++++++++++++++++++++++ storage/connect/jsonudf.h | 3 ++ storage/connect/mycat.cc | 6 +++- storage/connect/reldef.cpp | 13 +++++--- storage/connect/tabfmt.cpp | 34 +++++++++++++------- storage/connect/tabjdbc.cpp | 4 +++ storage/connect/tabjdbc.h | 1 + storage/connect/xindex.cpp | 8 +++-- 14 files changed, 147 insertions(+), 50 deletions(-) (limited to 'storage') diff --git a/storage/connect/ApacheInterface.java b/storage/connect/ApacheInterface.java index b4c8a4e9885..47b46dc0506 100644 --- a/storage/connect/ApacheInterface.java +++ b/storage/connect/ApacheInterface.java @@ -35,7 +35,10 @@ public class ApacheInterface extends JdbcInterface { ds.setPassword(parms[3]); pool.put(url, ds); } // endif ds - + + // if (parms.length > 4 && parms[4] != null) + // ds.setConnectionProperties(parms[4]); + // Get a connection from the data source conn = ds.getConnection(); diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 95d88538119..2122e56134b 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -38,7 +38,7 @@ user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT -DVCT_SUPPORT ) # diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index cf945a73f46..2222e51b083 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,7 +171,7 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0008 August 10, 2016"; + char version[]= "Version 1.04.0008 October 20, 2016"; #if defined(__WIN__) char compver[]= "Version 1.04.0008 " __DATE__ " " __TIME__; char slash= '\\'; @@ -5190,7 +5190,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, PJPARM sjp= NULL; char *driver= NULL; char *url= NULL; - char *tabtyp = NULL; +//char *prop= NULL; + char *tabtyp= NULL; #endif // JDBC_SUPPORT uint tm, fnc= FNC_NO, supfnc= (FNC_NO | FNC_COL); bool bif, ok= false, dbf= false; @@ -5256,6 +5257,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JDBC_SUPPORT) driver= GetListOption(g, "Driver", topt->oplist, NULL); // url= GetListOption(g, "URL", topt->oplist, NULL); +// prop = GetListOption(g, "Properties", topt->oplist, NULL); tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); #endif // JDBC_SUPPORT mxe= atoi(GetListOption(g,"maxerr", topt->oplist, "0")); @@ -5366,6 +5368,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, jdef->SetName(create_info->alias); sjp= (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); sjp->Driver= driver; +// sjp->Properties = prop; sjp->Fsize= 0; sjp->Scrollable= false; diff --git a/storage/connect/jdbccat.h b/storage/connect/jdbccat.h index 37f33d7063d..7108aa376ce 100644 --- a/storage/connect/jdbccat.h +++ b/storage/connect/jdbccat.h @@ -8,6 +8,7 @@ typedef struct jdbc_parms { char *Url; // Driver URL char *User; // User connect info char *Pwd; // Password connect info +//char *Properties; // Connection property list //int Cto; // Connect timeout //int Qto; // Query timeout int Fsize; // Fetch size diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index dca9bd0eac4..b31e77bf1ff 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -525,10 +525,10 @@ JDBConn::JDBConn(PGLOBAL g, TDBJDBC *tdbp) m_Wrap = strcat(strcpy(wn, "wrappers/"), m_Wrap); } // endif m_Wrap - m_Driver = NULL; - m_Url = NULL; - m_User = NULL; - m_Pwd = NULL; +//m_Driver = NULL; +//m_Url = NULL; +//m_User = NULL; +//m_Pwd = NULL; m_Ncol = 0; m_Aff = 0; m_Rows = 0; @@ -772,7 +772,7 @@ bool JDBConn::GetJVM(PGLOBAL g) /***********************************************************************/ int JDBConn::Open(PJPARM sop) { - + int irc = RC_FX; bool err = false; jboolean jt = (trace > 0); PGLOBAL& g = m_G; @@ -865,30 +865,37 @@ int JDBConn::Open(PJPARM sop) switch (rc) { case JNI_OK: strcpy(g->Message, "VM successfully created"); + irc = RC_OK; break; case JNI_ERR: strcpy(g->Message, "Initialising JVM failed: unknown error"); - return RC_FX; + break; case JNI_EDETACHED: strcpy(g->Message, "Thread detached from the VM"); - return RC_FX; + break; case JNI_EVERSION: strcpy(g->Message, "JNI version error"); - return RC_FX; + break; case JNI_ENOMEM: strcpy(g->Message, "Not enough memory"); - return RC_FX; + break; case JNI_EEXIST: strcpy(g->Message, "VM already created"); - return RC_FX; + break; case JNI_EINVAL: strcpy(g->Message, "Invalid arguments"); - return RC_FX; + break; default: - sprintf(g->Message, "Unknown return code %d", rc); - return RC_FX; + sprintf(g->Message, "Unknown return code %d", (int)rc); + break; } // endswitch rc + if (trace) + htrc("%s\n", g->Message); + + if (irc != RC_OK) + return irc; + //=============== Display JVM version =============== jint ver = env->GetVersion(); printf("JVM Version %d.%d\n", ((ver>>16)&0x0f), (ver&0x0f)); @@ -978,10 +985,10 @@ int JDBConn::Open(PJPARM sop) jobjectArray parms = env->NewObjectArray(4, // constructs java array of 4 env->FindClass("java/lang/String"), NULL); // Strings - m_Driver = sop->Driver; - m_Url = sop->Url; - m_User = sop->User; - m_Pwd = sop->Pwd; +//m_Driver = sop->Driver; +//m_Url = sop->Url; +//m_User = sop->User; +//m_Pwd = sop->Pwd; m_Scrollable = sop->Scrollable; m_RowsetSize = sop->Fsize; //m_LoginTimeout = sop->Cto; @@ -989,17 +996,20 @@ int JDBConn::Open(PJPARM sop) //m_UseCnc = sop->UseCnc; // change some elements - if (m_Driver) - env->SetObjectArrayElement(parms, 0, env->NewStringUTF(m_Driver)); + if (sop->Driver) + env->SetObjectArrayElement(parms, 0, env->NewStringUTF(sop->Driver)); + + if (sop->Url) + env->SetObjectArrayElement(parms, 1, env->NewStringUTF(sop->Url)); - if (m_Url) - env->SetObjectArrayElement(parms, 1, env->NewStringUTF(m_Url)); + if (sop->User) + env->SetObjectArrayElement(parms, 2, env->NewStringUTF(sop->User)); - if (m_User) - env->SetObjectArrayElement(parms, 2, env->NewStringUTF(m_User)); + if (sop->Pwd) + env->SetObjectArrayElement(parms, 3, env->NewStringUTF(sop->Pwd)); - if (m_Pwd) - env->SetObjectArrayElement(parms, 3, env->NewStringUTF(m_Pwd)); +//if (sop->Properties) +// env->SetObjectArrayElement(parms, 4, env->NewStringUTF(sop->Properties)); // call method rc = env->CallIntMethod(job, cid, parms, m_RowsetSize, m_Scrollable); diff --git a/storage/connect/jdbconn.h b/storage/connect/jdbconn.h index 0a1c52d4576..9d428142839 100644 --- a/storage/connect/jdbconn.h +++ b/storage/connect/jdbconn.h @@ -180,9 +180,9 @@ protected: char *Msg; char *m_Wrap; char m_IDQuoteChar[2]; - PSZ m_Driver; - PSZ m_Url; - PSZ m_User; +//PSZ m_Driver; +//PSZ m_Url; +//PSZ m_User; PSZ m_Pwd; int m_Ncol; int m_Aff; diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 8bddc68e2ae..f9034f25739 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -5264,3 +5264,50 @@ char *envar(UDF_INIT *initid, UDF_ARGS *args, char *result, return str; } // end of envar +/*********************************************************************************/ +/* Returns the distinct number of B occurences in A. */ +/*********************************************************************************/ +my_bool countin_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + if (args->arg_count != 2) { + strcpy(message, "This function must have 2 arguments"); + return true; + } else if (args->arg_type[0] != STRING_RESULT) { + strcpy(message, "First argument must be string"); + return true; + } else if (args->arg_type[1] != STRING_RESULT) { + strcpy(message, "Second argument is not a string"); + return true; + } // endif args + + return false; +} // end of countin_init + +long long countin(UDF_INIT *initid, UDF_ARGS *args, char *result, + unsigned long *res_length, char *is_null, char *) +{ + PSZ str1, str2; + char *s; + long long n = 0; + size_t lg; + + lg = (size_t)args->lengths[0]; + s = str1 = (PSZ)malloc(lg + 1); + memcpy(str1, args->args[0], lg); + str1[lg] = 0; + + lg = (size_t)args->lengths[1]; + str2 = (PSZ)malloc(lg + 1); + memcpy(str2, args->args[1], lg); + str2[lg] = 0; + + while (s = strstr(s, str2)) { + n++; + s += lg; + } // endwhile + + free(str1); + free(str2); + return n; +} // end of countin + diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index 1406d9f2f2e..d2890421c62 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -221,6 +221,9 @@ extern "C" { DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); + + DllExport my_bool countin_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport long long countin(UDF_EXEC_ARGS); } // extern "C" diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index b4b03e6ba4a..19c9f62b5bf 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -64,7 +64,9 @@ #include "filamtxt.h" #include "tabdos.h" #include "tabfmt.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT #include "tabsys.h" #if defined(__WIN__) #include "tabmac.h" @@ -549,7 +551,9 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #if defined(XML_SUPPORT) case TAB_XML: tdp= new(g) XMLDEF; break; #endif // XML_SUPPORT - case TAB_VEC: tdp= new(g) VCTDEF; break; +#if defined(VCT_SUPPORT) + case TAB_VEC: tdp = new(g)VCTDEF; break; +#endif // VCT_SUPPORT #if defined(ODBC_SUPPORT) case TAB_ODBC: tdp= new(g) ODBCDEF; break; #endif // ODBC_SUPPORT diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index ac2327212e0..a62fcbf9416 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -40,7 +40,9 @@ #include "tabcol.h" #include "filamap.h" #include "filamfix.h" +#if defined(VCT_SUPPORT) #include "filamvct.h" +#endif // VCT_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -683,16 +685,19 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) MPXFAM(defp); else txfp = new(g) FIXFAM(defp); - } else if (rfm == RECFM_VCT) { - assert (Pxdef->GetDefType() == TYPE_AM_VCT); +#if defined(VCT_SUPPORT) + assert(Pxdef->GetDefType() == TYPE_AM_VCT); if (map) txfp = new(g) VCMFAM((PVCTDEF)defp); else txfp = new(g) VCTFAM((PVCTDEF)defp); - - } // endif's +#else // !VCT_SUPPORT + strcpy(g->Message, "VCT no more supported"); + return NULL; +#endif // !VCT_SUPPORT + } // endif's ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Txfp diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 015f8d93b15..d21a8b977da 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -830,8 +830,9 @@ bool TDBCSV::SkipHeader(PGLOBAL g) /***********************************************************************/ int TDBCSV::ReadBuffer(PGLOBAL g) { - char *p1, *p2, *p = NULL; - int i, n, len, rc = Txfp->ReadBuffer(g); + //char *p1, *p2, *p = NULL; + char *p2, *p = NULL; + int i, n, len, rc = Txfp->ReadBuffer(g); bool bad = false; if (trace > 1) @@ -846,14 +847,23 @@ int TDBCSV::ReadBuffer(PGLOBAL g) for (i = 0; i < Fields; i++) { if (!bad) { if (Qot && *p2 == Qot) { // Quoted field - for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) - if (*(p + 1) == Qot) - n++; // Doubled internal quotes - else - break; // Final quote + //for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) + // if (*(p + 1) == Qot) + // n++; // Doubled internal quotes + // else + // break; // Final quote + + for (n = 0, p = ++p2; p; p++) + if (*p == Qot || *p == '\\') { + if (*(++p) == Qot) + n++; // Escaped internal quotes + else if (*(p - 1) == Qot) + break; // Final quote + } // endif *p if (p) { - len = p++ - p2; + //len = p++ - p2; + len = p - p2 - 1;; // if (Sep != ' ') // for (; *p == ' '; p++) ; // Skip blanks @@ -873,10 +883,12 @@ int TDBCSV::ReadBuffer(PGLOBAL g) if (n) { int j, k; - // Suppress the double of internal quotes + // Suppress the escape of internal quotes for (j = k = 0; j < len; j++, k++) { - if (p2[j] == Qot) - j++; // skip first one + if (p2[j] == Qot || (p2[j] == '\\' && p2[j + 1] == Qot)) + j++; // skip escape char + else if (p2[j] == '\\') + p2[k++] = p2[j++]; // avoid \\Qot p2[k] = p2[j]; } // endfor i, j diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index e398523892f..912e6c7d530 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -110,6 +110,7 @@ bool JDBCDEF::SetParms(PJPARM sjp) sjp->Url= Url; sjp->User= Username; sjp->Pwd= Password; +//sjp->Properties = Prop; return true; } // end of SetParms @@ -234,6 +235,7 @@ bool JDBCDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Read_Only = true; Wrapname = GetStringCatInfo(g, "Wrapper", NULL); +//Prop = GetStringCatInfo(g, "Properties", NULL); Tabcat = GetStringCatInfo(g, "Qualifier", NULL); Tabcat = GetStringCatInfo(g, "Catalog", Tabcat); Tabschema = GetStringCatInfo(g, "Dbname", NULL); @@ -337,6 +339,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Schema = tdp->Tabschema; Ops.User = tdp->Username; Ops.Pwd = tdp->Password; +// Ops.Properties = tdp->Prop; Catalog = tdp->Tabcat; Srcdef = tdp->Srcdef; Qrystr = tdp->Qrystr; @@ -356,6 +359,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Ops.Url = NULL; Ops.User = NULL; Ops.Pwd = NULL; +// Ops.Properties = NULL; Catalog = NULL; Srcdef = NULL; Qrystr = NULL; diff --git a/storage/connect/tabjdbc.h b/storage/connect/tabjdbc.h index 7244ebd3832..fee8223abaf 100644 --- a/storage/connect/tabjdbc.h +++ b/storage/connect/tabjdbc.h @@ -58,6 +58,7 @@ protected: PSZ Tabschema; /* External table schema */ PSZ Username; /* User connect name */ PSZ Password; /* Password connect info */ +//PSZ Prop; /* Connection Properties */ PSZ Tabcat; /* External table catalog */ PSZ Tabtype; /* External table type */ PSZ Colpat; /* Catalog column pattern */ diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 56312630278..a2cf4e77b80 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -45,7 +45,9 @@ //nclude "array.h" #include "filamtxt.h" #include "tabdos.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT /***********************************************************************/ /* Macro or external routine definition */ @@ -293,9 +295,11 @@ bool XINDEX::AddColumns(void) return false; // Not applying to static index else if (IsMul()) return false; // Not done yet for multiple index - else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) +#if defined(VCT_SUPPORT) + else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) return false; // This would require to read additional files - else +#endif // VCT_SUPPORT + else return true; } // end of AddColumns -- cgit v1.2.1 From aae67535cc399c92cac24b2b1f44e9a196806c9f Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 14 Nov 2016 19:20:40 +0100 Subject: - MDEV-11051 place Java classes ApacheInterface and JdbcInterface into single jar file. Try to fix the INSTALL command. modified: storage/connect/CMakeLists.txt - Make some JDBC tests available on Windows modified: storage/connect/mysql-test/connect/t/jdbc.test modified: storage/connect/mysql-test/connect/t/jdbc_new.test added: storage/connect/mysql-test/connect/t/windows.inc --- storage/connect/CMakeLists.txt | 31 ++++-- storage/connect/JavaWrappers.jar | Bin 0 -> 19615 bytes storage/connect/JdbcInterface.java | 13 +++ storage/connect/jdbconn.cpp | 118 +++++++++++---------- storage/connect/mysql-test/connect/disabled.def | 5 +- .../connect/mysql-test/connect/r/jdbc_new.result | 6 +- .../mysql-test/connect/std_data/JdbcMariaDB.jar | Bin 5993273 -> 6021866 bytes storage/connect/mysql-test/connect/t/jdbc.test | 1 + storage/connect/mysql-test/connect/t/jdbc_new.test | 1 + storage/connect/mysql-test/connect/t/windows.inc | 5 + 10 files changed, 109 insertions(+), 71 deletions(-) create mode 100644 storage/connect/JavaWrappers.jar create mode 100644 storage/connect/mysql-test/connect/t/windows.inc (limited to 'storage') diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 2122e56134b..46c4841ff97 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -20,25 +20,25 @@ SET(CONNECT_SOURCES ha_connect.cc connect.cc user_connect.cc mycat.cc fmdlex.c osutil.c plugutil.c rcmsg.c rcmsg.h array.cpp blkfil.cpp colblk.cpp csort.cpp -filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamvct.cpp filamzip.cpp +filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamzip.cpp filter.cpp json.cpp jsonudf.cpp maputil.cpp myconn.cpp myutil.cpp plgdbutl.cpp reldef.cpp tabcol.cpp tabdos.cpp tabfix.cpp tabfmt.cpp tabjson.cpp table.cpp tabmul.cpp tabmysql.cpp taboccur.cpp tabpivot.cpp tabsys.cpp tabtbl.cpp tabutil.cpp -tabvct.cpp tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp +tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp array.h blkfil.h block.h catalog.h checklvl.h colblk.h connect.h csort.h -engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamvct.h filamzip.h +engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamzip.h filter.h global.h ha_connect.h inihandl.h json.h jsonudf.h maputil.h msgid.h mycat.h myconn.h myutil.h os.h osutil.h plgcnx.h plgdbsem.h preparse.h reldef.h resource.h tabcol.h tabdos.h tabfix.h tabfmt.h tabjson.h tabmul.h tabmysql.h -taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvct.h tabvir.h tabxcl.h +taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvir.h tabxcl.h user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT -DVCT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) # @@ -89,6 +89,18 @@ ELSE(NOT UNIX) ENDIF(UNIX) +# +# VCT: the VEC format might be not supported in future versions +# + +OPTION(CONNECT_WITH_VCT "Compile CONNECT storage engine with VCT support" ON) + +IF(CONNECT_WITH_VCT) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamvct.cpp tabvct.cpp filamvct.h tabvct.h) + add_definitions(-DVCT_SUPPORT) +ENDIF(CONNECT_WITH_VCT) + + # # XML # @@ -236,9 +248,9 @@ ENDIF(CONNECT_WITH_ODBC) # JDBC # IF(APPLE) - OPTION(CONNECT_WITH_JDBC "some comment" OFF) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine without JDBC support" OFF) ELSE() - OPTION(CONNECT_WITH_JDBC "some comment" ON) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine with JDBC support" ON) ENDIF() IF(CONNECT_WITH_JDBC) @@ -252,12 +264,15 @@ IF(CONNECT_WITH_JDBC) SET(CONNECT_SOURCES ${CONNECT_SOURCES} jdbconn.cpp tabjdbc.cpp jdbconn.h tabjdbc.h jdbccat.h JdbcInterface.java ApacheInterface.java MariadbInterface.java - MysqlInterface.java OracleInterface.java PostgresqlInterface.java) + MysqlInterface.java OracleInterface.java PostgresqlInterface.java + JavaWrappers.jar) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() SET(JDBC_LIBRARY "") diff --git a/storage/connect/JavaWrappers.jar b/storage/connect/JavaWrappers.jar new file mode 100644 index 00000000000..d5353d2cbfd Binary files /dev/null and b/storage/connect/JavaWrappers.jar differ diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index 34af8c4e013..07dcaf985b4 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -220,6 +220,19 @@ public class JdbcInterface { } // end of SetTimestampParm + public int SetNullParm(int i, int typ) { + int rc = 0; + + try { + pstmt.setNull(i, typ); + } catch (Exception e) { + SetErrmsg(e); + rc = -1; + } // end try/catch + + return rc; + } // end of SetNullParm + public int ExecutePrep() { int n = -3; diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index b31e77bf1ff..a69f84a94a1 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -55,9 +55,8 @@ #if defined(__WIN__) extern "C" HINSTANCE s_hModule; // Saved module handle -#else // !__WIN__ +#endif // __WIN__ #define nullptr 0 -#endif // !__WIN__ TYPCONV GetTypeConv(); int GetConvSize(); @@ -1442,7 +1441,7 @@ bool JDBConn::SetParam(JDBCCOL *colp) PGLOBAL& g = m_G; bool rc = false; PVAL val = colp->GetValue(); - jint n, i = (jint)colp->GetRank(); + jint n, jrc = 0, i = (jint)colp->GetRank(); jshort s; jlong lg; //jfloat f; @@ -1452,69 +1451,74 @@ bool JDBConn::SetParam(JDBCCOL *colp) jstring jst = nullptr; jmethodID dtc, setid = nullptr; - switch (val->GetType()) { - case TYPE_STRING: - if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) + if (val->GetNullable() && val->IsNull()) { + if (gmID(g, setid, "SetNullParm", "(II)I")) return true; - jst = env->NewStringUTF(val->GetCharValue()); - env->CallVoidMethod(job, setid, i, jst); - break; - case TYPE_INT: - if (gmID(g, setid, "SetIntParm", "(II)V")) - return true; - - n = (jint)val->GetIntValue(); - env->CallVoidMethod(job, setid, i, n); - break; - case TYPE_TINY: - case TYPE_SHORT: - if (gmID(g, setid, "SetShortParm", "(IS)V")) - return true; + jrc = env->CallIntMethod(job, setid, i, (jint)GetJDBCType(val->GetType())); + } else switch (val->GetType()) { + case TYPE_STRING: + if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) + return true; - s = (jshort)val->GetShortValue(); - env->CallVoidMethod(job, setid, i, s); - break; - case TYPE_BIGINT: - if (gmID(g, setid, "SetBigintParm", "(IJ)V")) - return true; + jst = env->NewStringUTF(val->GetCharValue()); + env->CallVoidMethod(job, setid, i, jst); + break; + case TYPE_INT: + if (gmID(g, setid, "SetIntParm", "(II)V")) + return true; - lg = (jlong)val->GetBigintValue(); - env->CallVoidMethod(job, setid, i, lg); - break; - case TYPE_DOUBLE: - case TYPE_DECIM: - if (gmID(g, setid, "SetDoubleParm", "(ID)V")) - return true; + n = (jint)val->GetIntValue(); + env->CallVoidMethod(job, setid, i, n); + break; + case TYPE_TINY: + case TYPE_SHORT: + if (gmID(g, setid, "SetShortParm", "(IS)V")) + return true; - d = (jdouble)val->GetFloatValue(); - env->CallVoidMethod(job, setid, i, d); - break; - case TYPE_DATE: - if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { - strcpy(g->Message, "Cannot find Timestamp class"); - return true; - } else if (!(dtc = env->GetMethodID(dat, "", "(J)V"))) { - strcpy(g->Message, "Cannot find Timestamp class constructor"); - return true; - } // endif's + s = (jshort)val->GetShortValue(); + env->CallVoidMethod(job, setid, i, s); + break; + case TYPE_BIGINT: + if (gmID(g, setid, "SetBigintParm", "(IJ)V")) + return true; - lg = (jlong)val->GetBigintValue() * 1000; + lg = (jlong)val->GetBigintValue(); + env->CallVoidMethod(job, setid, i, lg); + break; + case TYPE_DOUBLE: + case TYPE_DECIM: + if (gmID(g, setid, "SetDoubleParm", "(ID)V")) + return true; - if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { - strcpy(g->Message, "Cannot make Timestamp object"); - return true; - } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + d = (jdouble)val->GetFloatValue(); + env->CallVoidMethod(job, setid, i, d); + break; + case TYPE_DATE: + if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { + strcpy(g->Message, "Cannot find Timestamp class"); + return true; + } else if (!(dtc = env->GetMethodID(dat, "", "(J)V"))) { + strcpy(g->Message, "Cannot find Timestamp class constructor"); + return true; + } // endif's + + lg = (jlong)val->GetBigintValue() * 1000; + + if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { + strcpy(g->Message, "Cannot make Timestamp object"); + return true; + } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + return true; + + env->CallVoidMethod(job, setid, i, datobj); + break; + default: + sprintf(g->Message, "Parm type %d not supported", val->GetType()); return true; + } // endswitch Type - env->CallVoidMethod(job, setid, i, datobj); - break; - default: - sprintf(g->Message, "Parm type %d not supported", val->GetType()); - return true; - } // endswitch Type - - if (Check()) { + if (Check(jrc)) { sprintf(g->Message, "SetParam: col=%s msg=%s", colp->GetName(), Msg); rc = true; } // endif msg diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index 9b4570915c7..5e15e0806ba 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -9,8 +9,7 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -#json_udf_bin : broken upstream in --ps (fixed) -jdbc : Variable settings depend on machine configuration -jdbc_new : Variable settings depend on machine configuration +#jdbc : Variable settings depend on machine configuration +#jdbc_new : Variable settings depend on machine configuration jdbc_oracle : Variable settings depend on machine configuration jdbc_postgresql : Variable settings depend on machine configuration diff --git a/storage/connect/mysql-test/connect/r/jdbc_new.result b/storage/connect/mysql-test/connect/r/jdbc_new.result index 14381b0b11f..5cc4826213d 100644 --- a/storage/connect/mysql-test/connect/r/jdbc_new.result +++ b/storage/connect/mysql-test/connect/r/jdbc_new.result @@ -56,7 +56,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`='JDBC' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -72,7 +72,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC `TABNAME`='t1' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -104,7 +104,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC SELECT * FROM t1; a b -0 NULL +NULL NULL 0 0 1 0 2 0 diff --git a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar index 81f91e4465a..9d461ad6223 100644 Binary files a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar and b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar differ diff --git a/storage/connect/mysql-test/connect/t/jdbc.test b/storage/connect/mysql-test/connect/t/jdbc.test index 41fd298776b..58a527a3e6b 100644 --- a/storage/connect/mysql-test/connect/t/jdbc.test +++ b/storage/connect/mysql-test/connect/t/jdbc.test @@ -1,3 +1,4 @@ +-- source windows.inc -- source jdbconn.inc SET GLOBAL time_zone='+1:00'; diff --git a/storage/connect/mysql-test/connect/t/jdbc_new.test b/storage/connect/mysql-test/connect/t/jdbc_new.test index d1ad5117b72..5586cf8c027 100644 --- a/storage/connect/mysql-test/connect/t/jdbc_new.test +++ b/storage/connect/mysql-test/connect/t/jdbc_new.test @@ -5,6 +5,7 @@ connect (master,127.0.0.1,root,,test,$MASTER_MYPORT,); connect (slave,127.0.0.1,root,,test,$SLAVE_MYPORT,); connection master; +-- source windows.inc -- source jdbconn.inc connection slave; diff --git a/storage/connect/mysql-test/connect/t/windows.inc b/storage/connect/mysql-test/connect/t/windows.inc new file mode 100644 index 00000000000..88553d8aa59 --- /dev/null +++ b/storage/connect/mysql-test/connect/t/windows.inc @@ -0,0 +1,5 @@ +if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows") = 0`) +{ + skip Need windows; +} + -- cgit v1.2.1 From 2d78b25c49407fb3f17fd4e44b551e8b43ffa4b5 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 27 Nov 2016 14:42:37 +0100 Subject: - Fix null pointer java error when connecting to jdbc:drill driver. By setting the context class loader. modified: storage/connect/JavaWrappers.jar modified: storage/connect/JdbcInterface.java modified: storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar --- storage/connect/JavaWrappers.jar | Bin 19615 -> 19696 bytes storage/connect/JdbcInterface.java | 3 +++ .../mysql-test/connect/std_data/JdbcMariaDB.jar | Bin 6021866 -> 6021947 bytes 3 files changed, 3 insertions(+) (limited to 'storage') diff --git a/storage/connect/JavaWrappers.jar b/storage/connect/JavaWrappers.jar index d5353d2cbfd..8c01c364a3f 100644 Binary files a/storage/connect/JavaWrappers.jar and b/storage/connect/JavaWrappers.jar differ diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index 07dcaf985b4..a1b1360e6ea 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -82,6 +82,9 @@ public class JdbcInterface { System.out.println("URL=" + parms[1]); CheckURL(parms[1], null); + + // This is required for drivers using context class loaders + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); if (parms[2] != null && !parms[2].isEmpty()) { if (DEBUG) diff --git a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar index 9d461ad6223..8525da74018 100644 Binary files a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar and b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar differ -- cgit v1.2.1 From 599d8cc2deee615526838ddc962778e51cd3a15a Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 2 Dec 2016 23:03:43 +0100 Subject: - MDEV-11366 SIGBUS errors in Connect Storage Engine for ArmHF and MIPS. Fix includes launchpad fix plus more to cover writing BIN tables. modified: storage/connect/tabfix.cpp modified: storage/connect/value.cpp modified: storage/connect/value.h - Typo: Change the name of filamzip to filamgz to prepare future ZIP tables. modified: storage/connect/CMakeLists.txt added: storage/connect/filamgz.cpp added: storage/connect/filamgz.h deleted: storage/connect/filamzip.cpp deleted: storage/connect/filamzip.h modified: storage/connect/plgdbsem.h modified: storage/connect/reldef.cpp modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfix.cpp modified: storage/connect/tabfmt.cpp modified: storage/connect/tabjson.cpp --- storage/connect/CMakeLists.txt | 12 +- storage/connect/filamgz.cpp | 1426 ++++++++++++++++++++++++++++++++++++++++ storage/connect/filamgz.h | 170 +++++ storage/connect/filamzip.cpp | 1426 ---------------------------------------- storage/connect/filamzip.h | 170 ----- storage/connect/plgdbsem.h | 2 +- storage/connect/reldef.cpp | 16 +- storage/connect/tabdos.cpp | 48 +- storage/connect/tabdos.h | 2 +- storage/connect/tabfix.cpp | 19 +- storage/connect/tabfmt.cpp | 22 +- storage/connect/tabjson.cpp | 20 +- storage/connect/value.cpp | 35 +- storage/connect/value.h | 66 +- 14 files changed, 1740 insertions(+), 1694 deletions(-) create mode 100644 storage/connect/filamgz.cpp create mode 100644 storage/connect/filamgz.h delete mode 100644 storage/connect/filamzip.cpp delete mode 100644 storage/connect/filamzip.h (limited to 'storage') diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 46c4841ff97..f1567730b26 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -20,14 +20,14 @@ SET(CONNECT_SOURCES ha_connect.cc connect.cc user_connect.cc mycat.cc fmdlex.c osutil.c plugutil.c rcmsg.c rcmsg.h array.cpp blkfil.cpp colblk.cpp csort.cpp -filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamzip.cpp +filamap.cpp filamdbf.cpp filamfix.cpp filamgz.cpp filamtxt.cpp filter.cpp json.cpp jsonudf.cpp maputil.cpp myconn.cpp myutil.cpp plgdbutl.cpp reldef.cpp tabcol.cpp tabdos.cpp tabfix.cpp tabfmt.cpp tabjson.cpp table.cpp tabmul.cpp tabmysql.cpp taboccur.cpp tabpivot.cpp tabsys.cpp tabtbl.cpp tabutil.cpp tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp array.h blkfil.h block.h catalog.h checklvl.h colblk.h connect.h csort.h -engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamzip.h +engmsg.h filamap.h filamdbf.h filamfix.h filamgz.h filamtxt.h filter.h global.h ha_connect.h inihandl.h json.h jsonudf.h maputil.h msgid.h mycat.h myconn.h myutil.h os.h osutil.h plgcnx.h plgdbsem.h preparse.h reldef.h resource.h tabcol.h tabdos.h tabfix.h tabfmt.h tabjson.h tabmul.h tabmysql.h @@ -38,7 +38,7 @@ user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT -DPIVOT_SUPPORT ) # @@ -270,9 +270,9 @@ IF(CONNECT_WITH_JDBC) # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) - install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) - INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar - DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() SET(JDBC_LIBRARY "") diff --git a/storage/connect/filamgz.cpp b/storage/connect/filamgz.cpp new file mode 100644 index 00000000000..07242ea633c --- /dev/null +++ b/storage/connect/filamgz.cpp @@ -0,0 +1,1426 @@ +/************ File AM GZ C++ Program Source Code File (.CPP) ***********/ +/* PROGRAM NAME: FILAMGZ */ +/* ------------- */ +/* Version 1.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the ZLIB compressed files classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#if defined(__WIN__) +#include +#include +#if defined(__BORLANDC__) +#define __MFC_COMPAT__ // To define min/max as macro +#endif +//#include +#else // !__WIN__ +#if defined(UNIX) +#include +#else // !UNIX +#include +#endif +#include +#endif // !__WIN__ + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* tabdos.h is header containing the TABDOS class declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +//#include "catalog.h" +//#include "reldef.h" +//#include "xobject.h" +//#include "kindex.h" +#include "filamtxt.h" +#include "tabdos.h" +#if defined(UNIX) +#include "osutil.h" +#endif + +/***********************************************************************/ +/* This define prepares ZLIB function declarations. */ +/***********************************************************************/ +//#define ZLIB_DLL + +#include "filamgz.h" + +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +extern int num_read, num_there, num_eq[]; // Statistics + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the GZFAM class. */ +/***********************************************************************/ +GZFAM::GZFAM(PGZFAM txfp) : TXTFAM(txfp) + { + Zfile = txfp->Zfile; + Zpos = txfp->Zpos; + } // end of GZFAM copy constructor + +/***********************************************************************/ +/* Zerror: Error function for gz calls. */ +/* gzerror returns the error message for the last error which occurred*/ +/* on the given compressed file. errnum is set to zlib error number. */ +/* If an error occurred in the file system and not in the compression */ +/* library, errnum is set to Z_ERRNO and the application may consult */ +/* errno to get the exact error code. */ +/***********************************************************************/ +int GZFAM::Zerror(PGLOBAL g) + { + int errnum; + + strcpy(g->Message, gzerror(Zfile, &errnum)); + + if (errnum == Z_ERRNO) +#if defined(__WIN__) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); +#else // !__WIN__ + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#endif // !__WIN__ + + return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; + } // end of Zerror + +/***********************************************************************/ +/* Reset: reset position values at the beginning of file. */ +/***********************************************************************/ +void GZFAM::Reset(void) + { + TXTFAM::Reset(); +//gzrewind(Zfile); // Useful ????? + Zpos = 0; + } // end of Reset + +/***********************************************************************/ +/* GZ GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int GZFAM::GetFileLength(PGLOBAL g) + { + int len = TXTFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 6 + len *= 6; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* GZ Access Method opening routine. */ +/***********************************************************************/ +bool GZFAM::OpenTableFile(PGLOBAL g) + { + char opmode[4], filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + switch (mode) { + case MODE_READ: + strcpy(opmode, "r"); + break; + case MODE_UPDATE: + /*****************************************************************/ + /* Updating GZ files not implemented yet. */ + /*****************************************************************/ + strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); + return true; + case MODE_DELETE: + if (!Tdbp->GetNext()) { + // Store the number of deleted lines + DelRows = Cardinality(g); + + // This will erase the entire file + strcpy(opmode, "w"); +// Block = 0; // For ZBKFAM +// Last = Nrec; // For ZBKFAM + Tdbp->ResetSize(); + } else { + sprintf(g->Message, MSG(NO_PART_DEL), "GZ"); + return true; + } // endif filter + + break; + case MODE_INSERT: + strcpy(opmode, "a+"); + break; + default: + sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); + return true; + } // endswitch Mode + + /*********************************************************************/ + /* Open according to logical input/output mode required. */ + /* Use specific zlib functions. */ + /* Treat files as binary. */ + /*********************************************************************/ + strcat(opmode, "b"); + Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); + + if (Zfile == NULL) { + sprintf(g->Message, MSG(GZOPEN_ERROR), + opmode, (int)errno, filename); + strcat(strcat(g->Message, ": "), strerror(errno)); + return (mode == MODE_READ && errno == ENOENT) + ? PushWarning(g, Tdbp) : true; + } // endif Zfile + + /*********************************************************************/ + /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ + /*********************************************************************/ +//To_Fb = dbuserp->Openlist; // Keep track of File block + + /*********************************************************************/ + /* Allocate the line buffer. */ + /*********************************************************************/ + return AllocateBuffer(g); + } // end of OpenTableFile + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool GZFAM::AllocateBuffer(PGLOBAL g) + { + MODE mode = Tdbp->GetMode(); + + Buflen = Lrecl + 2; // Lrecl does not include CRLF +//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + + if (trace) + htrc("SubAllocating a buffer of %d bytes\n", Buflen); + + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (mode == MODE_INSERT) { + /*******************************************************************/ + /* For Insert buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + To_Buf[Buflen - 2] = '\n'; + To_Buf[Buflen - 1] = '\0'; + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int GZFAM::GetRowID(void) + { + return Rows; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int GZFAM::GetPos(void) + { + return (int)Zpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: return the position of next record. */ +/***********************************************************************/ +int GZFAM::GetNextPos(void) + { + return gztell(Zfile); + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool GZFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 + Fpos = pos; + + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return true; + } // endif + + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/***********************************************************************/ +bool GZFAM::RecordPos(PGLOBAL) + { + Zpos = gztell(Zfile); + return false; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int GZFAM::SkipRecord(PGLOBAL g, bool header) + { + // Skip this record + if (gzeof(Zfile)) + return RC_EF; + else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) + return Zerror(g); + + if (header) + RecordPos(g); + + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int GZFAM::ReadBuffer(PGLOBAL g) + { + char *p; + int rc; + + if (!Zfile) + return RC_EF; + + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + next: + if (RecordPos(g)) + return RC_FX; + + CurBlk = Rows++; // Update RowID + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; + + if (gzeof(Zfile)) { + rc = RC_EF; + } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { + p = To_Buf + strlen(To_Buf) - 1; + + if (*p == '\n') + *p = '\0'; // Eliminate ending new-line character + + if (*(--p) == '\r') + *p = '\0'; // Eliminate eventuel carriage return + + strcpy(Tdbp->GetLine(), To_Buf); + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + if (trace > 1) + htrc(" Read: '%s' rc=%d\n", To_Buf, rc); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int GZFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* Now start the writing process. */ + /*********************************************************************/ + if (gzputs(Zfile, To_Buf) < 0) + return Zerror(g); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZDOS access method. (NIY) */ +/***********************************************************************/ +int GZFAM::DeleteRecords(PGLOBAL g, int) + { + strcpy(g->Message, MSG(NO_ZIP_DELETE)); + return RC_FX; + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for DOS access method. */ +/***********************************************************************/ +void GZFAM::CloseTableFile(PGLOBAL, bool) + { + int rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for GZ access method. */ +/***********************************************************************/ +void GZFAM::Rewind(void) + { + gzrewind(Zfile); + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZBKFAM::ZBKFAM(PDOSDEF tdp) : GZFAM(tdp) + { + Blocked = true; + Block = tdp->GetBlock(); + Last = tdp->GetLast(); + Nrec = tdp->GetElemt(); + CurLine = NULL; + NxtLine = NULL; + Closing = false; + BlkPos = tdp->GetTo_Pos(); + } // end of ZBKFAM standard constructor + +ZBKFAM::ZBKFAM(PZBKFAM txfp) : GZFAM(txfp) + { + CurLine = txfp->CurLine; + NxtLine = txfp->NxtLine; + Closing = txfp->Closing; + } // end of ZBKFAM copy constructor + +/***********************************************************************/ +/* Use BlockTest to reduce the table estimated size. */ +/***********************************************************************/ +int ZBKFAM::MaxBlkSize(PGLOBAL g, int) + { + int rc = RC_OK, savcur = CurBlk; + int size; + + // Roughly estimate the table size as the sum of blocks + // that can contain good rows + for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; + + CurBlk = savcur; + return size; + } // end of MaxBlkSize + +/***********************************************************************/ +/* ZBK Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZBKFAM::Cardinality(PGLOBAL g) + { + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZBKFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Nrec * (Lrecl + 2); + CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetRowID(void) + { + return CurNum + Nrec * CurBlk + 1; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetPos(void) + { + return CurNum + Nrec * CurBlk; // Computed file index + } // end of GetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/* Not used yet for fixed tables. */ +/***********************************************************************/ +bool ZBKFAM::RecordPos(PGLOBAL /*g*/) + { +//strcpy(g->Message, "RecordPos not implemented for gz blocked tables"); +//return true; + return RC_OK; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) + { +//strcpy(g->Message, "SkipRecord not implemented for gz blocked tables"); +//return RC_FX; + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZBKFAM::ReadBuffer(PGLOBAL g) + { + int n, skip, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + while (*NxtLine++ != '\n') ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = CurBlk - skip; i < CurBlk; i++) { + BlkLen = BlkPos[i + 1] - BlkPos[i]; + + if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; + + if (!(n = gzread(Zfile, To_Buf, BlkLen))) { + rc = RC_EF; + } else if (n > 0) { + // Get the position of the current line + CurLine = To_Buf; + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZBKFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = CurLine + strlen(CurLine) - To_Buf; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + } else + CurLine += strlen(CurLine); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZBK access method. */ +/* Implemented only for total deletion of the table, which is done */ +/* by opening the file in mode "wb". */ +/***********************************************************************/ +int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) + { + if (irc == RC_EF) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + defp->SetBlock(0); + defp->SetLast(Nrec); + + if (!defp->SetIntCatInfo("Blocks", 0) || + !defp->SetIntCatInfo("Last", 0)) { + sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); + return RC_FX; + } else + return RC_OK; + + } else + return irc; + + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = true; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + gzclose(Zfile); + } else if (Tdbp->GetMode() == MODE_DELETE) { + rc = DeleteRecords(g, RC_EF); + gzclose(Zfile); + } else + rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::Rewind(void) + { + gzrewind(Zfile); + CurBlk = -1; + CurNum = Rbuf; + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) + { +//Block = tdp->GetBlock(); +//Last = tdp->GetLast(); + Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; + Blksize = Nrec * Lrecl; + } // end of ZIXFAM standard constructor + +/***********************************************************************/ +/* ZIX Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZIXFAM::Cardinality(PGLOBAL g) + { + if (Last) + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + else // Last and Block not defined, cannot do it yet + return 0; + + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZIXFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Blksize; + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + /*******************************************************************/ + /* For Insert the buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + + if (Tdbp->GetFtype() < 2) + // if not binary, the file is physically a text file + for (int len = Lrecl; len <= Buflen; len += Lrecl) { +#if defined(__WIN__) + To_Buf[len - 2] = '\r'; +#endif // __WIN__ + To_Buf[len - 1] = '\n'; + } // endfor len + + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZIXFAM::ReadBuffer(PGLOBAL g) + { + int n, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + Tdbp->IncLine(Lrecl); // Used by DOSCOL functions + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + Tdbp->SetLine(To_Buf); + + int skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = 0; i < skip; i++) { + if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + if (!(n = gzread(Zfile, To_Buf, Buflen))) { + rc = RC_EF; + } else if (n > 0) { + Rbuf = n / Lrecl; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZIXFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = Rbuf * Lrecl; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + Tdbp->SetLine(To_Buf); + } else + Tdbp->IncLine(Lrecl); // Used by FIXCOL functions + + return RC_OK; + } // end of WriteBuffer + +/* --------------------------- Class ZLBFAM -------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) + { + Zstream = NULL; + Zbuffer = NULL; + Zlenp = NULL; + Optimized = tdp->IsOptimized(); + } // end of ZLBFAM standard constructor + +ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) + { + Zstream = txfp->Zstream; + Zbuffer = txfp->Zbuffer; + Zlenp = txfp->Zlenp; + Optimized = txfp->Optimized; + } // end of ZLBFAM (dummy?) copy constructor + +/***********************************************************************/ +/* ZLB GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int ZLBFAM::GetFileLength(PGLOBAL g) + { + int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 5 + len *= 5; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZLBFAM::AllocateBuffer(PGLOBAL g) + { + char *msg; + int n, zrc; + +#if 0 + if (!Optimized && Tdbp->NeedIndexing(g)) { + strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); + return TRUE; + } // endif indexing +#endif // 0 + +#if defined(NOLIB) + if (!zlib && LoadZlib()) { + sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); + return TRUE; + } // endif zlib +#endif + + BLKFAM::AllocateBuffer(g); +//Buflen = Nrec * (Lrecl + 2); +//Rbuf = Nrec; + + // Allocate the compressed buffer + n = Buflen + 16; // ????????????????????????????????? + Zlenp = (int*)PlugSubAlloc(g, NULL, n); + Zbuffer = (Byte*)(Zlenp + 1); + + // Allocate and initialize the Z stream + Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); + Zstream->zalloc = (alloc_func)0; + Zstream->zfree = (free_func)0; + Zstream->opaque = (voidpf)0; + Zstream->next_in = NULL; + Zstream->avail_in = 0; + + if (Tdbp->GetMode() == MODE_READ) { + msg = "inflateInit"; + zrc = inflateInit(Zstream); + } else { + msg = "deflateInit"; + zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); + } // endif Mode + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, "%s error: %s", msg, Zstream->msg); + else + sprintf(g->Message, "%s error: %d", msg, zrc); + + return TRUE; + } // endif zrc + + if (Tdbp->GetMode() == MODE_INSERT) { + // Write the file header block + if (Last == Nrec) { + CurBlk = Block; + CurNum = 0; + + if (!GetFileLength(g)) { + // Write the zlib header as an extra block + strcpy(To_Buf, "PlugDB"); + BlkLen = strlen("PlugDB") + 1; + + if (WriteCompressedBuffer(g)) + return TRUE; + + } // endif void file + + } else { + // In mode insert, if Last != Nrec, last block must be updated + CurBlk = Block - 1; + CurNum = Last; + + strcpy(g->Message, MSG(NO_PAR_BLK_INS)); + return TRUE; + } // endif Last + + } else { // MODE_READ + // First thing to do is to read the header block + void *rdbuf; + + if (Optimized) { + BlkLen = BlkPos[0]; + rdbuf = Zlenp; + } else { + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) + return FALSE; // Empty file + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_EF: + return FALSE; + case RC_FX: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + case RC_NF: + return TRUE; + } // endswitch + + // Some old tables can have PlugDB in their header + if (strcmp(To_Buf, "PlugDB")) { + sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); + return TRUE; + } // endif strcmp + + } // endif Mode + + return FALSE; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZLBFAM::GetPos(void) + { + return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: should not be called for this class. */ +/***********************************************************************/ +int ZLBFAM::GetNextPos(void) + { + if (Optimized) { + assert(FALSE); + return 0; + } else + return ftell(Stream); + + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 // All this must be checked + if (pos < 0) { + strcpy(g->Message, MSG(INV_REC_POS)); + return true; + } // endif recpos + + CurBlk = pos / Nrec; + CurNum = pos % Nrec; +#if defined(_DEBUG) + num_eq[(CurBlk == OldBlk) ? 1 : 0]++; +#endif + + // Indicate the table position was externally set + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* ReadBuffer: Read one line for a text file. */ +/***********************************************************************/ +int ZLBFAM::ReadBuffer(PGLOBAL g) + { + int n; + void *rdbuf; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (Placed) { + Placed = FALSE; + } else if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + if (Tdbp->GetFtype() == RECFM_VAR) + while (*NxtLine++ != '\n') ; + else + NxtLine += Lrecl; + + // Set caller line buffer + n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) { + CurNum--; // To have a correct Last value when optimizing + return RC_EF; + } else { + /*******************************************************************/ + /* New block. */ + /*******************************************************************/ + CurNum = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + if (Optimized) switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + + } // endif's + + if (OldBlk == CurBlk) + goto ok; // Block is already there + + if (Optimized) { + // Store the position of next block + Fpos = BlkPos[CurBlk]; + + // fseek is required only in non sequential reading + if (CurBlk != OldBlk + 1) + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif fseek + + // Calculate the length of block to read + BlkLen = BlkPos[CurBlk + 1] - Fpos; + rdbuf = Zlenp; + } else { // !Optimized + if (CurBlk != OldBlk + 1) { + strcpy(g->Message, MSG(INV_RAND_ACC)); + return RC_FX; + } else + Fpos = ftell(Stream); // Used when optimizing + + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { + if (feof(Stream)) + return RC_EF; + + goto err; + } // endif fread + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + // Read the next block + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_FX: goto err; + case RC_NF: return RC_FX; + case RC_EF: return RC_EF; + default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + } // endswitch ReadCompressedBuffer + + ok: + if (Tdbp->GetFtype() == RECFM_VAR) { + int i; + + // Get the position of the current line + for (i = 0, CurLine = To_Buf; i < CurNum; i++) + while (*CurLine++ != '\n') ; // What about Unix ??? + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + } else { + CurLine = To_Buf + CurNum * Lrecl; + NxtLine = CurLine + Lrecl; + n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + } // endif Ftype + + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + + OldBlk = CurBlk; // Last block actually read + IsRead = TRUE; // Is read indeed + return RC_OK; + + err: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + return RC_FX; + } // end of ReadBuffer + +/***********************************************************************/ +/* Read and decompress a block from the stream. */ +/***********************************************************************/ +int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) + { + if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { + int zrc; + + num_read++; + + if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { + sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); + return RC_NF; + } // endif BlkLen + + // HERE WE MUST INFLATE THE BLOCK + Zstream->next_in = Zbuffer; + Zstream->avail_in = (uInt)(*Zlenp); + Zstream->next_out = (Byte*)To_Buf; + Zstream->avail_out = Buflen; + zrc = inflate(Zstream, Z_SYNC_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); + + return RC_NF; + } // endif zrc + + } else if (feof(Stream)) { + return RC_EF; + } else + return RC_FX; + + return RC_OK; + } // end of ReadCompressedBuffer + +/***********************************************************************/ +/* WriteBuffer: File write routine for DOS access method. */ +/* Update is directly written back into the file, */ +/* with this (fast) method, record size cannot change. */ +/***********************************************************************/ +int ZLBFAM::WriteBuffer(PGLOBAL g) + { + assert (Tdbp->GetMode() == MODE_INSERT); + + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) { + if (Tdbp->GetFtype() == RECFM_BIN) + memcpy(CurLine, Tdbp->GetLine(), Lrecl); + else + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + +#if defined(_DEBUG) + if (Tdbp->GetFtype() == RECFM_FIX && + (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { + strcpy(g->Message, MSG(BAD_LINE_LEN)); + Closing = TRUE; + return RC_FX; + } // endif Lrecl +#endif // _DEBUG + } // endif Closing + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /*********************************************************************/ + if (++CurNum != Rbuf) { + if (Tdbp->GetFtype() == RECFM_VAR) + CurLine += strlen(CurLine); + else + CurLine += Lrecl; + + return RC_OK; // We write only full blocks + } // endif CurNum + + // HERE WE MUST DEFLATE THE BLOCK + if (Tdbp->GetFtype() == RECFM_VAR) + NxtLine = CurLine + strlen(CurLine); + else + NxtLine = CurLine + Lrecl; + + BlkLen = NxtLine - To_Buf; + + if (WriteCompressedBuffer(g)) { + Closing = TRUE; // To tell CloseDB about a Write error + return RC_FX; + } // endif WriteCompressedBuffer + + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Compress the buffer and write the deflated output to stream. */ +/***********************************************************************/ +bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) + { + int zrc; + + Zstream->next_in = (Byte*)To_Buf; + Zstream->avail_in = (uInt)BlkLen; + Zstream->next_out = Zbuffer; + Zstream->avail_out = Buflen + 16; + Zstream->total_out = 0; + zrc = deflate(Zstream, Z_FULL_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); + + return TRUE; + } else + *Zlenp = Zstream->total_out; + + // Now start the writing process. + BlkLen = *Zlenp + sizeof(int); + + if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + return TRUE; + } // endif size + + return FALSE; + } // end of WriteCompressedBuffer + +/***********************************************************************/ +/* Table file close routine for DOS access method. */ +/***********************************************************************/ +void ZLBFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + // Closing is True if last Write was in error + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = TRUE; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + fclose(Stream); + } else + rc = fclose(Stream); + + if (trace) + htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", + To_File, Tdbp->GetMode(), rc); + + Stream = NULL; // So we can know whether table is open + To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + + if (Tdbp->GetMode() == MODE_READ) + rc = inflateEnd(Zstream); + else + rc = deflateEnd(Zstream); + + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZLIB access method. */ +/***********************************************************************/ +void ZLBFAM::Rewind(void) + { + // We must be positioned after the header block + if (CurBlk >= 0) { // Nothing to do if no block read yet + if (!Optimized) { // If optimized, fseek will be done in ReadBuffer + size_t st; + + rewind(Stream); + + if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) + htrc("fread error %d in Rewind", errno); + + fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); + OldBlk = -1; + } // endif Optimized + + CurBlk = -1; + CurNum = Rbuf; + } // endif CurBlk + +//OldBlk = -1; +//Rbuf = 0; commented out in case we reuse last read block + } // end of Rewind + +/* ------------------------ End of GzFam ---------------------------- */ diff --git a/storage/connect/filamgz.h b/storage/connect/filamgz.h new file mode 100644 index 00000000000..d667fdddcc2 --- /dev/null +++ b/storage/connect/filamgz.h @@ -0,0 +1,170 @@ +/*************** FilAmGz H Declares Source Code File (.H) **************/ +/* Name: FILAMGZ.H Version 1.3 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* This file contains the GZIP access method classes declares. */ +/***********************************************************************/ +#ifndef __FILAMGZ_H +#define __FILAMGZ_H + +#include "zlib.h" + +typedef class GZFAM *PGZFAM; +typedef class ZBKFAM *PZBKFAM; +typedef class ZIXFAM *PZIXFAM; +typedef class ZLBFAM *PZLBFAM; + +/***********************************************************************/ +/* This is the access method class declaration for not optimized */ +/* variable record length files compressed using the gzip library */ +/* functions. File is accessed record by record (row). */ +/***********************************************************************/ +class DllExport GZFAM : public TXTFAM { +// friend class DOSCOL; + public: + // Constructor + GZFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} + GZFAM(PGZFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_GZ;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) GZFAM(this);} + + // Methods + virtual void Reset(void); + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} + virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual bool OpenTableFile(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + int Zerror(PGLOBAL g); // GZ error function + + // Members + gzFile Zfile; // Points to GZ file structure + z_off_t Zpos; // Uncompressed file position + }; // end of class GZFAM + +/***********************************************************************/ +/* This is the access method class declaration for optimized variable */ +/* record length files compressed using the gzip library functions. */ +/* The File is accessed by block (requires an opt file). */ +/***********************************************************************/ +class DllExport ZBKFAM : public GZFAM { + public: + // Constructor + ZBKFAM(PDOSDEF tdp); + ZBKFAM(PZBKFAM txfp); + + // Implementation + virtual int GetPos(void); + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZBKFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int MaxBlkSize(PGLOBAL g, int s); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + // Members + char *CurLine; // Position of current line in buffer + char *NxtLine; // Position of Next line in buffer + bool Closing; // True when closing on Insert + }; // end of class ZBKFAM + +/***********************************************************************/ +/* This is the access method class declaration for fixed record */ +/* length files compressed using the gzip library functions. */ +/* The file is always accessed by block. */ +/***********************************************************************/ +class DllExport ZIXFAM : public ZBKFAM { + public: + // Constructor + ZIXFAM(PDOSDEF tdp); + ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} + + // Implementation + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZIXFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + + protected: + // No additional Members + }; // end of class ZIXFAM + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for PlugDB */ +/* fixed/variable files compressed using the zlib library functions. */ +/* Physically these are written and read using the same technique */ +/* than blocked variable files, only the contain of each block is */ +/* compressed using the deflate zlib function. The purpose of this */ +/* specific format is to have a fast mechanism for direct access of */ +/* records so blocked optimization is fast and direct access (joins) */ +/* is allowed. Note that the block length is written ahead of each */ +/* block to enable reading when optimization file is not available. */ +/***********************************************************************/ +class DllExport ZLBFAM : public BLKFAM { + public: + // Constructor + ZLBFAM(PDOSDEF tdp); + ZLBFAM(PZLBFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZLBFAM(this);} + inline void SetOptimized(bool b) {Optimized = b;} + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + bool WriteCompressedBuffer(PGLOBAL g); + int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); + + // Members + z_streamp Zstream; // Compression/decompression stream + Byte *Zbuffer; // Compressed block buffer + int *Zlenp; // Pointer to block length + bool Optimized; // true when opt file is available + }; // end of class ZLBFAM + +#endif // __FILAMGZ_H diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp deleted file mode 100644 index d9834e56dcd..00000000000 --- a/storage/connect/filamzip.cpp +++ /dev/null @@ -1,1426 +0,0 @@ -/*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ -/* PROGRAM NAME: FILAMZIP */ -/* ------------- */ -/* Version 1.5 */ -/* */ -/* COPYRIGHT: */ -/* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2015 */ -/* */ -/* WHAT THIS PROGRAM DOES: */ -/* ----------------------- */ -/* This program are the ZLIB compressed files classes. */ -/* */ -/***********************************************************************/ - -/***********************************************************************/ -/* Include relevant MariaDB header file. */ -/***********************************************************************/ -#include "my_global.h" -#if defined(__WIN__) -#include -#include -#if defined(__BORLANDC__) -#define __MFC_COMPAT__ // To define min/max as macro -#endif -//#include -#else // !__WIN__ -#if defined(UNIX) -#include -#else // !UNIX -#include -#endif -#include -#endif // !__WIN__ - -/***********************************************************************/ -/* Include application header files: */ -/* global.h is header containing all global declarations. */ -/* plgdbsem.h is header containing the DB application declarations. */ -/* tabdos.h is header containing the TABDOS class declarations. */ -/***********************************************************************/ -#include "global.h" -#include "plgdbsem.h" -//#include "catalog.h" -//#include "reldef.h" -//#include "xobject.h" -//#include "kindex.h" -#include "filamtxt.h" -#include "tabdos.h" -#if defined(UNIX) -#include "osutil.h" -#endif - -/***********************************************************************/ -/* This define prepares ZLIB function declarations. */ -/***********************************************************************/ -//#define ZLIB_DLL - -#include "filamzip.h" - -/***********************************************************************/ -/* DB static variables. */ -/***********************************************************************/ -extern int num_read, num_there, num_eq[]; // Statistics - -/* ------------------------------------------------------------------- */ - -/***********************************************************************/ -/* Implementation of the ZIPFAM class. */ -/***********************************************************************/ -ZIPFAM::ZIPFAM(PZIPFAM txfp) : TXTFAM(txfp) - { - Zfile = txfp->Zfile; - Zpos = txfp->Zpos; - } // end of ZIPFAM copy constructor - -/***********************************************************************/ -/* Zerror: Error function for gz calls. */ -/* gzerror returns the error message for the last error which occurred*/ -/* on the given compressed file. errnum is set to zlib error number. */ -/* If an error occurred in the file system and not in the compression */ -/* library, errnum is set to Z_ERRNO and the application may consult */ -/* errno to get the exact error code. */ -/***********************************************************************/ -int ZIPFAM::Zerror(PGLOBAL g) - { - int errnum; - - strcpy(g->Message, gzerror(Zfile, &errnum)); - - if (errnum == Z_ERRNO) -#if defined(__WIN__) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); -#else // !__WIN__ - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#endif // !__WIN__ - - return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; - } // end of Zerror - -/***********************************************************************/ -/* Reset: reset position values at the beginning of file. */ -/***********************************************************************/ -void ZIPFAM::Reset(void) - { - TXTFAM::Reset(); -//gzrewind(Zfile); // Useful ????? - Zpos = 0; - } // end of Reset - -/***********************************************************************/ -/* ZIP GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ -/***********************************************************************/ -int ZIPFAM::GetFileLength(PGLOBAL g) - { - int len = TXTFAM::GetFileLength(g); - - if (len > 0) - // Estimate size reduction to a max of 6 - len *= 6; - - return len; - } // end of GetFileLength - -/***********************************************************************/ -/* ZIP Access Method opening routine. */ -/***********************************************************************/ -bool ZIPFAM::OpenTableFile(PGLOBAL g) - { - char opmode[4], filename[_MAX_PATH]; - MODE mode = Tdbp->GetMode(); - - switch (mode) { - case MODE_READ: - strcpy(opmode, "r"); - break; - case MODE_UPDATE: - /*****************************************************************/ - /* Updating ZIP files not implemented yet. */ - /*****************************************************************/ - strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); - return true; - case MODE_DELETE: - if (!Tdbp->GetNext()) { - // Store the number of deleted lines - DelRows = Cardinality(g); - - // This will erase the entire file - strcpy(opmode, "w"); -// Block = 0; // For ZBKFAM -// Last = Nrec; // For ZBKFAM - Tdbp->ResetSize(); - } else { - sprintf(g->Message, MSG(NO_PART_DEL), "ZIP"); - return true; - } // endif filter - - break; - case MODE_INSERT: - strcpy(opmode, "a+"); - break; - default: - sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); - return true; - } // endswitch Mode - - /*********************************************************************/ - /* Open according to logical input/output mode required. */ - /* Use specific zlib functions. */ - /* Treat files as binary. */ - /*********************************************************************/ - strcat(opmode, "b"); - Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); - - if (Zfile == NULL) { - sprintf(g->Message, MSG(GZOPEN_ERROR), - opmode, (int)errno, filename); - strcat(strcat(g->Message, ": "), strerror(errno)); - return (mode == MODE_READ && errno == ENOENT) - ? PushWarning(g, Tdbp) : true; - } // endif Zfile - - /*********************************************************************/ - /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ - /*********************************************************************/ -//To_Fb = dbuserp->Openlist; // Keep track of File block - - /*********************************************************************/ - /* Allocate the line buffer. */ - /*********************************************************************/ - return AllocateBuffer(g); - } // end of OpenTableFile - -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZIPFAM::AllocateBuffer(PGLOBAL g) - { - MODE mode = Tdbp->GetMode(); - - Buflen = Lrecl + 2; // Lrecl does not include CRLF -//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY - - if (trace) - htrc("SubAllocating a buffer of %d bytes\n", Buflen); - - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (mode == MODE_INSERT) { - /*******************************************************************/ - /* For Insert buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - To_Buf[Buflen - 2] = '\n'; - To_Buf[Buflen - 1] = '\0'; - } // endif Insert - - return false; - } // end of AllocateBuffer - -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetRowID(void) - { - return Rows; - } // end of GetRowID - -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetPos(void) - { - return (int)Zpos; - } // end of GetPos - -/***********************************************************************/ -/* GetNextPos: return the position of next record. */ -/***********************************************************************/ -int ZIPFAM::GetNextPos(void) - { - return gztell(Zfile); - } // end of GetNextPos - -/***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ -/***********************************************************************/ -bool ZIPFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 - Fpos = pos; - - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return true; - } // endif - - Placed = true; - return false; -#endif // 0 - } // end of SetPos - -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/***********************************************************************/ -bool ZIPFAM::RecordPos(PGLOBAL) - { - Zpos = gztell(Zfile); - return false; - } // end of RecordPos - -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZIPFAM::SkipRecord(PGLOBAL g, bool header) - { - // Skip this record - if (gzeof(Zfile)) - return RC_EF; - else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) - return Zerror(g); - - if (header) - RecordPos(g); - - return RC_OK; - } // end of SkipRecord - -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZIPFAM::ReadBuffer(PGLOBAL g) - { - char *p; - int rc; - - if (!Zfile) - return RC_EF; - - if (!Placed) { - /*******************************************************************/ - /* Record file position in case of UPDATE or DELETE. */ - /*******************************************************************/ - next: - if (RecordPos(g)) - return RC_FX; - - CurBlk = Rows++; // Update RowID - - /*******************************************************************/ - /* Check whether optimization on ROWID */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - // Skip this record - if ((rc = SkipRecord(g, FALSE)) != RC_OK) - return rc; - - goto next; - } // endswitch rc - - } else - Placed = false; - - if (gzeof(Zfile)) { - rc = RC_EF; - } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { - p = To_Buf + strlen(To_Buf) - 1; - - if (*p == '\n') - *p = '\0'; // Eliminate ending new-line character - - if (*(--p) == '\r') - *p = '\0'; // Eliminate eventuel carriage return - - strcpy(Tdbp->GetLine(), To_Buf); - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - if (trace > 1) - htrc(" Read: '%s' rc=%d\n", To_Buf, rc); - - return rc; - } // end of ReadBuffer - -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIPFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* Now start the writing process. */ - /*********************************************************************/ - if (gzputs(Zfile, To_Buf) < 0) - return Zerror(g); - - return RC_OK; - } // end of WriteBuffer - -/***********************************************************************/ -/* Data Base delete line routine for ZDOS access method. (NIY) */ -/***********************************************************************/ -int ZIPFAM::DeleteRecords(PGLOBAL g, int) - { - strcpy(g->Message, MSG(NO_ZIP_DELETE)); - return RC_FX; - } // end of DeleteRecords - -/***********************************************************************/ -/* Data Base close routine for DOS access method. */ -/***********************************************************************/ -void ZIPFAM::CloseTableFile(PGLOBAL, bool) - { - int rc = gzclose(Zfile); - - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); - - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile - -/***********************************************************************/ -/* Rewind routine for ZIP access method. */ -/***********************************************************************/ -void ZIPFAM::Rewind(void) - { - gzrewind(Zfile); - } // end of Rewind - -/* ------------------------------------------------------------------- */ - -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZBKFAM::ZBKFAM(PDOSDEF tdp) : ZIPFAM(tdp) - { - Blocked = true; - Block = tdp->GetBlock(); - Last = tdp->GetLast(); - Nrec = tdp->GetElemt(); - CurLine = NULL; - NxtLine = NULL; - Closing = false; - BlkPos = tdp->GetTo_Pos(); - } // end of ZBKFAM standard constructor - -ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) - { - CurLine = txfp->CurLine; - NxtLine = txfp->NxtLine; - Closing = txfp->Closing; - } // end of ZBKFAM copy constructor - -/***********************************************************************/ -/* Use BlockTest to reduce the table estimated size. */ -/***********************************************************************/ -int ZBKFAM::MaxBlkSize(PGLOBAL g, int) - { - int rc = RC_OK, savcur = CurBlk; - int size; - - // Roughly estimate the table size as the sum of blocks - // that can contain good rows - for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) - if ((rc = Tdbp->TestBlock(g)) == RC_OK) - size += (CurBlk == Block - 1) ? Last : Nrec; - else if (rc == RC_EF) - break; - - CurBlk = savcur; - return size; - } // end of MaxBlkSize - -/***********************************************************************/ -/* ZBK Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZBKFAM::Cardinality(PGLOBAL g) - { - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - } // end of Cardinality - -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZBKFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Nrec * (Lrecl + 2); - CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer - -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZBKFAM::GetRowID(void) - { - return CurNum + Nrec * CurBlk + 1; - } // end of GetRowID - -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZBKFAM::GetPos(void) - { - return CurNum + Nrec * CurBlk; // Computed file index - } // end of GetPos - -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/* Not used yet for fixed tables. */ -/***********************************************************************/ -bool ZBKFAM::RecordPos(PGLOBAL /*g*/) - { -//strcpy(g->Message, "RecordPos not implemented for zip blocked tables"); -//return true; - return RC_OK; - } // end of RecordPos - -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) - { -//strcpy(g->Message, "SkipRecord not implemented for zip blocked tables"); -//return RC_FX; - return RC_OK; - } // end of SkipRecord - -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZBKFAM::ReadBuffer(PGLOBAL g) - { - int n, skip, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - while (*NxtLine++ != '\n') ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = CurBlk - skip; i < CurBlk; i++) { - BlkLen = BlkPos[i + 1] - BlkPos[i]; - - if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; - - if (!(n = gzread(Zfile, To_Buf, BlkLen))) { - rc = RC_EF; - } else if (n > 0) { - // Get the position of the current line - CurLine = To_Buf; - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer - -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZBKFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = CurLine + strlen(CurLine) - To_Buf; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - } else - CurLine += strlen(CurLine); - - return RC_OK; - } // end of WriteBuffer - -/***********************************************************************/ -/* Data Base delete line routine for ZBK access method. */ -/* Implemented only for total deletion of the table, which is done */ -/* by opening the file in mode "wb". */ -/***********************************************************************/ -int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) - { - if (irc == RC_EF) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - defp->SetBlock(0); - defp->SetLast(Nrec); - - if (!defp->SetIntCatInfo("Blocks", 0) || - !defp->SetIntCatInfo("Last", 0)) { - sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); - return RC_FX; - } else - return RC_OK; - - } else - return irc; - - } // end of DeleteRecords - -/***********************************************************************/ -/* Data Base close routine for ZBK access method. */ -/***********************************************************************/ -void ZBKFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = true; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - gzclose(Zfile); - } else if (Tdbp->GetMode() == MODE_DELETE) { - rc = DeleteRecords(g, RC_EF); - gzclose(Zfile); - } else - rc = gzclose(Zfile); - - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); - - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile - -/***********************************************************************/ -/* Rewind routine for ZBK access method. */ -/***********************************************************************/ -void ZBKFAM::Rewind(void) - { - gzrewind(Zfile); - CurBlk = -1; - CurNum = Rbuf; - } // end of Rewind - -/* ------------------------------------------------------------------- */ - -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) - { -//Block = tdp->GetBlock(); -//Last = tdp->GetLast(); - Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; - Blksize = Nrec * Lrecl; - } // end of ZIXFAM standard constructor - -/***********************************************************************/ -/* ZIX Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZIXFAM::Cardinality(PGLOBAL g) - { - if (Last) - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - else // Last and Block not defined, cannot do it yet - return 0; - - } // end of Cardinality - -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZIXFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Blksize; - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - /*******************************************************************/ - /* For Insert the buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - - if (Tdbp->GetFtype() < 2) - // if not binary, the file is physically a text file - for (int len = Lrecl; len <= Buflen; len += Lrecl) { -#if defined(__WIN__) - To_Buf[len - 2] = '\r'; -#endif // __WIN__ - To_Buf[len - 1] = '\n'; - } // endfor len - - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer - -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZIXFAM::ReadBuffer(PGLOBAL g) - { - int n, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - Tdbp->IncLine(Lrecl); // Used by DOSCOL functions - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - Tdbp->SetLine(To_Buf); - - int skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = 0; i < skip; i++) { - if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - if (!(n = gzread(Zfile, To_Buf, Buflen))) { - rc = RC_EF; - } else if (n > 0) { - Rbuf = n / Lrecl; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer - -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIXFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = Rbuf * Lrecl; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - Tdbp->SetLine(To_Buf); - } else - Tdbp->IncLine(Lrecl); // Used by FIXCOL functions - - return RC_OK; - } // end of WriteBuffer - -/* --------------------------- Class ZLBFAM -------------------------- */ - -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) - { - Zstream = NULL; - Zbuffer = NULL; - Zlenp = NULL; - Optimized = tdp->IsOptimized(); - } // end of ZLBFAM standard constructor - -ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) - { - Zstream = txfp->Zstream; - Zbuffer = txfp->Zbuffer; - Zlenp = txfp->Zlenp; - Optimized = txfp->Optimized; - } // end of ZLBFAM (dummy?) copy constructor - -/***********************************************************************/ -/* ZLB GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ -/***********************************************************************/ -int ZLBFAM::GetFileLength(PGLOBAL g) - { - int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); - - if (len > 0) - // Estimate size reduction to a max of 5 - len *= 5; - - return len; - } // end of GetFileLength - -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZLBFAM::AllocateBuffer(PGLOBAL g) - { - char *msg; - int n, zrc; - -#if 0 - if (!Optimized && Tdbp->NeedIndexing(g)) { - strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); - return TRUE; - } // endif indexing -#endif // 0 - -#if defined(NOLIB) - if (!zlib && LoadZlib()) { - sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); - return TRUE; - } // endif zlib -#endif - - BLKFAM::AllocateBuffer(g); -//Buflen = Nrec * (Lrecl + 2); -//Rbuf = Nrec; - - // Allocate the compressed buffer - n = Buflen + 16; // ????????????????????????????????? - Zlenp = (int*)PlugSubAlloc(g, NULL, n); - Zbuffer = (Byte*)(Zlenp + 1); - - // Allocate and initialize the Z stream - Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); - Zstream->zalloc = (alloc_func)0; - Zstream->zfree = (free_func)0; - Zstream->opaque = (voidpf)0; - Zstream->next_in = NULL; - Zstream->avail_in = 0; - - if (Tdbp->GetMode() == MODE_READ) { - msg = "inflateInit"; - zrc = inflateInit(Zstream); - } else { - msg = "deflateInit"; - zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); - } // endif Mode - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, "%s error: %s", msg, Zstream->msg); - else - sprintf(g->Message, "%s error: %d", msg, zrc); - - return TRUE; - } // endif zrc - - if (Tdbp->GetMode() == MODE_INSERT) { - // Write the file header block - if (Last == Nrec) { - CurBlk = Block; - CurNum = 0; - - if (!GetFileLength(g)) { - // Write the zlib header as an extra block - strcpy(To_Buf, "PlugDB"); - BlkLen = strlen("PlugDB") + 1; - - if (WriteCompressedBuffer(g)) - return TRUE; - - } // endif void file - - } else { - // In mode insert, if Last != Nrec, last block must be updated - CurBlk = Block - 1; - CurNum = Last; - - strcpy(g->Message, MSG(NO_PAR_BLK_INS)); - return TRUE; - } // endif Last - - } else { // MODE_READ - // First thing to do is to read the header block - void *rdbuf; - - if (Optimized) { - BlkLen = BlkPos[0]; - rdbuf = Zlenp; - } else { - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) - return FALSE; // Empty file - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_EF: - return FALSE; - case RC_FX: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); -#endif - case RC_NF: - return TRUE; - } // endswitch - - // Some old tables can have PlugDB in their header - if (strcmp(To_Buf, "PlugDB")) { - sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); - return TRUE; - } // endif strcmp - - } // endif Mode - - return FALSE; - } // end of AllocateBuffer - -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZLBFAM::GetPos(void) - { - return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; - } // end of GetPos - -/***********************************************************************/ -/* GetNextPos: should not be called for this class. */ -/***********************************************************************/ -int ZLBFAM::GetNextPos(void) - { - if (Optimized) { - assert(FALSE); - return 0; - } else - return ftell(Stream); - - } // end of GetNextPos - -/***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ -/***********************************************************************/ -bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 // All this must be checked - if (pos < 0) { - strcpy(g->Message, MSG(INV_REC_POS)); - return true; - } // endif recpos - - CurBlk = pos / Nrec; - CurNum = pos % Nrec; -#if defined(_DEBUG) - num_eq[(CurBlk == OldBlk) ? 1 : 0]++; -#endif - - // Indicate the table position was externally set - Placed = true; - return false; -#endif // 0 - } // end of SetPos - -/***********************************************************************/ -/* ReadBuffer: Read one line for a text file. */ -/***********************************************************************/ -int ZLBFAM::ReadBuffer(PGLOBAL g) - { - int n; - void *rdbuf; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (Placed) { - Placed = FALSE; - } else if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - if (Tdbp->GetFtype() == RECFM_VAR) - while (*NxtLine++ != '\n') ; - else - NxtLine += Lrecl; - - // Set caller line buffer - n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) { - CurNum--; // To have a correct Last value when optimizing - return RC_EF; - } else { - /*******************************************************************/ - /* New block. */ - /*******************************************************************/ - CurNum = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*******************************************************************/ - /* Before reading a new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - if (Optimized) switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - goto next; - } // endswitch rc - - } // endif's - - if (OldBlk == CurBlk) - goto ok; // Block is already there - - if (Optimized) { - // Store the position of next block - Fpos = BlkPos[CurBlk]; - - // fseek is required only in non sequential reading - if (CurBlk != OldBlk + 1) - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return RC_FX; - } // endif fseek - - // Calculate the length of block to read - BlkLen = BlkPos[CurBlk + 1] - Fpos; - rdbuf = Zlenp; - } else { // !Optimized - if (CurBlk != OldBlk + 1) { - strcpy(g->Message, MSG(INV_RAND_ACC)); - return RC_FX; - } else - Fpos = ftell(Stream); // Used when optimizing - - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { - if (feof(Stream)) - return RC_EF; - - goto err; - } // endif fread - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - // Read the next block - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_FX: goto err; - case RC_NF: return RC_FX; - case RC_EF: return RC_EF; - default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - } // endswitch ReadCompressedBuffer - - ok: - if (Tdbp->GetFtype() == RECFM_VAR) { - int i; - - // Get the position of the current line - for (i = 0, CurLine = To_Buf; i < CurNum; i++) - while (*CurLine++ != '\n') ; // What about Unix ??? - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - } else { - CurLine = To_Buf + CurNum * Lrecl; - NxtLine = CurLine + Lrecl; - n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - } // endif Ftype - - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - - OldBlk = CurBlk; // Last block actually read - IsRead = TRUE; // Is read indeed - return RC_OK; - - err: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); -#endif - return RC_FX; - } // end of ReadBuffer - -/***********************************************************************/ -/* Read and decompress a block from the stream. */ -/***********************************************************************/ -int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) - { - if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { - int zrc; - - num_read++; - - if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { - sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); - return RC_NF; - } // endif BlkLen - - // HERE WE MUST INFLATE THE BLOCK - Zstream->next_in = Zbuffer; - Zstream->avail_in = (uInt)(*Zlenp); - Zstream->next_out = (Byte*)To_Buf; - Zstream->avail_out = Buflen; - zrc = inflate(Zstream, Z_SYNC_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); - - return RC_NF; - } // endif zrc - - } else if (feof(Stream)) { - return RC_EF; - } else - return RC_FX; - - return RC_OK; - } // end of ReadCompressedBuffer - -/***********************************************************************/ -/* WriteBuffer: File write routine for DOS access method. */ -/* Update is directly written back into the file, */ -/* with this (fast) method, record size cannot change. */ -/***********************************************************************/ -int ZLBFAM::WriteBuffer(PGLOBAL g) - { - assert (Tdbp->GetMode() == MODE_INSERT); - - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) { - if (Tdbp->GetFtype() == RECFM_BIN) - memcpy(CurLine, Tdbp->GetLine(), Lrecl); - else - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - -#if defined(_DEBUG) - if (Tdbp->GetFtype() == RECFM_FIX && - (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { - strcpy(g->Message, MSG(BAD_LINE_LEN)); - Closing = TRUE; - return RC_FX; - } // endif Lrecl -#endif // _DEBUG - } // endif Closing - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /*********************************************************************/ - if (++CurNum != Rbuf) { - if (Tdbp->GetFtype() == RECFM_VAR) - CurLine += strlen(CurLine); - else - CurLine += Lrecl; - - return RC_OK; // We write only full blocks - } // endif CurNum - - // HERE WE MUST DEFLATE THE BLOCK - if (Tdbp->GetFtype() == RECFM_VAR) - NxtLine = CurLine + strlen(CurLine); - else - NxtLine = CurLine + Lrecl; - - BlkLen = NxtLine - To_Buf; - - if (WriteCompressedBuffer(g)) { - Closing = TRUE; // To tell CloseDB about a Write error - return RC_FX; - } // endif WriteCompressedBuffer - - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - return RC_OK; - } // end of WriteBuffer - -/***********************************************************************/ -/* Compress the buffer and write the deflated output to stream. */ -/***********************************************************************/ -bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) - { - int zrc; - - Zstream->next_in = (Byte*)To_Buf; - Zstream->avail_in = (uInt)BlkLen; - Zstream->next_out = Zbuffer; - Zstream->avail_out = Buflen + 16; - Zstream->total_out = 0; - zrc = deflate(Zstream, Z_FULL_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); - - return TRUE; - } else - *Zlenp = Zstream->total_out; - - // Now start the writing process. - BlkLen = *Zlenp + sizeof(int); - - if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { - sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); - return TRUE; - } // endif size - - return FALSE; - } // end of WriteCompressedBuffer - -/***********************************************************************/ -/* Table file close routine for DOS access method. */ -/***********************************************************************/ -void ZLBFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - // Closing is True if last Write was in error - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = TRUE; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - fclose(Stream); - } else - rc = fclose(Stream); - - if (trace) - htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", - To_File, Tdbp->GetMode(), rc); - - Stream = NULL; // So we can know whether table is open - To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - - if (Tdbp->GetMode() == MODE_READ) - rc = inflateEnd(Zstream); - else - rc = deflateEnd(Zstream); - - } // end of CloseTableFile - -/***********************************************************************/ -/* Rewind routine for ZLIB access method. */ -/***********************************************************************/ -void ZLBFAM::Rewind(void) - { - // We must be positioned after the header block - if (CurBlk >= 0) { // Nothing to do if no block read yet - if (!Optimized) { // If optimized, fseek will be done in ReadBuffer - size_t st; - - rewind(Stream); - - if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) - htrc("fread error %d in Rewind", errno); - - fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); - OldBlk = -1; - } // endif Optimized - - CurBlk = -1; - CurNum = Rbuf; - } // endif CurBlk - -//OldBlk = -1; -//Rbuf = 0; commented out in case we reuse last read block - } // end of Rewind - -/* ------------------------ End of ZipFam ---------------------------- */ diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h deleted file mode 100644 index edb8b5db323..00000000000 --- a/storage/connect/filamzip.h +++ /dev/null @@ -1,170 +0,0 @@ -/************** FilAmZip H Declares Source Code File (.H) **************/ -/* Name: FILAMZIP.H Version 1.2 */ -/* */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ -/* */ -/* This file contains the GZIP access method classes declares. */ -/***********************************************************************/ -#ifndef __FILAMZIP_H -#define __FILAMZIP_H - -#include "zlib.h" - -typedef class ZIPFAM *PZIPFAM; -typedef class ZBKFAM *PZBKFAM; -typedef class ZIXFAM *PZIXFAM; -typedef class ZLBFAM *PZLBFAM; - -/***********************************************************************/ -/* This is the access method class declaration for not optimized */ -/* variable record length files compressed using the gzip library */ -/* functions. File is accessed record by record (row). */ -/***********************************************************************/ -class DllExport ZIPFAM : public TXTFAM { -// friend class DOSCOL; - public: - // Constructor - ZIPFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} - ZIPFAM(PZIPFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIPFAM(this);} - - // Methods - virtual void Reset(void); - virtual int GetFileLength(PGLOBAL g); - virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} - virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual bool OpenTableFile(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - int Zerror(PGLOBAL g); // GZ error function - - // Members - gzFile Zfile; // Points to GZ file structure - z_off_t Zpos; // Uncompressed file position - }; // end of class ZIPFAM - -/***********************************************************************/ -/* This is the access method class declaration for optimized variable */ -/* record length files compressed using the gzip library functions. */ -/* The File is accessed by block (requires an opt file). */ -/***********************************************************************/ -class DllExport ZBKFAM : public ZIPFAM { - public: - // Constructor - ZBKFAM(PDOSDEF tdp); - ZBKFAM(PZBKFAM txfp); - - // Implementation - virtual int GetPos(void); - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZBKFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual int MaxBlkSize(PGLOBAL g, int s); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - // Members - char *CurLine; // Position of current line in buffer - char *NxtLine; // Position of Next line in buffer - bool Closing; // True when closing on Insert - }; // end of class ZBKFAM - -/***********************************************************************/ -/* This is the access method class declaration for fixed record */ -/* length files compressed using the gzip library functions. */ -/* The file is always accessed by block. */ -/***********************************************************************/ -class DllExport ZIXFAM : public ZBKFAM { - public: - // Constructor - ZIXFAM(PDOSDEF tdp); - ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} - - // Implementation - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIXFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - - protected: - // No additional Members - }; // end of class ZIXFAM - -/***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for PlugDB */ -/* fixed/variable files compressed using the zlib library functions. */ -/* Physically these are written and read using the same technique */ -/* than blocked variable files, only the contain of each block is */ -/* compressed using the deflate zlib function. The purpose of this */ -/* specific format is to have a fast mechanism for direct access of */ -/* records so blocked optimization is fast and direct access (joins) */ -/* is allowed. Note that the block length is written ahead of each */ -/* block to enable reading when optimization file is not available. */ -/***********************************************************************/ -class DllExport ZLBFAM : public BLKFAM { - public: - // Constructor - ZLBFAM(PDOSDEF tdp); - ZLBFAM(PZLBFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZLBFAM(this);} - inline void SetOptimized(bool b) {Optimized = b;} - - // Methods - virtual int GetFileLength(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - bool WriteCompressedBuffer(PGLOBAL g); - int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); - - // Members - z_streamp Zstream; // Compression/decompression stream - Byte *Zbuffer; // Compressed block buffer - int *Zlenp; // Pointer to block length - bool Optimized; // true when opt file is available - }; // end of class ZLBFAM - -#endif // __FILAMZIP_H diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 910ce97f48a..e99f5f36444 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -123,7 +123,7 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_PRX = 129, /* PROXY access method type no */ TYPE_AM_XTB = 130, /* SYS table access method type */ TYPE_AM_BLK = 131, /* BLK access method type no */ - TYPE_AM_ZIP = 132, /* ZIP access method type no */ + TYPE_AM_GZ = 132, /* GZ access method type no */ TYPE_AM_ZLIB = 133, /* ZLIB access method type no */ TYPE_AM_JSON = 134, /* JSON access method type no */ TYPE_AM_JSN = 135, /* JSN access method type no */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index a62fcbf9416..30e4d49d249 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -1,7 +1,7 @@ /************* RelDef CPP Program Source Code File (.CPP) **************/ /* PROGRAM NAME: RELDEF */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.6 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -43,9 +43,9 @@ #if defined(VCT_SUPPORT) #include "filamvct.h" #endif // VCT_SUPPORT -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabdos.h" #include "valblk.h" #include "tabmul.h" @@ -665,15 +665,15 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (!((PTDBDOS)tdbp)->GetTxfp()) { if (cmpr) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (cmpr == 1) - txfp = new(g) ZIPFAM(defp); + txfp = new(g) GZFAM(defp); else txfp = new(g) ZLBFAM(defp); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (rfm == RECFM_VAR) { if (map) txfp = new(g) MAPFAM(defp); diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 98633f49d23..06dde34a27f 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,9 +51,9 @@ #include "filamap.h" #include "filamfix.h" #include "filamdbf.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabdos.h" #include "tabfix.h" #include "tabmul.h" @@ -350,28 +350,28 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) else if (map) txfp = new(g) MPXFAM(this); else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) txfp = new(g) ZIXFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -396,7 +396,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -404,7 +404,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -531,13 +531,13 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) // except for ZLIB access method. if (Txfp->GetAmType() == TYPE_AM_MAP) { Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - } else if (Txfp->GetAmType() == TYPE_AM_ZIP) { - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_GZ) { + Txfp = new(g) GZFAM((PDOSDEF)To_Def); } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); -#endif // ZIP_SUPPORT +#endif // GZ_SUPPORT } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); @@ -2079,10 +2079,10 @@ bool TDBDOS::OpenDB(PGLOBAL g) /*******************************************************************/ if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - else if (Txfp->GetAmType() == TYPE_AM_ZIP) - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) + else if (Txfp->GetAmType() == TYPE_AM_GZ) + Txfp = new(g) GZFAM((PDOSDEF)To_Def); +#endif // GZ_SUPPORT else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? Txfp = new(g) DOSFAM((PDOSDEF)To_Def); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index c098886f14b..c70e0032f47 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -91,7 +91,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int Maxerr; /* Maximum number of bad records (DBF) */ int ReadMode; /* Specific to DBF */ int Ending; /* Length of end of lines */ - int Teds; /* Binary table default endian setting */ + char Teds; /* Binary table default endian setting */ }; // end of DOSDEF /***********************************************************************/ diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 55c254f41ea..d99f7800f26 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -1,11 +1,11 @@ /************* TabFix C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFIX */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -589,9 +589,10 @@ void BINCOL::WriteColumn(PGLOBAL g) switch (Fmt) { case 'X': // Standard not converted values - if (Eds && IsTypeChar(Buf_Type)) - *(longlong *)p = Value->GetBigintValue(); - else if (Value->GetBinValue(p, Long, Status)) { + if (Eds && IsTypeChar(Buf_Type)) { + if (Status) + Value->GetValueNonAligned(p, Value->GetBigintValue()); + } else if (Value->GetBinValue(p, Long, Status)) { sprintf(g->Message, MSG(BIN_F_TOO_LONG), Name, Value->GetSize(), Long); longjmp(g->jumper[g->jump_level], 31); @@ -605,7 +606,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(short *)p = (short)n; + Value->GetValueNonAligned(p, (short)n); break; case 'T': // Tiny integer @@ -625,7 +626,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(int *)p = Value->GetIntValue(); + Value->GetValueNonAligned(p, (int)n); break; case 'G': // Large (great) integer @@ -636,12 +637,12 @@ void BINCOL::WriteColumn(PGLOBAL g) case 'F': // Float case 'R': // Real if (Status) - *(float *)p = (float)Value->GetFloatValue(); + Value->GetValueNonAligned(p, (float)Value->GetFloatValue()); break; case 'D': // Double if (Status) - *(double *)p = Value->GetFloatValue(); + Value->GetValueNonAligned(p, Value->GetFloatValue()); break; case 'C': // Characters diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index d21a8b977da..4a39ecd6e0f 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,11 +1,11 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9 */ +/* Version 3.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2001 - 2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001 - 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,9 +51,9 @@ #include "plgdbsem.h" #include "mycat.h" #include "filamap.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabfmt.h" #include "tabmul.h" #define NO_FUNC @@ -462,16 +462,16 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) // Should be now compatible with UNIX txfp = new(g) MAPFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) DOSFAM(this); @@ -498,7 +498,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -506,7 +506,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 17260836371..5f864f0bd48 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,6 +1,6 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.1 */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2015 */ +/* PROGRAM NAME: tabjson Version 1.2 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -25,9 +25,9 @@ //#include "resource.h" // for IDS_COLUMNS #include "tabjson.h" #include "filamap.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabmul.h" #include "checklvl.h" #include "resource.h" @@ -396,15 +396,15 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) (m == MODE_UPDATE || m == MODE_DELETE)); if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index 64d0e13e8c4..ced690e77c0 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -1,7 +1,7 @@ /************* Value C++ Functions Source Code File (.CPP) *************/ -/* Name: VALUE.CPP Version 2.5 */ +/* Name: VALUE.CPP Version 2.6 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes family functions. */ /* These classes contain values of different types. They are used so */ @@ -792,19 +792,29 @@ uchar TYPVAL::GetTypedValue(PVBLK blk, int n) /***********************************************************************/ /* TYPVAL SetBinValue: with bytes extracted from a line. */ +/* Currently only used reading column of binary files. */ /***********************************************************************/ template void TYPVAL::SetBinValue(void *p) - { - Tval = *(TYPE *)p; - Null = false; - } // end of SetBinValue +{ +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + Tval = *(TYPE *)p; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru + memcpy(&Tval, p, sizeof(TYPE)); +#endif + Null = false; +} // end of SetBinValue /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ /* returns true if not. Actual filling occurs only if go is true. */ -/* Currently used by WriteColumn of binary files. */ +/* Currently only used writing column of binary files. */ /***********************************************************************/ template bool TYPVAL::GetBinValue(void *buf, int buflen, bool go) @@ -819,7 +829,16 @@ bool TYPVAL::GetBinValue(void *buf, int buflen, bool go) //#endif if (go) - *(TYPE *)buf = Tval; +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + *(TYPE *)buf = Tval; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru + memcpy(buf, &Tval, sizeof(TYPE)); +#endif Null = false; return false; diff --git a/storage/connect/value.h b/storage/connect/value.h index c5a381e89da..a670ade4c28 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -1,7 +1,7 @@ /**************** Value H Declares Source Code File (.H) ***************/ -/* Name: VALUE.H Version 2.1 */ +/* Name: VALUE.H Version 2.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes declares. */ /***********************************************************************/ @@ -16,6 +16,13 @@ #include "assert.h" #include "block.h" +/***********************************************************************/ +/* This should list the processors accepting unaligned numeral values.*/ +/***********************************************************************/ +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || defined(_M_IA64) +#define UNALIGNED_OK +#endif + /***********************************************************************/ /* Types used in some class definitions. */ /***********************************************************************/ @@ -116,27 +123,46 @@ class DllExport VALUE : public BLOCK { virtual bool Compute(PGLOBAL g, PVAL *vp, int np, OPVAL op); virtual bool FormatValue(PVAL vp, char *fmt) = 0; - /** - Set value from a non-aligned in-memory value in the machine byte order. - TYPE can be either of: - - int, short, longlong - - uint, ushort, ulonglong - - float, double - @param - a pointer to a non-aligned value of type TYPE. - */ - template - void SetValueNonAligned(const char *p) - { -#if defined(__i386__) || defined(__x86_64__) - SetValue(*((TYPE*) p)); // x86 can cast non-aligned memory directly + /** + Set value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @param - a pointer to a non-aligned value of type TYPE. + */ + template + void SetValueNonAligned(const char *p) + { +#if defined(UNALIGNED_OK) + SetValue(*((TYPE*)p)); // x86 can cast non-aligned memory directly #else - TYPE tmp; // a slower version for non-x86 platforms - memcpy(&tmp, p, sizeof(tmp)); - SetValue(tmp); + TYPE tmp; // a slower version for non-x86 platforms + memcpy(&tmp, p, sizeof(tmp)); + SetValue(tmp); #endif - } + } // end of SetValueNonAligned + + /** + Get value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @params - a pointer to a non-aligned value of type TYPE, the TYPE value. + */ + template + void GetValueNonAligned(char *p, TYPE n) + { +#if defined(UNALIGNED_OK) + *(TYPE *)p = n; // x86 can cast non-aligned memory directly +#else + TYPE tmp = n; // a slower version for non-x86 platforms + memcpy(p, &tmp, sizeof(tmp)); +#endif + } // end of SetValueNonAligned - protected: +protected: virtual bool SetConstFormat(PGLOBAL, FORMAT&) = 0; const char *GetXfmt(void); -- cgit v1.2.1 From d44723e62153d9fb4165d038e9448c20a3ad890b Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 12 Dec 2016 10:57:19 +0100 Subject: - MDEV-11295: developing handling files contained in ZIP file. A first experimental and limited implementation. modified: storage/connect/CMakeLists.txt modified: storage/connect/filamap.cpp new file: storage/connect/filamzip.cpp new file: storage/connect/filamzip.h modified: storage/connect/ha_connect.cc new file: storage/connect/ioapi.c new file: storage/connect/ioapi.h modified: storage/connect/mycat.cc modified: storage/connect/plgdbsem.h modified: storage/connect/plgdbutl.cpp modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfmt.cpp modified: storage/connect/tabfmt.h modified: storage/connect/tabjson.cpp modified: storage/connect/tabjson.h new file: storage/connect/tabzip.cpp new file: storage/connect/tabzip.h new file: storage/connect/unzip.c new file: storage/connect/unzip.h new file: storage/connect/zip.c --- storage/connect/CMakeLists.txt | 12 + storage/connect/filamap.cpp | 6 +- storage/connect/filamzip.cpp | 501 ++++++++++ storage/connect/filamzip.h | 83 ++ storage/connect/ha_connect.cc | 27 +- storage/connect/ioapi.c | 247 +++++ storage/connect/ioapi.h | 208 ++++ storage/connect/mycat.cc | 16 +- storage/connect/plgdbsem.h | 15 +- storage/connect/plgdbutl.cpp | 14 +- storage/connect/tabdos.cpp | 36 +- storage/connect/tabdos.h | 6 +- storage/connect/tabfmt.cpp | 168 ++-- storage/connect/tabfmt.h | 29 +- storage/connect/tabjson.cpp | 72 +- storage/connect/tabjson.h | 3 +- storage/connect/tabzip.cpp | 230 +++++ storage/connect/tabzip.h | 100 ++ storage/connect/unzip.c | 2125 ++++++++++++++++++++++++++++++++++++++++ storage/connect/unzip.h | 437 +++++++++ storage/connect/zip.c | 2007 +++++++++++++++++++++++++++++++++++++ storage/connect/zip.h | 362 +++++++ 22 files changed, 6579 insertions(+), 125 deletions(-) create mode 100644 storage/connect/filamzip.cpp create mode 100644 storage/connect/filamzip.h create mode 100644 storage/connect/ioapi.c create mode 100644 storage/connect/ioapi.h create mode 100644 storage/connect/tabzip.cpp create mode 100644 storage/connect/tabzip.h create mode 100644 storage/connect/unzip.c create mode 100644 storage/connect/unzip.h create mode 100644 storage/connect/zip.c create mode 100644 storage/connect/zip.h (limited to 'storage') diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index f1567730b26..ce6de424421 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -279,6 +279,18 @@ IF(CONNECT_WITH_JDBC) ENDIF() ENDIF(CONNECT_WITH_JDBC) +# +# ZIP +# + +OPTION(CONNECT_WITH_ZIP "Compile CONNECT storage engine with ZIP support" ON) + +IF(CONNECT_WITH_ZIP) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamzip.cpp tabzip.cpp unzip.c ioapi.c zip.c + filamzip.h tabzip.h ioapi.h unzip.h zip.h) + add_definitions(-DZIP_SUPPORT -DNOCRYPT) +ENDIF(CONNECT_WITH_ZIP) + # # XMAP diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index 5cf9a4d945c..3c5b3ae7592 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -87,7 +87,7 @@ int MAPFAM::GetFileLength(PGLOBAL g) { int len; - len = (To_Fb) ? To_Fb->Length : TXTFAM::GetFileLength(g); + len = (To_Fb && To_Fb->Count) ? To_Fb->Length : TXTFAM::GetFileLength(g); if (trace) htrc("Mapped file length=%d\n", len); @@ -413,7 +413,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) if (Tpos == Spos) { /*******************************************************************/ - /* First line to delete. Move of eventual preceeding lines is */ + /* First line to delete. Move of eventual preceding lines is */ /* not required here, just setting of future Spos and Tpos. */ /*******************************************************************/ Tpos = Spos = Fpos; @@ -498,7 +498,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) void MAPFAM::CloseTableFile(PGLOBAL g, bool) { PlugCloseFile(g, To_Fb); - To_Fb = NULL; // To get correct file size in Cardinality +//To_Fb = NULL; // To get correct file size in Cardinality if (trace) htrc("MAP Close: closing %s count=%d\n", diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp new file mode 100644 index 00000000000..ea8b827974b --- /dev/null +++ b/storage/connect/filamzip.cpp @@ -0,0 +1,501 @@ +/*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ +/* PROGRAM NAME: FILAMZIP */ +/* ------------- */ +/* Version 1.0 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the ZIP file access method classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the System header files. */ +/***********************************************************************/ +#include "my_global.h" +#if !defined(__WIN__) +#if defined(UNIX) +#include +#include +#else // !UNIX +#include +#endif // !UNIX +#include +#endif // !__WIN__ + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "osutil.h" +#include "filamtxt.h" +#include "tabfmt.h" +//#include "tabzip.h" +#include "filamzip.h" + +/* -------------------------- class ZIPFAM --------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +{ + zipfile = NULL; + zfn = tdp->Zipfn; + target = tdp->Fn; +//*fn = 0; + entryopen = false; + multiple = tdp->Multiple; + + // Init the case mapping table. +#if defined(__WIN__) + for (int i = 0; i < 256; ++i) mapCaseTable[i] = toupper(i); +#else + for (int i = 0; i < 256; ++i) mapCaseTable[i] = i; +#endif +} // end of ZIPFAM standard constructor + +ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +{ + zipfile = txfp->zipfile; + zfn = txfp->zfn; + target = txfp->target; +//strcpy(fn, txfp->fn); + finfo = txfp->finfo; + entryopen = txfp->entryopen; + multiple = txfp->multiple; + for (int i = 0; i < 256; ++i) mapCaseTable[i] = txfp->mapCaseTable[i]; +} // end of ZIPFAM copy constructor + +/***********************************************************************/ +/* This code is the copyright property of Alessandro Felice Cantatore. */ +/* http://xoomer.virgilio.it/acantato/dev/wildcard/wildmatch.html */ +/***********************************************************************/ +bool ZIPFAM::WildMatch(PSZ pat, PSZ str) { + PSZ s, p; + bool star = FALSE; + + loopStart: + for (s = str, p = pat; *s; ++s, ++p) { + switch (*p) { + case '?': + if (*s == '.') goto starCheck; + break; + case '*': + star = TRUE; + str = s, pat = p; + if (!*++pat) return TRUE; + goto loopStart; + default: + if (mapCaseTable[*s] != mapCaseTable[*p]) + goto starCheck; + break; + } /* endswitch */ + } /* endfor */ + if (*p == '*') ++p; + return (!*p); + + starCheck: + if (!star) return FALSE; + str++; + goto loopStart; +} // end of WildMatch + +/***********************************************************************/ +/* ZIP GetFileLength: returns file size in number of bytes. */ +/***********************************************************************/ +int ZIPFAM::GetFileLength(PGLOBAL g) +{ + int len = (entryopen) ? Top - Memory : 100; // not 0 to avoid ASSERT + + if (trace) + htrc("Zipped file length=%d\n", len); + + return len; +} // end of GetFileLength + +/***********************************************************************/ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ +/***********************************************************************/ +bool ZIPFAM::open(PGLOBAL g, const char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error"); + + return (zipfile == NULL); +} // end of open + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void ZIPFAM::close() +{ + if (zipfile) { + closeEntry(); + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile + +} // end of close + +/***********************************************************************/ +/* Find next entry matching target pattern. */ +/***********************************************************************/ +int ZIPFAM::findEntry(PGLOBAL g, bool next) +{ + int rc; + char fn[FILENAME_MAX]; // The current entry file name + + do { + if (next) { + rc = unzGoToNextFile(zipfile); + + if (rc == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (rc != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile rc = ", rc); + return RC_FX; + } // endif rc + + } // endif next + + if (target && *target) { + rc = unzGetCurrentFileInfo(zipfile, NULL, fn, sizeof(fn), + NULL, 0, NULL, 0); + if (rc == UNZ_OK) { + if (WildMatch(target, fn)) + return RC_OK; + + } else { + sprintf(g->Message, "GetCurrentFileInfo rc = %d", rc); + return RC_FX; + } // endif rc + + } else + return RC_OK; + + next = true; + } while (true); + + strcpy(g->Message, "FindNext logical error"); + return RC_FX; +} // end of FindNext + +/***********************************************************************/ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/***********************************************************************/ +bool ZIPFAM::OpenTableFile(PGLOBAL g) +{ + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + PFBLOCK fp; + PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; + + /*********************************************************************/ + /* The file will be decompressed into virtual memory. */ + /*********************************************************************/ + if (mode == MODE_READ) { + // We used the file name relative to recorded datapath + PlugSetPath(filename, zfn, Tdbp->GetPath()); + + bool b = open(g, filename); + + if (!b) { + int rc; + + if (target && *target) { + if (!multiple) { + rc = unzLocateFile(zipfile, target, 0); + + if (rc == UNZ_END_OF_LIST_OF_FILE) { + sprintf(g->Message, "Target file %s not in %s", target, filename); + return true; + } else if (rc != UNZ_OK) { + sprintf(g->Message, "unzLocateFile rc=%d", rc); + return true; + } // endif's rc + + } else { + if ((rc = findEntry(g, false)) == RC_FX) + return true; + else if (rc == RC_NF) { + sprintf(g->Message, "No match of %s in %s", target, filename); + return true; + } // endif rc + + } // endif multiple + + } // endif target + + if (openEntry(g)) + return true; + + if (Top > Memory) { + /*******************************************************************/ + /* Link a Fblock. This make possible to automatically close it */ + /* in case of error g->jump. */ + /*******************************************************************/ + fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK)); + fp->Type = TYPE_FB_ZIP; + fp->Fname = PlugDup(g, filename); + fp->Next = dbuserp->Openlist; + dbuserp->Openlist = fp; + fp->Count = 1; + fp->Length = Top - Memory; + fp->Memory = Memory; + fp->Mode = mode; + fp->File = this; + fp->Handle = NULL; + } // endif fp + + To_Fb = fp; // Useful when closing + } // endif b + + } else { + strcpy(g->Message, "Only READ mode supported for ZIP files"); + return true; + } // endif mode + + return false; + } // end of OpenTableFile + +/***********************************************************************/ +/* Open target in zip file. */ +/***********************************************************************/ +bool ZIPFAM::openEntry(PGLOBAL g) +{ + int rc; + uint size; + + rc = unzGetCurrentFileInfo(zipfile, &finfo, 0, 0, 0, 0, 0, 0); + + if (rc != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 rc=%d", rc); + return true; + } else if ((rc = unzOpenCurrentFile(zipfile)) != UNZ_OK) { + sprintf(g->Message, "unzOpenCurrentFile rc=%d", rc); + return true; + } // endif rc + + size = finfo.uncompressed_size; + Memory = new char[size]; + + if ((rc = unzReadCurrentFile(zipfile, Memory, size)) < 0) { + sprintf(g->Message, "unzReadCurrentFile rc = ", rc); + unzCloseCurrentFile(zipfile); + free(Memory); + entryopen = false; + } else { + // The pseudo "buffer" is here the entire real buffer + Fpos = Mempos = Memory; + Top = Memory + size; + + if (trace) + htrc("Memory=%p size=%ud Top=%p\n", Memory, size, Top); + + entryopen = true; + } // endif rc + + return !entryopen; +} // end of openEntry + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void ZIPFAM::closeEntry() +{ + if (entryopen) { + unzCloseCurrentFile(zipfile); + entryopen = false; + } // endif entryopen + + if (Memory) { + free(Memory); + Memory = NULL; + } // endif Memory + +} // end of closeEntry + +/***********************************************************************/ +/* ReadBuffer: Read one line for a ZIP file. */ +/***********************************************************************/ +int ZIPFAM::ReadBuffer(PGLOBAL g) +{ + int rc, len; + + // Are we at the end of the memory + if (Mempos >= Top) { + if (multiple) { + closeEntry(); + + if ((rc = findEntry(g, true)) != RC_OK) + return rc; + + if (openEntry(g)) + return RC_FX; + + } else + return RC_EF; + + } // endif Mempos + +#if 0 + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + int rc; + + next: + Fpos = Mempos; + CurBlk = (int)Rows++; + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, false)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; +#else + // Perhaps unuseful + Fpos = Mempos; + CurBlk = (int)Rows++; + Placed = false; +#endif + + // Immediately calculate next position (Used by DeleteDB) + while (*Mempos++ != '\n'); // What about Unix ??? + + // Set caller line buffer + len = (Mempos - Fpos) - 1; + + // Don't rely on ENDING setting + if (len > 0 && *(Mempos - 2) == '\r') + len--; // Line ends by CRLF + + memcpy(Tdbp->GetLine(), Fpos, len); + Tdbp->GetLine()[len] = '\0'; + return RC_OK; +} // end of ReadBuffer + +#if 0 +/***********************************************************************/ +/* Table file close routine for MAP access method. */ +/***********************************************************************/ +void ZIPFAM::CloseTableFile(PGLOBAL g, bool) +{ + close(); +} // end of CloseTableFile +#endif // 0 + +/* -------------------------- class ZPXFAM --------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZPXFAM::ZPXFAM(PDOSDEF tdp) : ZIPFAM(tdp) +{ + Lrecl = tdp->GetLrecl(); +} // end of ZPXFAM standard constructor + +ZPXFAM::ZPXFAM(PZPXFAM txfp) : ZIPFAM(txfp) +{ + Lrecl = txfp->Lrecl; +} // end of ZPXFAM copy constructor + +/***********************************************************************/ +/* ReadBuffer: Read one line for a fixed ZIP file. */ +/***********************************************************************/ +int ZPXFAM::ReadBuffer(PGLOBAL g) +{ + int rc, len; + + // Are we at the end of the memory + if (Mempos >= Top) { + if (multiple) { + closeEntry(); + + if ((rc = findEntry(g, true)) != RC_OK) + return rc; + + if (openEntry(g)) + return RC_FX; + + } else + return RC_EF; + + } // endif Mempos + +#if 0 + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + int rc; + + next: + Fpos = Mempos; + CurBlk = (int)Rows++; + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, false)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; +#else + // Perhaps unuseful + Fpos = Mempos; + CurBlk = (int)Rows++; + Placed = false; +#endif + + // Immediately calculate next position (Used by DeleteDB) + Mempos += Lrecl; + + // Set caller line buffer + len = Lrecl; + + // Don't rely on ENDING setting + if (len > 0 && *(Mempos - 1) == '\n') + len--; // Line ends by LF + + if (len > 0 && *(Mempos - 2) == '\r') + len--; // Line ends by CRLF + + memcpy(Tdbp->GetLine(), Fpos, len); + Tdbp->GetLine()[len] = '\0'; + return RC_OK; +} // end of ReadBuffer + diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h new file mode 100644 index 00000000000..85c1f907d20 --- /dev/null +++ b/storage/connect/filamzip.h @@ -0,0 +1,83 @@ +/************** filamzip H Declares Source Code File (.H) **************/ +/* Name: filamzip.h Version 1.0 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* This file contains the ZIP file access method classes declares. */ +/***********************************************************************/ +#ifndef __FILAMZIP_H +#define __FILAMZIP_H + +#include "block.h" +#include "filamap.h" +#include "unzip.h" + +#define DLLEXPORT extern "C" + +typedef class ZIPFAM *PZIPFAM; +typedef class ZPXFAM *PZPXFAM; + +/***********************************************************************/ +/* This is the ZIP file access method. */ +/***********************************************************************/ +class DllExport ZIPFAM : public MAPFAM { +public: + // Constructor + ZIPFAM(PDOSDEF tdp); + ZIPFAM(PZIPFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + virtual PTXF Duplicate(PGLOBAL g) {return (PTXF) new(g) ZIPFAM(this);} + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g) {return (g) ? 10 : 1;} +//virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool OpenTableFile(PGLOBAL g); + virtual bool DeferReading(void) {return false;} + virtual int ReadBuffer(PGLOBAL g); +//virtual int WriteBuffer(PGLOBAL g); +//virtual int DeleteRecords(PGLOBAL g, int irc); +//virtual void CloseTableFile(PGLOBAL g, bool abort); + void close(void); + +protected: + bool open(PGLOBAL g, const char *filename); + bool openEntry(PGLOBAL g); + void closeEntry(void); + bool WildMatch(PSZ pat, PSZ str); + int findEntry(PGLOBAL g, bool next); + + // Members + unzFile zipfile; // The ZIP container file + PSZ zfn; // The ZIP file name + PSZ target; // The target file name + unz_file_info finfo; // The current file info +//char fn[FILENAME_MAX]; // The current file name + bool entryopen; // True when open current entry + int multiple; // Multiple targets + char mapCaseTable[256]; +}; // end of ZIPFAM + +/***********************************************************************/ +/* This is the fixed ZIP file access method. */ +/***********************************************************************/ +class DllExport ZPXFAM : public ZIPFAM { +public: + // Constructor + ZPXFAM(PDOSDEF tdp); + ZPXFAM(PZPXFAM txfp); + + // Implementation + virtual PTXF Duplicate(PGLOBAL g) {return (PTXF) new(g) ZPXFAM(this);} + + // Methods + virtual int ReadBuffer(PGLOBAL g); + +protected: + // Members + int Lrecl; +}; // end of ZPXFAM + +#endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 2222e51b083..b690dff24f4 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,9 +171,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0008 October 20, 2016"; + char version[]= "Version 1.04.0009 December 09, 2016"; #if defined(__WIN__) - char compver[]= "Version 1.04.0008 " __DATE__ " " __TIME__; + char compver[]= "Version 1.04.0009 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -4165,7 +4165,8 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_MAC: case TAB_WMI: - case TAB_OEM: + case TAB_ZIP: + case TAB_OEM: #ifdef NO_EMBEDDED_ACCESS_CHECKS return false; #endif @@ -5173,13 +5174,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, char v=0, spc= ',', qch= 0; const char *fncn= "?"; const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src; - const char *col, *ocl, *rnk, *pic, *fcl, *skc; + const char *col, *ocl, *rnk, *pic, *fcl, *skc, *zfn; char *tab, *dsn, *shm, *dpath; #if defined(__WIN__) char *nsp= NULL, *cls= NULL; #endif // __WIN__ - int port= 0, hdr= 0, mxr= 0, mxe= 0, rc= 0; - int cop __attribute__((unused))= 0, lrecl= 0; +//int hdr, mxe; + int port = 0, mxr = 0, rc = 0, mul = 0, lrecl = 0; #if defined(ODBC_SUPPORT) POPARM sop= NULL; char *ucnc= NULL; @@ -5211,7 +5212,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!g) return HA_ERR_INTERNAL_ERROR; - user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= NULL; + user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= zfn= dsn= NULL; // Get the useful create options ttp= GetTypeID(topt->type); @@ -5224,7 +5225,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, sep= topt->separator; spc= (!sep) ? ',' : *sep; qch= topt->qchar ? *topt->qchar : (signed)topt->quoted >= 0 ? '"' : 0; - hdr= (int)topt->header; + mul = (int)topt->multiple; tbl= topt->tablist; col= topt->colist; @@ -5260,11 +5261,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, // prop = GetListOption(g, "Properties", topt->oplist, NULL); tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); #endif // JDBC_SUPPORT - mxe= atoi(GetListOption(g,"maxerr", topt->oplist, "0")); #if defined(PROMPT_OK) cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0")); #endif // PROMPT_OK - } else { +#if defined(ZIP_SUPPORT) + zfn = GetListOption(g, "Zipfile", topt->oplist, NULL); +#endif // ZIP_SUPPORT + } else { host= "localhost"; user= (ttp == TAB_ODBC ? NULL : "root"); } // endif option_list @@ -5471,7 +5474,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_XML: #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT case TAB_JSON: - if (!fn) + if (!fn && !zfn && !mul) sprintf(g->Message, "Missing %s file name", topt->type); else ok= true; @@ -5585,7 +5588,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, NULL, port, fnc == FNC_COL); break; case TAB_CSV: - qrp= CSVColumns(g, dpath, fn, spc, qch, hdr, mxe, fnc == FNC_COL); + qrp = CSVColumns(g, dpath, topt, fnc == FNC_COL); break; #if defined(__WIN__) case TAB_WMI: diff --git a/storage/connect/ioapi.c b/storage/connect/ioapi.c new file mode 100644 index 00000000000..7f5c191b2af --- /dev/null +++ b/storage/connect/ioapi.c @@ -0,0 +1,247 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) + #define _CRT_SECURE_NO_WARNINGS +#endif + +#if defined(__APPLE__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include "ioapi.h" + +voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) +{ + if (pfilefunc->zfile_func64.zopen64_file != NULL) + return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode); + else + { + return (*(pfilefunc->zopen32_file))(pfilefunc->zfile_func64.opaque,(const char*)filename,mode); + } +} + +long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin); + else + { + uLong offsetTruncated = (uLong)offset; + if (offsetTruncated != offset) + return -1; + else + return (*(pfilefunc->zseek32_file))(pfilefunc->zfile_func64.opaque,filestream,offsetTruncated,origin); + } +} + +ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream); + else + { + uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); + if ((tell_uLong) == MAXU32) + return (ZPOS64_T)-1; + else + return tell_uLong; + } +} + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32) +{ + p_filefunc64_32->zfile_func64.zopen64_file = NULL; + p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file; + p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file; + p_filefunc64_32->zfile_func64.ztell64_file = NULL; + p_filefunc64_32->zfile_func64.zseek64_file = NULL; + p_filefunc64_32->zfile_func64.zclose_file = p_filefunc32->zclose_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.opaque = p_filefunc32->opaque; + p_filefunc64_32->zseek32_file = p_filefunc32->zseek_file; + p_filefunc64_32->ztell32_file = p_filefunc32->ztell_file; +} + + + +static voidpf ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode)); +static uLong ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +static uLong ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size)); +static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream)); +static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); +static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); + +static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = fopen(filename, mode_fopen); + return file; +} + +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = FOPEN_FUNC((const char*)filename, mode_fopen); + return file; +} + + +static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +{ + long ret; + ret = ftell((FILE *)stream); + return ret; +} + + +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret; + ret = FTELLO_FUNC((FILE *)stream); + return ret; +} + +static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + if (fseek((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + return ret; +} + +static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + + return ret; +} + + +static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = fclose((FILE *)stream); + return ret; +} + +static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = ferror((FILE *)stream); + return ret; +} + +void fill_fopen_filefunc (pzlib_filefunc_def) + zlib_filefunc_def* pzlib_filefunc_def; +{ + pzlib_filefunc_def->zopen_file = fopen_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell_file = ftell_file_func; + pzlib_filefunc_def->zseek_file = fseek_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_fopen64_filefunc (zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = fopen64_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell64_file = ftell64_file_func; + pzlib_filefunc_def->zseek64_file = fseek64_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h new file mode 100644 index 00000000000..8dcbdb06e35 --- /dev/null +++ b/storage/connect/ioapi.h @@ -0,0 +1,208 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + + Oct-2009 - Defined ZPOS64_T to fpos_t on windows and u_int64_t on linux. (might need to find a better why for this) + Oct-2009 - Change to fseeko64, ftello64 and fopen64 so large files would work on linux. + More if/def section may be needed to support other platforms + Oct-2009 - Defined fxxxx64 calls to normal fopen/ftell/fseek so they would compile on windows. + (but you should use iowin32.c for windows instead) + +*/ + +#ifndef _ZLIBIOAPI64_H +#define _ZLIBIOAPI64_H + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) + + // Linux needs this to support file operation on files larger then 4+GB + // But might need better if/def to select just the platforms that needs them. + + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif + +#endif + +#include +#include +#include "zlib.h" + +#if defined(USE_FILE32API) +#define fopen64 fopen +#define ftello64 ftell +#define fseeko64 fseek +#else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif +#ifdef _MSC_VER + #define fopen64 fopen + #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) + #define ftello64 _ftelli64 + #define fseeko64 _fseeki64 + #else // old MSC + #define ftello64 ftell + #define fseeko64 fseek + #endif +#endif +#endif + +/* +#ifndef ZPOS64_T + #ifdef _WIN32 + #define ZPOS64_T fpos_t + #else + #include + #define ZPOS64_T uint64_t + #endif +#endif +*/ + +#ifdef HAVE_MINIZIP64_CONF_H +#include "mz64conf.h" +#endif + +/* a type choosen by DEFINE */ +#ifdef HAVE_64BIT_INT_CUSTOM +typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; +#else +#ifdef HAS_STDINT_H +#include "stdint.h" +typedef uint64_t ZPOS64_T; +#else + +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef unsigned __int64 ZPOS64_T; +#else +typedef unsigned long long int ZPOS64_T; +#endif +#endif +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define ZLIB_FILEFUNC_SEEK_CUR (1) +#define ZLIB_FILEFUNC_SEEK_END (2) +#define ZLIB_FILEFUNC_SEEK_SET (0) + +#define ZLIB_FILEFUNC_MODE_READ (1) +#define ZLIB_FILEFUNC_MODE_WRITE (2) +#define ZLIB_FILEFUNC_MODE_READWRITEFILTER (3) + +#define ZLIB_FILEFUNC_MODE_EXISTING (4) +#define ZLIB_FILEFUNC_MODE_CREATE (8) + + +#ifndef ZCALLBACK + #if (defined(WIN32) || defined(_WIN32) || defined (WINDOWS) || defined (_WINDOWS)) && defined(CALLBACK) && defined (USEWINDOWS_CALLBACK) + #define ZCALLBACK CALLBACK + #else + #define ZCALLBACK + #endif +#endif + + + + +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); + +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); + + +/* here is the "old" 32 bits structure structure */ +typedef struct zlib_filefunc_def_s +{ + open_file_func zopen_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell_file_func ztell_file; + seek_file_func zseek_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc_def; + +typedef ZPOS64_T (ZCALLBACK *tell64_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek64_file_func) OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +typedef voidpf (ZCALLBACK *open64_file_func) OF((voidpf opaque, const void* filename, int mode)); + +typedef struct zlib_filefunc64_def_s +{ + open64_file_func zopen64_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell64_file_func ztell64_file; + seek64_file_func zseek64_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc64_def; + +void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); + +/* now internal definition, only for zip.c and unzip.h */ +typedef struct zlib_filefunc64_32_def_s +{ + zlib_filefunc64_def zfile_func64; + open_file_func zopen32_file; + tell_file_func ztell32_file; + seek_file_func zseek32_file; +} zlib_filefunc64_32_def; + + +#define ZREAD64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zread_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +#define ZWRITE64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zwrite_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +//#define ZTELL64(filefunc,filestream) ((*((filefunc).ztell64_file)) ((filefunc).opaque,filestream)) +//#define ZSEEK64(filefunc,filestream,pos,mode) ((*((filefunc).zseek64_file)) ((filefunc).opaque,filestream,pos,mode)) +#define ZCLOSE64(filefunc,filestream) ((*((filefunc).zfile_func64.zclose_file)) ((filefunc).zfile_func64.opaque,filestream)) +#define ZERROR64(filefunc,filestream) ((*((filefunc).zfile_func64.zerror_file)) ((filefunc).zfile_func64.opaque,filestream)) + +voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)); +long call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)); +ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)); + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32); + +#define ZOPEN64(filefunc,filename,mode) (call_zopen64((&(filefunc)),(filename),(mode))) +#define ZTELL64(filefunc,filestream) (call_ztell64((&(filefunc)),(filestream))) +#define ZSEEK64(filefunc,filestream,pos,mode) (call_zseek64((&(filefunc)),(filestream),(pos),(mode))) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index 19c9f62b5bf..497fe5e1aa8 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -16,7 +16,7 @@ /*************** Mycat CC Program Source Code File (.CC) ***************/ /* PROGRAM NAME: MYCAT */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* Author: Olivier Bertrand 2012 - 2016 */ /* */ @@ -95,6 +95,9 @@ #if defined(XML_SUPPORT) #include "tabxml.h" #endif // XML_SUPPORT +#if defined(ZIP_SUPPORT) +#include "tabzip.h" +#endif // ZIP_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -154,7 +157,10 @@ TABTYPE GetTypeID(const char *type) #endif : (!stricmp(type, "VIR")) ? TAB_VIR : (!stricmp(type, "JSON")) ? TAB_JSON - : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; +#ifdef ZIP_SUPPORT + : (!stricmp(type, "ZIP")) ? TAB_ZIP +#endif + : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; } // end of GetTypeID /***********************************************************************/ @@ -175,6 +181,7 @@ bool IsFileType(TABTYPE type) case TAB_INI: case TAB_VEC: case TAB_JSON: +// case TAB_ZIP: isfile= true; break; default: @@ -575,7 +582,10 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #endif // PIVOT_SUPPORT case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; - default: +#if defined(ZIP_SUPPORT) + case TAB_ZIP: tdp= new(g) ZIPDEF; break; +#endif // ZIP_SUPPORT + default: sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); } // endswitch diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index e99f5f36444..cb408494319 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -1,9 +1,9 @@ /************** PlgDBSem H Declares Source Code File (.H) **************/ -/* Name: PLGDBSEM.H Version 3.6 */ +/* Name: PLGDBSEM.H Version 3.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ -/* This file contains the PlugDB++ application type definitions. */ +/* This file contains the CONNECT storage engine definitions. */ /***********************************************************************/ /***********************************************************************/ @@ -49,7 +49,8 @@ enum BLKTYP {TYPE_TABLE = 50, /* Table Name/Srcdef/... Block */ TYPE_FB_MAP = 23, /* Mapped file block (storage) */ TYPE_FB_HANDLE = 24, /* File block (handle) */ TYPE_FB_XML = 21, /* DOM XML file block */ - TYPE_FB_XML2 = 27}; /* libxml2 XML file block */ + TYPE_FB_XML2 = 27, /* libxml2 XML file block */ + TYPE_FB_ZIP = 28}; /* ZIP file block */ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_DOS = 1, /* Fixed column offset, variable LRECL */ @@ -78,7 +79,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JCT = 24, /* Junction tables NIY */ TAB_DMY = 25, /* DMY Dummy tables NIY */ TAB_JDBC = 26, /* Table accessed via JDBC */ - TAB_NIY = 27}; /* Table not implemented yet */ + TAB_ZIP = 27, /* ZIP file info table */ + TAB_NIY = 28}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ @@ -140,7 +142,8 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_MYSQL = 192, /* MYSQL access method type no */ TYPE_AM_MYX = 193, /* MYSQL EXEC access method type */ TYPE_AM_CAT = 195, /* Catalog access method type no */ - TYPE_AM_OUT = 200}; /* Output relations (storage) */ + TYPE_AM_ZIP = 198, /* ZIP access method type no */ + TYPE_AM_OUT = 200}; /* Output relations (storage) */ enum RECFM {RECFM_NAF = -2, /* Not a file */ RECFM_OEM = -1, /* OEM file access method */ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index 13c0dfd1e18..31c040c6957 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -68,6 +68,9 @@ #include "tabcol.h" // header of XTAB and COLUMN classes #include "valblk.h" #include "rcmsg.h" +#ifdef ZIP_SUPPORT +#include "filamzip.h" +#endif // ZIP_SUPPORT /***********************************************************************/ /* DB static variables. */ @@ -934,7 +937,16 @@ int PlugCloseFile(PGLOBAL g __attribute__((unused)), PFBLOCK fp, bool all) CloseXML2File(g, fp, all); break; #endif // LIBXML2_SUPPORT - default: +#ifdef ZIP_SUPPORT + case TYPE_FB_ZIP: + ((PZIPFAM)fp->File)->close(); + fp->Memory = NULL; + fp->Mode = MODE_ANY; + fp->Count = 0; + fp->File = NULL; + break; +#endif // ZIP_SUPPORT + default: rc = RC_FX; } // endswitch Type diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 06dde34a27f..9bcac0b5f1a 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,7 +1,7 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9.1 */ +/* Version 4.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -54,6 +54,9 @@ #if defined(GZ_SUPPORT) #include "filamgz.h" #endif // GZ_SUPPORT +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT #include "tabdos.h" #include "tabfix.h" #include "tabmul.h" @@ -93,6 +96,7 @@ DOSDEF::DOSDEF(void) Pseudo = 3; Fn = NULL; Ofn = NULL; + Zipfn = NULL; To_Indx = NULL; Recfm = RECFM_VAR; Mapped = false; @@ -126,7 +130,20 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && !stricmp(am, "DBF")) ? "D" : "V"; - Desc = Fn = GetStringCatInfo(g, "Filename", NULL); + if (*dfm != 'D') + Zipfn = GetStringCatInfo(g, "Zipfile", NULL); + + if (Zipfn && Multiple) { + // Prevent Fn to default to table name + Desc = GetStringCatInfo(g, "Filename", NULL); + Fn = GetStringCatInfo(g, "Filename", "<%>"); + + if (!strcmp(Fn, "<%>")) + Fn = NULL; + + } else + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); + Ofn = GetStringCatInfo(g, "Optname", Fn); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : @@ -333,7 +350,20 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Recfm == RECFM_DBF) { + if (Zipfn) { +#if defined(ZIP_SUPPORT) + if (Recfm == RECFM_VAR) + txfp = new(g) ZIPFAM(this); + else + txfp = new(g) ZPXFAM(this); + + tdbp = new(g) TDBDOS(this, txfp); + return tdbp; +#else // !ZIP_SUPPORT + strcpy(g->Message, "ZIP not supported"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Recfm == RECFM_DBF) { if (Catfunc == FNC_NO) { if (map) txfp = new(g) DBMFAM(this); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index c70e0032f47..501ddbc2e0b 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -28,6 +28,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class TDBFIX; friend class TXTFAM; friend class DBFBASE; + friend class ZIPFAM; public: // Constructor DOSDEF(void); @@ -58,7 +59,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Methods virtual int Indexable(void) - {return (!Multiple && Compressed != 1) ? 1 : 0;} + {return (!Multiple && !Zipfn && Compressed != 1) ? 1 : 0;} virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); @@ -72,7 +73,8 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Members PSZ Fn; /* Path/Name of corresponding file */ PSZ Ofn; /* Base Path/Name of matching index files*/ - PIXDEF To_Indx; /* To index definitions blocks */ + PSZ Zipfn; /* Zip container name */ + PIXDEF To_Indx; /* To index definitions blocks */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ bool Padded; /* true for padded table file */ diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 4a39ecd6e0f..d6649a0093b 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,7 +1,7 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9.1 */ +/* Version 3.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -54,6 +54,9 @@ #if defined(GZ_SUPPORT) #include "filamgz.h" #endif // GZ_SUPPORT +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT #include "tabfmt.h" #include "tabmul.h" #define NO_FUNC @@ -78,20 +81,24 @@ USETEMP UseTemp(void); /* of types (TYPE_STRING < TYPE_DOUBLE < TYPE_INT) (1 < 2 < 7). */ /* If these values are changed, this will have to be revisited. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info) +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) { static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE}; static unsigned int length[] = {6, 6, 8, 10, 10, 6}; - char *p, *colname[MAXCOL], dechar, filename[_MAX_PATH], buf[4096]; + const char *fn; + char sep, q; + int rc, mxr; + bool hdr; + char *p, *colname[MAXCOL], dechar, buf[8]; int i, imax, hmax, n, nerr, phase, blank, digit, dec, type; int ncol = sizeof(buftyp) / sizeof(int); int num_read = 0, num_max = 10000000; // Statistics int len[MAXCOL], typ[MAXCOL], prc[MAXCOL]; - FILE *infile; + PCSVDEF tdp; + PTDBCSV tdbp; PQRYRES qrp; PCOLRES crp; @@ -102,26 +109,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endif info // num_max = atoi(p+1); // Max num of record to test -#if defined(__WIN__) - if (sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) - dechar = '.'; - else - dechar = ','; -#else // !__WIN__ - dechar = '.'; -#endif // !__WIN__ - - if (trace) - htrc("File %s sep=%c q=%c hdr=%d mxr=%d\n", - SVP(fn), sep, q, hdr, mxr); - - if (!fn) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } // endif fn - imax = hmax = nerr = 0; - mxr = MY_MAX(0, mxr); for (i = 0; i < MAXCOL; i++) { colname[i] = NULL; @@ -131,12 +119,73 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endfor i /*********************************************************************/ - /* Open the input file. */ + /* Get the CSV table description block. */ /*********************************************************************/ - PlugSetPath(filename, fn, dp); + tdp = new(g) CSVDEF; +#if defined(ZIP_SUPPORT) + tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL); + tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + + fn = (tdp->Fn) ? tdp->Fn : "unnamed"; + + if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) + tdp->Lrecl = 4096; - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "r"))) - return NULL; + p = GetStringTableOption(g, topt, "Separator", ","); + tdp->Sep = (strlen(p) == 2 && p[0] == '\\' && p[1] == 't') ? '\t' : *p; + +#if defined(__WIN__) + if (tdp->Sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) + dechar = '.'; + else + dechar = ','; +#else // !__WIN__ + dechar = '.'; +#endif // !__WIN__ + + sep = tdp->Sep; + tdp->Quoted = GetIntegerTableOption(g, topt, "Quoted", -1); + p = GetStringTableOption(g, topt, "Qchar", ""); + tdp->Qot = *p; + + if (tdp->Qot && tdp->Quoted < 0) + tdp->Quoted = 0; + else if (!tdp->Qot && tdp->Quoted >= 0) + tdp->Qot = '"'; + + q = tdp->Qot; + hdr = GetBooleanTableOption(g, topt, "Header", false); + tdp->Maxerr = GetIntegerTableOption(g, topt, "Maxerr", 0); + tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); + + if (tdp->Accept && tdp->Maxerr == 0) + tdp->Maxerr = INT_MAX32; // Accept all bad lines + + mxr = MY_MAX(0, tdp->Maxerr); + + if (trace) + htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", + SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); + + if (tdp->Zipfn) + tdbp = new(g) TDBCSV(tdp, new(g) ZIPFAM(tdp)); + else + tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); + + tdbp->SetMode(MODE_READ); + + /*********************************************************************/ + /* Open the CSV file. */ + /*********************************************************************/ + if (tdbp->OpenDB(g)) + return NULL; if (hdr) { /*******************************************************************/ @@ -144,16 +193,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ phase = 0; - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf) + 1; - buf[n - 2] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 3] == '\r') - buf[n - 3] = 0; -#endif // UNIX - p = (char*)PlugSubAlloc(g, NULL, n); - memcpy(p, buf, n); + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + p = PlgDBDup(g, tdbp->To_Line); //skip leading blanks for (; *p == ' '; p++) ; @@ -165,10 +206,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endif q colname[0] = p; - } else { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(FILE_IS_EMPTY), fn); goto err; - } // endif's + } else + goto err; for (i = 1; *p; p++) if (phase == 1 && *p == q) { @@ -201,15 +243,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ /* Now start the reading process. Read one line. */ /*******************************************************************/ - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf); - buf[n - 1] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 2] == '\r') - buf[n - 2] = 0; -#endif // UNIX - } else if (feof(infile)) { + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(EOF_AFTER_LINE), num_read -1); break; } else { @@ -222,7 +257,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ i = n = phase = blank = digit = dec = 0; - for (p = buf; *p; p++) + for (p = tdbp->To_Line; *p; p++) if (*p == sep) { if (phase != 1) { if (i == MAXCOL - 1) { @@ -331,7 +366,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, htrc("\n"); } // endif trace - fclose(infile); + tdbp->CloseDB(g); skipit: if (trace) @@ -381,7 +416,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, return qrp; err: - fclose(infile); + tdbp->CloseDB(g); return NULL; } // end of CSVCColumns @@ -458,7 +493,21 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) /*******************************************************************/ /* Allocate a file processing class of the proper type. */ /*******************************************************************/ - if (map) { + if (Zipfn) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); + + if (!Fmtd) + tdbp = new(g) TDBCSV(this, txfp); + else + tdbp = new(g) TDBFMT(this, txfp); + + return tdbp; +#else // !ZIP_SUPPORT + strcpy(g->Message, "ZIP not supported"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (map) { // Should be now compatible with UNIX txfp = new(g) MAPFAM(this); } else if (Compressed) { @@ -1476,21 +1525,16 @@ void CSVCOL::WriteColumn(PGLOBAL g) /* TDBCCL class constructor. */ /***********************************************************************/ TDBCCL::TDBCCL(PCSVDEF tdp) : TDBCAT(tdp) - { - Fn = tdp->GetFn(); - Hdr = tdp->Header; - Mxr = tdp->Maxerr; - Qtd = tdp->Quoted; - Sep = tdp->Sep; - } // end of TDBCCL constructor +{ + Topt = tdp->GetTopt(); +} // end of TDBCCL constructor /***********************************************************************/ /* GetResult: Get the list the CSV file columns. */ /***********************************************************************/ PQRYRES TDBCCL::GetResult(PGLOBAL g) { - return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), - Fn, Sep, Qtd, Hdr, Mxr, false); + return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), Topt, false); } // end of GetResult /* ------------------------ End of TabFmt ---------------------------- */ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index ce80a276cdc..5ce8d399a64 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -1,7 +1,7 @@ /*************** TabFmt H Declares Source Code File (.H) ***************/ -/* Name: TABFMT.H Version 2.4 */ +/* Name: TABFMT.H Version 2.5 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the CSV and FMT classes declares. */ /***********************************************************************/ @@ -13,8 +13,7 @@ typedef class TDBFMT *PTDBFMT; /***********************************************************************/ /* Functions used externally. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info); +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info); /***********************************************************************/ /* CSV table. */ @@ -22,7 +21,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, class DllExport CSVDEF : public DOSDEF { /* Logical table description */ friend class TDBCSV; friend class TDBCCL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor CSVDEF(void); @@ -50,9 +50,10 @@ class DllExport CSVDEF : public DOSDEF { /* Logical table description */ /* This is the DOS/UNIX Access Method class declaration for files */ /* that are CSV files with columns separated by the Sep character. */ /***********************************************************************/ -class TDBCSV : public TDBDOS { +class DllExport TDBCSV : public TDBDOS { friend class CSVCOL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor TDBCSV(PCSVDEF tdp, PTXF txfp); TDBCSV(PGLOBAL g, PTDBCSV tdbp); @@ -101,7 +102,7 @@ class TDBCSV : public TDBDOS { /* Class CSVCOL: CSV access method column descriptor. */ /* This A.M. is used for Comma Separated V(?) files. */ /***********************************************************************/ -class CSVCOL : public DOSCOL { +class DllExport CSVCOL : public DOSCOL { friend class TDBCSV; friend class TDBFMT; public: @@ -129,7 +130,7 @@ class CSVCOL : public DOSCOL { /* This is the DOS/UNIX Access Method class declaration for files */ /* whose record format is described by a Format keyword. */ /***********************************************************************/ -class TDBFMT : public TDBCSV { +class DllExport TDBFMT : public TDBCSV { friend class CSVCOL; //friend class FMTCOL; public: @@ -173,7 +174,7 @@ class TDBFMT : public TDBCSV { /***********************************************************************/ /* This is the class declaration for the CSV catalog table. */ /***********************************************************************/ -class TDBCCL : public TDBCAT { +class DllExport TDBCCL : public TDBCAT { public: // Constructor TDBCCL(PCSVDEF tdp); @@ -183,11 +184,7 @@ class TDBCCL : public TDBCAT { virtual PQRYRES GetResult(PGLOBAL g); // Members - char *Fn; // The CSV file (path) name - bool Hdr; // true if first line contains headers - int Mxr; // Maximum number of bad records - int Qtd; // Quoting level for quoted fields - char Sep; // Separator for standard CSV files - }; // end of class TDBCCL + PTOS Topt; +}; // end of class TDBCCL /* ------------------------- End of TabFmt.H ------------------------- */ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 5f864f0bd48..73c6a6d85a4 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,5 +1,5 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.2 */ +/* PROGRAM NAME: tabjson Version 1.3 */ /* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -28,6 +28,9 @@ #if defined(GZ_SUPPORT) #include "filamgz.h" #endif // GZ_SUPPORT +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT #include "tabmul.h" #include "checklvl.h" #include "resource.h" @@ -67,7 +70,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - char *fn, colname[65], fmt[129]; + char colname[65], fmt[129]; int i, j, lvl, n = 0; int ncol = sizeof(buftyp) / sizeof(int); PVAL valp; @@ -94,16 +97,21 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ - if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; - } // endif fn + lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; tdp = new(g) JSONDEF; - tdp->Fn = fn; +#if defined(ZIP_SUPPORT) + tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL); + tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + tdp->Database = SetPath(g, db); tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; @@ -114,7 +122,10 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + if (tdp->Zipfn) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); + else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) return NULL; @@ -127,10 +138,28 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) } // endif lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + if (tdp->Zipfn) + tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp)); + else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + tjnp->SetMode(MODE_READ); - if (tjnp->OpenDB(g)) +#if USE_G + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G, G->Sarea, G->Sarea_Size); + G->jump_level = -1; + tjnp->SetG(G); +#else + tjnp->SetG(g); +#endif + + if (tjnp->OpenDB(g)) return NULL; switch (tjnp->ReadDB(g)) { @@ -395,7 +424,14 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Compressed) { + if (Zipfn) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Compressed) { #if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) GZFAM(this); @@ -426,12 +462,16 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - txfp = new(g) MAPFAM(this); + if (Zipfn) + txfp = new(g) ZIPFAM(this); + else + txfp = new(g) MAPFAM(this); + tdbp = new(g) TDBJSON(this, txfp); ((TDBJSON*)tdbp)->G = g; } // endif Pretty - if (Multiple) + if (Multiple && !Zipfn) tdbp = new(g) TDBMUL(tdbp); return tdbp; diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index f7cb74c3c4d..c9d30d48f2a 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -78,7 +78,8 @@ public: virtual AMT GetAmType(void) {return TYPE_AM_JSN;} virtual bool SkipHeader(PGLOBAL g); virtual PTDB Duplicate(PGLOBAL g) {return (PTDB)new(g) TDBJSN(this);} - PJSON GetRow(void) {return Row;} + PJSON GetRow(void) {return Row;} + void SetG(PGLOBAL g) {G = g;} // Methods virtual PTDB CopyOne(PTABS t); diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp new file mode 100644 index 00000000000..11f414ee154 --- /dev/null +++ b/storage/connect/tabzip.cpp @@ -0,0 +1,230 @@ +/************* TabZip C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: TABZIP Version 1.0 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* This program are the TABZIP class DB execution routines. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the MariaDB header file. */ +/***********************************************************************/ +#include + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* (x)table.h is header containing the TDBASE declarations. */ +/* tabzip.h is header containing the TABZIP classes declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "filamtxt.h" +#include "filamzip.h" +#include "resource.h" // for IDS_COLUMNS +#include "tabdos.h" +#include "tabzip.h" + +/* -------------------------- Class ZIPDEF --------------------------- */ + +/************************************************************************/ +/* DefineAM: define specific AM block values. */ +/************************************************************************/ +bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) +{ +//target = GetStringCatInfo(g, "Target", NULL); + return DOSDEF::DefineAM(g, "ZIP", poff); +} // end of DefineAM + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) +{ + return new(g) TDBZIP(this); +} // end of GetTable + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the TDBZIP class. */ +/***********************************************************************/ +TDBZIP::TDBZIP(PZIPDEF tdp) : TDBASE(tdp) +{ + zipfile = NULL; + zfn = tdp->Fn; +//target = tdp->target; + nexterr = UNZ_OK; +} // end of TDBZIP standard constructor + +/***********************************************************************/ +/* Allocate ZIP column description block. */ +/***********************************************************************/ +PCOL TDBZIP::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) +{ + return new(g) ZIPCOL(cdp, this, cprec, n); +} // end of MakeCol + +/***********************************************************************/ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ +/***********************************************************************/ +bool TDBZIP::open(PGLOBAL g, const char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error"); + + return (zipfile == NULL); +} // end of open + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void TDBZIP::close() +{ + if (zipfile) { + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile + +} // end of close + +/***********************************************************************/ +/* ZIP Cardinality: returns table size in number of rows. */ +/***********************************************************************/ +int TDBZIP::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + else if (Cardinal < 0) { + if (!open(g, zfn)) { + unz_global_info64 ginfo; + int err = unzGetGlobalInfo64(zipfile, &ginfo); + + Cardinal = (err == UNZ_OK) ? ginfo.number_entry : 0; + } else + Cardinal = 0; + + } // endif Cardinal + + return Cardinal; +} // end of Cardinality + +/***********************************************************************/ +/* ZIP GetMaxSize: returns file size estimate in number of lines. */ +/***********************************************************************/ +int TDBZIP::GetMaxSize(PGLOBAL g) +{ + if (MaxSize < 0) + MaxSize = Cardinality(g); + + return MaxSize; +} // end of GetMaxSize + +/***********************************************************************/ +/* ZIP Access Method opening routine. */ +/***********************************************************************/ +bool TDBZIP::OpenDB(PGLOBAL g) +{ + if (Use == USE_OPEN) + // Table already open + return false; + + Use = USE_OPEN; // To be clean + return open(g, zfn); +} // end of OpenDB + +/***********************************************************************/ +/* ReadDB: Data Base read routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::ReadDB(PGLOBAL g) +{ + if (nexterr == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (nexterr != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile error %d", nexterr); + return RC_FX; + } // endif nexterr + + int err = unzGetCurrentFileInfo64(zipfile, &finfo, fn, + sizeof(fn), NULL, 0, NULL, 0); + + if (err != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 error %d", err); + return RC_FX; + } // endif err + + nexterr = unzGoToNextFile(zipfile); + return RC_OK; +} // end of ReadDB + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::WriteDB(PGLOBAL g) +{ + strcpy(g->Message, "ZIP tables are read only"); + return RC_FX; +} // end of WriteDB + +/***********************************************************************/ +/* Data Base delete line routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::DeleteDB(PGLOBAL g, int irc) +{ + strcpy(g->Message, "Delete not enabled for ZIP tables"); + return RC_FX; +} // end of DeleteDB + +/***********************************************************************/ +/* Data Base close routine for ZIP access method. */ +/***********************************************************************/ +void TDBZIP::CloseDB(PGLOBAL g) +{ + close(); + Use = USE_READY; // Just to be clean +} // end of CloseDB + +/* ---------------------------- ZIPCOL ------------------------------- */ + +/***********************************************************************/ +/* ZIPCOL public constructor. */ +/***********************************************************************/ +ZIPCOL::ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am) + : COLBLK(cdp, tdbp, i) +{ + if (cprec) { + Next = cprec->GetNext(); + cprec->SetNext(this); + } else { + Next = tdbp->GetColumns(); + tdbp->SetColumns(this); + } // endif cprec + + Tdbz = (TDBZIP*)tdbp; + flag = cdp->GetOffset(); +} // end of ZIPCOL constructor + +/***********************************************************************/ +/* ReadColumn: */ +/***********************************************************************/ +void ZIPCOL::ReadColumn(PGLOBAL g) +{ + switch (flag) { + case 1: + Value->SetValue(Tdbz->finfo.compressed_size); + break; + case 2: + Value->SetValue(Tdbz->finfo.uncompressed_size); + break; + case 3: + Value->SetValue((int)Tdbz->finfo.compression_method); + break; + default: + Value->SetValue_psz((PSZ)Tdbz->fn); + } // endswitch flag + +} // end of ReadColumn + +/* -------------------------- End of tabzip -------------------------- */ diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h new file mode 100644 index 00000000000..6f1735258e7 --- /dev/null +++ b/storage/connect/tabzip.h @@ -0,0 +1,100 @@ +/*************** tabzip H Declares Source Code File (.H) ***************/ +/* Name: tabzip.h Version 1.0 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* This file contains the ZIP classe declares. */ +/***********************************************************************/ +#include "osutil.h" +#include "block.h" +#include "colblk.h" +#include "xtable.h" +#include "unzip.h" + +typedef class ZIPDEF *PZIPDEF; +typedef class TDBZIP *PTDBZIP; +typedef class ZIPCOL *PZIPCOL; + +/***********************************************************************/ +/* ZIP table: display info about a ZIP file. */ +/***********************************************************************/ +class DllExport ZIPDEF : public DOSDEF { /* Table description */ + friend class TDBZIP; + friend class ZIPFAM; +public: + // Constructor + ZIPDEF(void) {} + + // Implementation + virtual const char *GetType(void) {return "ZIP";} + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + +protected: + // Members + PSZ target; // The inside file to query +}; // end of ZIPDEF + +/***********************************************************************/ +/* This is the ZIP Access Method class declaration. */ +/***********************************************************************/ +class DllExport TDBZIP : public TDBASE { + friend class ZIPCOL; +public: + // Constructor + TDBZIP(PZIPDEF tdp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + + // Methods + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); + virtual int Cardinality(PGLOBAL g); + virtual int GetMaxSize(PGLOBAL g); + virtual int GetRecpos(void) {return 0;} + + // Database routines + virtual bool OpenDB(PGLOBAL g); + virtual int ReadDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); + virtual int DeleteDB(PGLOBAL g, int irc); + virtual void CloseDB(PGLOBAL g); + +protected: + bool open(PGLOBAL g, const char *filename); + void close(void); + + // Members + unzFile zipfile; // The ZIP container file + PSZ zfn; // The ZIP file name +//PSZ target; + unz_file_info64 finfo; // The current file info + char fn[FILENAME_MAX]; // The current file name + int nexterr; // Next file error +}; // end of class TDBZIP + +/***********************************************************************/ +/* Class ZIPCOL: ZIP access method column descriptor. */ +/***********************************************************************/ +class DllExport ZIPCOL : public COLBLK { + friend class TDBZIP; +public: + // Constructors + ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am = "ZIP"); + + // Implementation + virtual int GetAmType(void) { return TYPE_AM_ZIP; } + + // Methods + virtual void ReadColumn(PGLOBAL g); + +protected: + // Default constructor not to be used + ZIPCOL(void) {} + + // Members + TDBZIP *Tdbz; + int flag; +}; // end of class ZIPCOL diff --git a/storage/connect/unzip.c b/storage/connect/unzip.c new file mode 100644 index 00000000000..909350435a5 --- /dev/null +++ b/storage/connect/unzip.c @@ -0,0 +1,2125 @@ +/* unzip.c -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + + ------------------------------------------------------------------------------------ + Decryption code comes from crypt.c by Info-ZIP but has been greatly reduced in terms of + compatibility with older software. The following is from the original crypt.c. + Code woven in by Terry Thorsen 1/2003. + + Copyright (c) 1990-2000 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html + + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + ------------------------------------------------------------------------------------ + + Changes in unzip.c + + 2007-2008 - Even Rouault - Addition of cpl_unzGetCurrentFileZStreamPos + 2007-2008 - Even Rouault - Decoration of symbol names unz* -> cpl_unz* + 2007-2008 - Even Rouault - Remove old C style function prototypes + 2007-2008 - Even Rouault - Add unzip support for ZIP64 + + Copyright (C) 2007-2008 Even Rouault + + + Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again). + Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G + should only read the compressed/uncompressed size from the Zip64 format if + the size from normal header was 0xFFFFFFFF + Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant + Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required) + Patch created by Daniel Borca + + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + + Copyright (C) 1998 - 2010 Gilles Vollant, Even Rouault, Mathias Svensson + +*/ + + +#include +#include +#include + +#ifndef NOUNCRYPT + #define NOUNCRYPT +#endif + +#include "zlib.h" +#include "unzip.h" + +#ifdef STDC +# include +# include +# include +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + + +#ifndef CASESENSITIVITYDEFAULT_NO +# if !defined(unix) && !defined(CASESENSITIVITYDEFAULT_YES) +# define CASESENSITIVITYDEFAULT_NO +# endif +#endif + + +#ifndef UNZ_BUFSIZE +#define UNZ_BUFSIZE (16384) +#endif + +#ifndef UNZ_MAXFILENAMEINZIP +#define UNZ_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) + + +const char unz_copyright[] = + " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + +/* unz_file_info_interntal contain internal info about a file in zipfile*/ +typedef struct unz_file_info64_internal_s +{ + ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ +} unz_file_info64_internal; + + +/* file_in_zip_read_info_s contain internal information about a file in zipfile, + when reading and decompress it */ +typedef struct +{ + char *read_buffer; /* internal buffer for compressed data */ + z_stream stream; /* zLib stream structure for inflate */ + +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + ZPOS64_T pos_in_zipfile; /* position in byte on the zipfile, for fseek*/ + uLong stream_initialised; /* flag set if stream structure is initialised*/ + + ZPOS64_T offset_local_extrafield;/* offset of the local extra field */ + uInt size_local_extrafield;/* size of the local extra field */ + ZPOS64_T pos_local_extrafield; /* position in the local extra field in read*/ + ZPOS64_T total_out_64; + + uLong crc32; /* crc32 of all data uncompressed */ + uLong crc32_wait; /* crc32 we must obtain after decompress all */ + ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */ + ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + uLong compression_method; /* compression method (0==store) */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + int raw; +} file_in_zip64_read_info_s; + + +/* unz64_s contain internal information about the zipfile +*/ +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + int is64bitOpenFunction; + voidpf filestream; /* io structore of the zipfile */ + unz_global_info64 gi; /* public global information */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + ZPOS64_T num_file; /* number of the current file in the zipfile*/ + ZPOS64_T pos_in_central_dir; /* pos of the current file in the central dir*/ + ZPOS64_T current_file_ok; /* flag about the usability of the current file*/ + ZPOS64_T central_pos; /* position of the beginning of the central dir*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory with + respect to the starting disk number */ + + unz_file_info64 cur_file_info; /* public info about the current file in zip*/ + unz_file_info64_internal cur_file_info_internal; /* private info about it*/ + file_in_zip64_read_info_s* pfile_in_zip_read; /* structure about the current + file if we are decompressing it */ + int encrypted; + + int isZip64; + +# ifndef NOUNCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; +# endif +} unz64_s; + + +#ifndef NOUNCRYPT +#include "crypt.h" +#endif + +/* =========================================================================== + Read a byte from a gz_stream; update next_in and avail_in. Return EOF + for end of file. + IN assertion: the stream s has been sucessfully opened for reading. +*/ + + +local int unz64local_getByte OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + int *pi)); + +local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return UNZ_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return UNZ_ERRNO; + else + return UNZ_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int unz64local_getShort OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX)); + + +local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX) +{ + ZPOS64_T x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<24; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<32; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<40; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<48; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<56; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +/* My own strcmpi / strcasecmp */ +local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2) +{ + for (;;) + { + char c1=*(fileName1++); + char c2=*(fileName2++); + if ((c1>='a') && (c1<='z')) + c1 -= 0x20; + if ((c2>='a') && (c2<='z')) + c2 -= 0x20; + if (c1=='\0') + return ((c2=='\0') ? 0 : -1); + if (c2=='\0') + return 1; + if (c1c2) + return 1; + } +} + + +#ifdef CASESENSITIVITYDEFAULT_NO +#define CASESENSITIVITYDEFAULTVALUE 2 +#else +#define CASESENSITIVITYDEFAULTVALUE 1 +#endif + +#ifndef STRCMPCASENOSENTIVEFUNCTION +#define STRCMPCASENOSENTIVEFUNCTION strcmpcasenosensitive_internal +#endif + +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) + +*/ +extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, + const char* fileName2, + int iCaseSensitivity) + +{ + if (iCaseSensitivity==0) + iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE; + + if (iCaseSensitivity==1) + return strcmp(fileName1,fileName2); + + return STRCMPCASENOSENTIVEFUNCTION(fileName1,fileName2); +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif + +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); +local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackReaduMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + + +/* + Locate the Central directory 64 of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream)); + +local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackReaduMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK) + return 0; + + /* total number of disks */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + if (uL != 0x06064b50) + return 0; + + return relativeOffset; +} + +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows NT computer "c:\\test\\zlib114.zip" or on an Unix computer + "zlib/zlib114.zip". + If the zipfile cannot be opened (file doesn't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. +*/ +local unzFile unzOpenInternal (const void *path, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def, + int is64bitOpenFunction) +{ + unz64_s us; + unz64_s *s; + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + + int err=UNZ_OK; + + if (unz_copyright[0]!=' ') + return NULL; + + us.z_filefunc.zseek32_file = NULL; + us.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&us.z_filefunc.zfile_func64); + else + us.z_filefunc = *pzlib_filefunc64_32_def; + us.is64bitOpenFunction = is64bitOpenFunction; + + + + us.filestream = ZOPEN64(us.z_filefunc, + path, + ZLIB_FILEFUNC_MODE_READ | + ZLIB_FILEFUNC_MODE_EXISTING); + if (us.filestream==NULL) + return NULL; + + central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream); + if (central_pos) + { + uLong uS; + ZPOS64_T uL64; + + us.isZip64 = 1; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* size of zip64 end of central directory record */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&uL64)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version made by */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version needed to extract */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.gi.number_entry)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&number_entry_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.size_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.offset_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + us.gi.size_comment = 0; + } + else + { + central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream); + if (central_pos==0) + err=UNZ_ERRNO; + + us.isZip64 = 0; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central dir on this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.gi.number_entry = uL; + + /* total number of entries in the central dir */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + number_entry_CD = uL; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.size_central_dir = uL; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.offset_central_dir = uL; + + /* zipfile comment length */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&us.gi.size_comment)!=UNZ_OK) + err=UNZ_ERRNO; + } + + if ((central_pospfile_in_zip_read!=NULL) + unzCloseCurrentFile(file); + + ZCLOSE64(s->z_filefunc, s->filestream); + TRYFREE(s); + return UNZ_OK; +} + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + *pglobal_info=s->gi; + return UNZ_OK; +} + +extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + /* to do : check if number_entry is not truncated */ + pglobal_info32->number_entry = (uLong)s->gi.number_entry; + pglobal_info32->size_comment = s->gi.size_comment; + return UNZ_OK; +} +/* + Translate date/time from Dos format to tm_unz (readable more easilty) +*/ +local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm) +{ + ZPOS64_T uDate; + uDate = (ZPOS64_T)(ulDosDate>>16); + ptm->tm_mday = (uInt)(uDate&0x1f) ; + ptm->tm_mon = (uInt)((((uDate)&0x1E0)/0x20)-1) ; + ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ; + + ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800); + ptm->tm_min = (uInt) ((ulDosDate&0x7E0)/0x20) ; + ptm->tm_sec = (uInt) (2*(ulDosDate&0x1f)) ; +} + +/* + Get Info about the current file in the zipfile, with internal only info +*/ +local int unz64local_GetCurrentFileInfoInternal OF((unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +local int unz64local_GetCurrentFileInfoInternal (unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize) +{ + unz64_s* s; + unz_file_info64 file_info; + unz_file_info64_internal file_info_internal; + int err=UNZ_OK; + uLong uMagic; + long lSeek=0; + uLong uL; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pos_in_central_dir+s->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + + /* we check the magic */ + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x02014b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version_needed) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.flag) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.compression_method) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.dosDate) != UNZ_OK) + err=UNZ_ERRNO; + + unz64local_DosDateToTmuDate(file_info.dosDate,&file_info.tmu_date); + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.crc) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.compressed_size = uL; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.uncompressed_size = uL; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_filename) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_extra) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_comment) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.internal_fa) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.external_fa) != UNZ_OK) + err=UNZ_ERRNO; + + // relative offset of local header + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info_internal.offset_curfile = uL; + + lSeek+=file_info.size_filename; + if ((err==UNZ_OK) && (szFileName!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_filename0) && (fileNameBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szFileName,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek -= uSizeRead; + } + + // Read extrafield + if ((err==UNZ_OK) && (extraField!=NULL)) + { + ZPOS64_T uSizeRead ; + if (file_info.size_file_extraz_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,extraField,(uLong)uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + + lSeek += file_info.size_file_extra - (uLong)uSizeRead; + } + else + lSeek += file_info.size_file_extra; + + + if ((err==UNZ_OK) && (file_info.size_file_extra != 0)) + { + uLong acc = 0; + + // since lSeek now points to after the extra field we need to move back + lSeek -= file_info.size_file_extra; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + while(acc < file_info.size_file_extra) + { + uLong headerId; + uLong dataSize; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&headerId) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&dataSize) != UNZ_OK) + err=UNZ_ERRNO; + + /* ZIP64 extra fields */ + if (headerId == 0x0001) + { + uLong uL; + + if(file_info.uncompressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.compressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info_internal.offset_curfile == MAXU32) + { + /* Relative Header offset */ + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.disk_num_start == MAXU32) + { + /* Disk Start Number */ + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + } + + } + else + { + if (ZSEEK64(s->z_filefunc, s->filestream,dataSize,ZLIB_FILEFUNC_SEEK_CUR)!=0) + err=UNZ_ERRNO; + } + + acc += 2 + 2 + dataSize; + } + } + + if ((err==UNZ_OK) && (szComment!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_file_commentz_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_comment>0) && (commentBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek+=file_info.size_file_comment - uSizeRead; + } + else + lSeek+=file_info.size_file_comment; + + + if ((err==UNZ_OK) && (pfile_info!=NULL)) + *pfile_info=file_info; + + if ((err==UNZ_OK) && (pfile_info_internal!=NULL)) + *pfile_info_internal=file_info_internal; + + return err; +} + + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. +*/ +extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file, + unz_file_info64 * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); +} + +extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, + unz_file_info * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + int err; + unz_file_info64 file_info64; + err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); + if ((err==UNZ_OK) && (pfile_info != NULL)) + { + pfile_info->version = file_info64.version; + pfile_info->version_needed = file_info64.version_needed; + pfile_info->flag = file_info64.flag; + pfile_info->compression_method = file_info64.compression_method; + pfile_info->dosDate = file_info64.dosDate; + pfile_info->crc = file_info64.crc; + + pfile_info->size_filename = file_info64.size_filename; + pfile_info->size_file_extra = file_info64.size_file_extra; + pfile_info->size_file_comment = file_info64.size_file_comment; + + pfile_info->disk_num_start = file_info64.disk_num_start; + pfile_info->internal_fa = file_info64.internal_fa; + pfile_info->external_fa = file_info64.external_fa; + + pfile_info->tmu_date = file_info64.tmu_date, + + + pfile_info->compressed_size = (uLong)file_info64.compressed_size; + pfile_info->uncompressed_size = (uLong)file_info64.uncompressed_size; + + } + return err; +} +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ +extern int ZEXPORT unzGoToFirstFile (unzFile file) +{ + int err=UNZ_OK; + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + s->pos_in_central_dir=s->offset_central_dir; + s->num_file=0; + err=unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ +extern int ZEXPORT unzGoToNextFile (unzFile file) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + if (s->gi.number_entry != 0xffff) /* 2^16 files overflow hack */ + if (s->num_file+1==s->gi.number_entry) + return UNZ_END_OF_LIST_OF_FILE; + + s->pos_in_central_dir += SIZECENTRALDIRITEM + s->cur_file_info.size_filename + + s->cur_file_info.size_file_extra + s->cur_file_info.size_file_comment ; + s->num_file++; + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + + +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ +extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity) +{ + unz64_s* s; + int err; + + /* We remember the 'current' position in the file so that we can jump + * back there if we fail. + */ + unz_file_info64 cur_file_infoSaved; + unz_file_info64_internal cur_file_info_internalSaved; + ZPOS64_T num_fileSaved; + ZPOS64_T pos_in_central_dirSaved; + + + if (file==NULL) + return UNZ_PARAMERROR; + + if (strlen(szFileName)>=UNZ_MAXFILENAMEINZIP) + return UNZ_PARAMERROR; + + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + /* Save the current state */ + num_fileSaved = s->num_file; + pos_in_central_dirSaved = s->pos_in_central_dir; + cur_file_infoSaved = s->cur_file_info; + cur_file_info_internalSaved = s->cur_file_info_internal; + + err = unzGoToFirstFile(file); + + while (err == UNZ_OK) + { + char szCurrentFileName[UNZ_MAXFILENAMEINZIP+1]; + err = unzGetCurrentFileInfo64(file,NULL, + szCurrentFileName,sizeof(szCurrentFileName)-1, + NULL,0,NULL,0); + if (err == UNZ_OK) + { + if (unzStringFileNameCompare(szCurrentFileName, + szFileName,iCaseSensitivity)==0) + return UNZ_OK; + err = unzGoToNextFile(file); + } + } + + /* We failed, so restore the state of the 'current file' to where we + * were. + */ + s->num_file = num_fileSaved ; + s->pos_in_central_dir = pos_in_central_dirSaved ; + s->cur_file_info = cur_file_infoSaved; + s->cur_file_info_internal = cur_file_info_internalSaved; + return err; +} + + +/* +/////////////////////////////////////////// +// Contributed by Ryan Haksi (mailto://cryogen@infoserve.net) +// I need random access +// +// Further optimization could be realized by adding an ability +// to cache the directory in memory. The goal being a single +// comprehensive file read to put the file I need in a memory. +*/ + +/* +typedef struct unz_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; // offset in file + ZPOS64_T num_of_file; // # of file +} unz_file_pos; +*/ + +extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) +{ + unz64_s* s; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + file_pos->pos_in_zip_directory = s->pos_in_central_dir; + file_pos->num_of_file = s->num_file; + + return UNZ_OK; +} + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + int err = unzGetFilePos64(file,&file_pos64); + if (err==UNZ_OK) + { + file_pos->pos_in_zip_directory = (uLong)file_pos64.pos_in_zip_directory; + file_pos->num_of_file = (uLong)file_pos64.num_of_file; + } + return err; +} + +extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) +{ + unz64_s* s; + int err; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + /* jump to the right spot */ + s->pos_in_central_dir = file_pos->pos_in_zip_directory; + s->num_file = file_pos->num_of_file; + + /* set the current file */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + /* return results */ + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + if (file_pos == NULL) + return UNZ_PARAMERROR; + + file_pos64.pos_in_zip_directory = file_pos->pos_in_zip_directory; + file_pos64.num_of_file = file_pos->num_of_file; + return unzGoToFilePos64(file,&file_pos64); +} + +/* +// Unzip Helper Functions - should be here? +/////////////////////////////////////////// +*/ + +/* + Read the local header of the current zipfile + Check the coherency of the local header and info in the end of central + directory about this file + store in *piSizeVar the size of extra info in local header + (filename and size of extra field data) +*/ +local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar, + ZPOS64_T * poffset_local_extrafield, + uInt * psize_local_extrafield) +{ + uLong uMagic,uData,uFlags; + uLong size_filename; + uLong size_extra_field; + int err=UNZ_OK; + + *piSizeVar = 0; + *poffset_local_extrafield = 0; + *psize_local_extrafield = 0; + + if (ZSEEK64(s->z_filefunc, s->filestream,s->cur_file_info_internal.offset_curfile + + s->byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x04034b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; +/* + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.wVersion)) + err=UNZ_BADZIPFILE; +*/ + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uFlags) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.compression_method)) + err=UNZ_BADZIPFILE; + + if ((err==UNZ_OK) && (s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* date/time */ + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* crc */ + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.crc) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size compr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.compressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size uncompr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.uncompressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_filename) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (size_filename!=s->cur_file_info.size_filename)) + err=UNZ_BADZIPFILE; + + *piSizeVar += (uInt)size_filename; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_extra_field) != UNZ_OK) + err=UNZ_ERRNO; + *poffset_local_extrafield= s->cur_file_info_internal.offset_curfile + + SIZEZIPLOCALHEADER + size_filename; + *psize_local_extrafield = (uInt)size_extra_field; + + *piSizeVar += (uInt)size_extra_field; + + return err; +} + +/* + Open for reading data the current file in the zipfile. + If there is no error and the file is opened, the return value is UNZ_OK. +*/ +extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, + int* level, int raw, const char* password) +{ + int err=UNZ_OK; + uInt iSizeVar; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + ZPOS64_T offset_local_extrafield; /* offset of the local extra field */ + uInt size_local_extrafield; /* size of the local extra field */ +# ifndef NOUNCRYPT + char source[12]; +# else + if (password != NULL) + return UNZ_PARAMERROR; +# endif + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_PARAMERROR; + + if (s->pfile_in_zip_read != NULL) + unzCloseCurrentFile(file); + + if (unz64local_CheckCurrentFileCoherencyHeader(s,&iSizeVar, &offset_local_extrafield,&size_local_extrafield)!=UNZ_OK) + return UNZ_BADZIPFILE; + + pfile_in_zip_read_info = (file_in_zip64_read_info_s*)ALLOC(sizeof(file_in_zip64_read_info_s)); + if (pfile_in_zip_read_info==NULL) + return UNZ_INTERNALERROR; + + pfile_in_zip_read_info->read_buffer=(char*)ALLOC(UNZ_BUFSIZE); + pfile_in_zip_read_info->offset_local_extrafield = offset_local_extrafield; + pfile_in_zip_read_info->size_local_extrafield = size_local_extrafield; + pfile_in_zip_read_info->pos_local_extrafield=0; + pfile_in_zip_read_info->raw=raw; + + if (pfile_in_zip_read_info->read_buffer==NULL) + { + TRYFREE(pfile_in_zip_read_info); + return UNZ_INTERNALERROR; + } + + pfile_in_zip_read_info->stream_initialised=0; + + if (method!=NULL) + *method = (int)s->cur_file_info.compression_method; + + if (level!=NULL) + { + *level = 6; + switch (s->cur_file_info.flag & 0x06) + { + case 6 : *level = 1; break; + case 4 : *level = 2; break; + case 2 : *level = 9; break; + } + } + + if ((s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + + err=UNZ_BADZIPFILE; + + pfile_in_zip_read_info->crc32_wait=s->cur_file_info.crc; + pfile_in_zip_read_info->crc32=0; + pfile_in_zip_read_info->total_out_64=0; + pfile_in_zip_read_info->compression_method = s->cur_file_info.compression_method; + pfile_in_zip_read_info->filestream=s->filestream; + pfile_in_zip_read_info->z_filefunc=s->z_filefunc; + pfile_in_zip_read_info->byte_before_the_zipfile=s->byte_before_the_zipfile; + + pfile_in_zip_read_info->stream.total_out = 0; + + if ((s->cur_file_info.compression_method==Z_BZIP2ED) && (!raw)) + { +#ifdef HAVE_BZIP2 + pfile_in_zip_read_info->bstream.bzalloc = (void *(*) (void *, int, int))0; + pfile_in_zip_read_info->bstream.bzfree = (free_func)0; + pfile_in_zip_read_info->bstream.opaque = (voidpf)0; + pfile_in_zip_read_info->bstream.state = (voidpf)0; + + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = (voidpf)0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=BZ2_bzDecompressInit(&pfile_in_zip_read_info->bstream, 0, 0); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } +#else + pfile_in_zip_read_info->raw=1; +#endif + } + else if ((s->cur_file_info.compression_method==Z_DEFLATED) && (!raw)) + { + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = 0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=inflateInit2(&pfile_in_zip_read_info->stream, -MAX_WBITS); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_DEFLATED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } + /* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. + * In unzip, i don't wait absolutely Z_STREAM_END because I known the + * size of both compressed and uncompressed data + */ + } + pfile_in_zip_read_info->rest_read_compressed = + s->cur_file_info.compressed_size ; + pfile_in_zip_read_info->rest_read_uncompressed = + s->cur_file_info.uncompressed_size ; + + + pfile_in_zip_read_info->pos_in_zipfile = + s->cur_file_info_internal.offset_curfile + SIZEZIPLOCALHEADER + + iSizeVar; + + pfile_in_zip_read_info->stream.avail_in = (uInt)0; + + s->pfile_in_zip_read = pfile_in_zip_read_info; + s->encrypted = 0; + +# ifndef NOUNCRYPT + if (password != NULL) + { + int i; + s->pcrc_32_tab = get_crc_table(); + init_keys(password,s->keys,s->pcrc_32_tab); + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pfile_in_zip_read->pos_in_zipfile + + s->pfile_in_zip_read->byte_before_the_zipfile, + SEEK_SET)!=0) + return UNZ_INTERNALERROR; + if(ZREAD64(s->z_filefunc, s->filestream,source, 12)<12) + return UNZ_INTERNALERROR; + + for (i = 0; i<12; i++) + zdecode(s->keys,s->pcrc_32_tab,source[i]); + + s->pfile_in_zip_read->pos_in_zipfile+=12; + s->encrypted=1; + } +# endif + + + return UNZ_OK; +} + +extern int ZEXPORT unzOpenCurrentFile (unzFile file) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL); +} + +extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char* password) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, password); +} + +extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw) +{ + return unzOpenCurrentFile3(file, method, level, raw, NULL); +} + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + s=(unz64_s*)file; + if (file==NULL) + return 0; //UNZ_PARAMERROR; + pfile_in_zip_read_info=s->pfile_in_zip_read; + if (pfile_in_zip_read_info==NULL) + return 0; //UNZ_PARAMERROR; + return pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile; +} + +/** Addition for GDAL : END */ + +/* + Read bytes from the current file. + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ +extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) +{ + int err=UNZ_OK; + uInt iRead = 0; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if (pfile_in_zip_read_info->read_buffer == NULL) + return UNZ_END_OF_LIST_OF_FILE; + if (len==0) + return 0; + + pfile_in_zip_read_info->stream.next_out = (Bytef*)buf; + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + + if ((len>pfile_in_zip_read_info->rest_read_uncompressed) && + (!(pfile_in_zip_read_info->raw))) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_uncompressed; + + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in; + + while (pfile_in_zip_read_info->stream.avail_out>0) + { + if ((pfile_in_zip_read_info->stream.avail_in==0) && + (pfile_in_zip_read_info->rest_read_compressed>0)) + { + uInt uReadThis = UNZ_BUFSIZE; + if (pfile_in_zip_read_info->rest_read_compressedrest_read_compressed; + if (uReadThis == 0) + return UNZ_EOF; + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->read_buffer, + uReadThis)!=uReadThis) + return UNZ_ERRNO; + + +# ifndef NOUNCRYPT + if(s->encrypted) + { + uInt i; + for(i=0;iread_buffer[i] = + zdecode(s->keys,s->pcrc_32_tab, + pfile_in_zip_read_info->read_buffer[i]); + } +# endif + + + pfile_in_zip_read_info->pos_in_zipfile += uReadThis; + + pfile_in_zip_read_info->rest_read_compressed-=uReadThis; + + pfile_in_zip_read_info->stream.next_in = + (Bytef*)pfile_in_zip_read_info->read_buffer; + pfile_in_zip_read_info->stream.avail_in = (uInt)uReadThis; + } + + if ((pfile_in_zip_read_info->compression_method==0) || (pfile_in_zip_read_info->raw)) + { + uInt uDoCopy,i ; + + if ((pfile_in_zip_read_info->stream.avail_in == 0) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + return (iRead==0) ? UNZ_EOF : iRead; + + if (pfile_in_zip_read_info->stream.avail_out < + pfile_in_zip_read_info->stream.avail_in) + uDoCopy = pfile_in_zip_read_info->stream.avail_out ; + else + uDoCopy = pfile_in_zip_read_info->stream.avail_in ; + + for (i=0;istream.next_out+i) = + *(pfile_in_zip_read_info->stream.next_in+i); + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uDoCopy; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32, + pfile_in_zip_read_info->stream.next_out, + uDoCopy); + pfile_in_zip_read_info->rest_read_uncompressed-=uDoCopy; + pfile_in_zip_read_info->stream.avail_in -= uDoCopy; + pfile_in_zip_read_info->stream.avail_out -= uDoCopy; + pfile_in_zip_read_info->stream.next_out += uDoCopy; + pfile_in_zip_read_info->stream.next_in += uDoCopy; + pfile_in_zip_read_info->stream.total_out += uDoCopy; + iRead += uDoCopy; + } + else if (pfile_in_zip_read_info->compression_method==Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + uLong uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + uLong uOutThis; + + pfile_in_zip_read_info->bstream.next_in = (char*)pfile_in_zip_read_info->stream.next_in; + pfile_in_zip_read_info->bstream.avail_in = pfile_in_zip_read_info->stream.avail_in; + pfile_in_zip_read_info->bstream.total_in_lo32 = pfile_in_zip_read_info->stream.total_in; + pfile_in_zip_read_info->bstream.total_in_hi32 = 0; + pfile_in_zip_read_info->bstream.next_out = (char*)pfile_in_zip_read_info->stream.next_out; + pfile_in_zip_read_info->bstream.avail_out = pfile_in_zip_read_info->stream.avail_out; + pfile_in_zip_read_info->bstream.total_out_lo32 = pfile_in_zip_read_info->stream.total_out; + pfile_in_zip_read_info->bstream.total_out_hi32 = 0; + + uTotalOutBefore = pfile_in_zip_read_info->bstream.total_out_lo32; + bufBefore = (const Bytef *)pfile_in_zip_read_info->bstream.next_out; + + err=BZ2_bzDecompress(&pfile_in_zip_read_info->bstream); + + uTotalOutAfter = pfile_in_zip_read_info->bstream.total_out_lo32; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32,bufBefore, (uInt)(uOutThis)); + pfile_in_zip_read_info->rest_read_uncompressed -= uOutThis; + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + pfile_in_zip_read_info->stream.next_in = (Bytef*)pfile_in_zip_read_info->bstream.next_in; + pfile_in_zip_read_info->stream.avail_in = pfile_in_zip_read_info->bstream.avail_in; + pfile_in_zip_read_info->stream.total_in = pfile_in_zip_read_info->bstream.total_in_lo32; + pfile_in_zip_read_info->stream.next_out = (Bytef*)pfile_in_zip_read_info->bstream.next_out; + pfile_in_zip_read_info->stream.avail_out = pfile_in_zip_read_info->bstream.avail_out; + pfile_in_zip_read_info->stream.total_out = pfile_in_zip_read_info->bstream.total_out_lo32; + + if (err==BZ_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=BZ_OK) + break; +#endif + } // end Z_BZIP2ED + else + { + ZPOS64_T uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + ZPOS64_T uOutThis; + int flush=Z_SYNC_FLUSH; + + uTotalOutBefore = pfile_in_zip_read_info->stream.total_out; + bufBefore = pfile_in_zip_read_info->stream.next_out; + + /* + if ((pfile_in_zip_read_info->rest_read_uncompressed == + pfile_in_zip_read_info->stream.avail_out) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + flush = Z_FINISH; + */ + err=inflate(&pfile_in_zip_read_info->stream,flush); + + if ((err>=0) && (pfile_in_zip_read_info->stream.msg!=NULL)) + err = Z_DATA_ERROR; + + uTotalOutAfter = pfile_in_zip_read_info->stream.total_out; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = + crc32(pfile_in_zip_read_info->crc32,bufBefore, + (uInt)(uOutThis)); + + pfile_in_zip_read_info->rest_read_uncompressed -= + uOutThis; + + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + if (err==Z_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=Z_OK) + break; + } + } + + if (err==Z_OK) + return iRead; + return err; +} + + +/* + Give the current position in uncompressed data +*/ +extern z_off_t ZEXPORT unztell (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + return (z_off_t)pfile_in_zip_read_info->stream.total_out; +} + +extern ZPOS64_T ZEXPORT unztell64 (unzFile file) +{ + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return (ZPOS64_T)-1; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return (ZPOS64_T)-1; + + return pfile_in_zip_read_info->total_out_64; +} + + +/* + return 1 if the end of file was reached, 0 elsewhere +*/ +extern int ZEXPORT unzeof (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + if (pfile_in_zip_read_info->rest_read_uncompressed == 0) + return 1; + else + return 0; +} + + + +/* +Read extra field from the current file (opened by unzOpenCurrentFile) +This is the local-header version of the extra field (sometimes, there is +more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field that can be read + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ +extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + uInt read_now; + ZPOS64_T size_to_read; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + size_to_read = (pfile_in_zip_read_info->size_local_extrafield - + pfile_in_zip_read_info->pos_local_extrafield); + + if (buf==NULL) + return (int)size_to_read; + + if (len>size_to_read) + read_now = (uInt)size_to_read; + else + read_now = (uInt)len ; + + if (read_now==0) + return 0; + + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->offset_local_extrafield + + pfile_in_zip_read_info->pos_local_extrafield, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + buf,read_now)!=read_now) + return UNZ_ERRNO; + + return (int)read_now; +} + +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ +extern int ZEXPORT unzCloseCurrentFile (unzFile file) +{ + int err=UNZ_OK; + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if ((pfile_in_zip_read_info->rest_read_uncompressed == 0) && + (!pfile_in_zip_read_info->raw)) + { + if (pfile_in_zip_read_info->crc32 != pfile_in_zip_read_info->crc32_wait) + err=UNZ_CRCERROR; + } + + + TRYFREE(pfile_in_zip_read_info->read_buffer); + pfile_in_zip_read_info->read_buffer = NULL; + if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED) + inflateEnd(&pfile_in_zip_read_info->stream); +#ifdef HAVE_BZIP2 + else if (pfile_in_zip_read_info->stream_initialised == Z_BZIP2ED) + BZ2_bzDecompressEnd(&pfile_in_zip_read_info->bstream); +#endif + + + pfile_in_zip_read_info->stream_initialised = 0; + TRYFREE(pfile_in_zip_read_info); + + s->pfile_in_zip_read=NULL; + + return err; +} + + +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ +extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf) +{ + unz64_s* s; + uLong uReadThis ; + if (file==NULL) + return (int)UNZ_PARAMERROR; + s=(unz64_s*)file; + + uReadThis = uSizeBuf; + if (uReadThis>s->gi.size_comment) + uReadThis = s->gi.size_comment; + + if (ZSEEK64(s->z_filefunc,s->filestream,s->central_pos+22,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (uReadThis>0) + { + *szComment='\0'; + if (ZREAD64(s->z_filefunc,s->filestream,szComment,uReadThis)!=uReadThis) + return UNZ_ERRNO; + } + + if ((szComment != NULL) && (uSizeBuf > s->gi.size_comment)) + *(szComment+s->gi.size_comment)='\0'; + return (int)uReadThis; +} + +/* Additions by RX '2004 */ +extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) +{ + unz64_s* s; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return 0; + if (s->gi.number_entry != 0 && s->gi.number_entry != 0xffff) + if (s->num_file==s->gi.number_entry) + return 0; + return s->pos_in_central_dir; +} + +extern uLong ZEXPORT unzGetOffset (unzFile file) +{ + ZPOS64_T offset64; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + offset64 = unzGetOffset64(file); + return (uLong)offset64; +} + +extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + s->pos_in_central_dir = pos; + s->num_file = s->gi.number_entry; /* hack */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) +{ + return unzSetOffset64(file,pos); +} diff --git a/storage/connect/unzip.h b/storage/connect/unzip.h new file mode 100644 index 00000000000..2104e391507 --- /dev/null +++ b/storage/connect/unzip.h @@ -0,0 +1,437 @@ +/* unzip.h -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------------- + + Changes + + See header of unzip64.c + +*/ + +#ifndef _unz64_H +#define _unz64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagunzFile__ { int unused; } unzFile__; +typedef unzFile__ *unzFile; +#else +typedef voidp unzFile; +#endif + + +#define UNZ_OK (0) +#define UNZ_END_OF_LIST_OF_FILE (-100) +#define UNZ_ERRNO (Z_ERRNO) +#define UNZ_EOF (0) +#define UNZ_PARAMERROR (-102) +#define UNZ_BADZIPFILE (-103) +#define UNZ_INTERNALERROR (-104) +#define UNZ_CRCERROR (-105) + +/* tm_unz contain date/time info */ +typedef struct tm_unz_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_unz; + +/* unz_global_info structure contain global data about the ZIPfile + These data comes from the end of central dir */ +typedef struct unz_global_info64_s +{ + ZPOS64_T number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info64; + +typedef struct unz_global_info_s +{ + uLong number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info; + +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_info64_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + ZPOS64_T compressed_size; /* compressed size 8 bytes */ + ZPOS64_T uncompressed_size; /* uncompressed size 8 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info64; + +typedef struct unz_file_info_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + uLong compressed_size; /* compressed size 4 bytes */ + uLong uncompressed_size; /* uncompressed size 4 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info; + +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, + const char* fileName2, + int iCaseSensitivity)); +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) +*/ + + +extern unzFile ZEXPORT unzOpen OF((const char *path)); +extern unzFile ZEXPORT unzOpen64 OF((const void *path)); +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer + "zlib/zlib113.zip". + If the zipfile cannot be opened (file don't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. + the "64" function take a const void* pointer, because the path is just the + value passed to the open64_file_func callback. + Under Windows, if UNICODE is defined, using fill_fopen64_filefunc, the path + is a pointer to a wide unicode string (LPCTSTR is LPCWSTR), so const char* + does not describe the reality +*/ + + +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unzOpen, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, + zlib_filefunc64_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unz64Open, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern int ZEXPORT unzClose OF((unzFile file)); +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ + +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); + +extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file, + unz_global_info64 *pglobal_info)); +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ + + +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, + char *szComment, + uLong uSizeBuf)); +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ + + +/***************************************************************************/ +/* Unzip package allow you browse the directory of the zipfile */ + +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ + +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ + +extern int ZEXPORT unzLocateFile OF((unzFile file, + const char *szFileName, + int iCaseSensitivity)); +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ + + +/* ****************************************** */ +/* Ryan supplied functions */ +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_pos_s +{ + uLong pos_in_zip_directory; /* offset in zip file directory */ + uLong num_of_file; /* # of file */ +} unz_file_pos; + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos); + +typedef struct unz64_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; /* offset in zip file directory */ + ZPOS64_T num_of_file; /* # of file */ +} unz64_file_pos; + +extern int ZEXPORT unzGetFilePos64( + unzFile file, + unz64_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos64( + unzFile file, + const unz64_file_pos* file_pos); + +/* ****************************************** */ + +extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file, + unz_file_info64 *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, + unz_file_info *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); +/* + Get Info about the current file + if pfile_info!=NULL, the *pfile_info structure will contain somes info about + the current file + if szFileName!=NULL, the filemane string will be copied in szFileName + (fileNameBufferSize is the size of the buffer) + if extraField!=NULL, the extra field information will be copied in extraField + (extraFieldBufferSize is the size of the buffer). + This is the Central-header version of the extra field + if szComment!=NULL, the comment string of the file will be copied in szComment + (commentBufferSize is the size of the buffer) +*/ + + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file)); + +/** Addition for GDAL : END */ + + +/***************************************************************************/ +/* for reading the content of the current zipfile, you can open it, read data + from it, and close it (you can close it before reading all the file) + */ + +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); +/* + Open for reading data the current file in the zipfile. + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); +/* + Open for reading data the current file in the zipfile. + password is a crypting password + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, + int* method, + int* level, + int raw)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, + int* method, + int* level, + int raw, + const char* password)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + + +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ + +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read bytes from the current file (opened by unzOpenCurrentFile) + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ + +extern z_off_t ZEXPORT unztell OF((unzFile file)); + +extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file)); +/* + Give the current position in uncompressed data +*/ + +extern int ZEXPORT unzeof OF((unzFile file)); +/* + return 1 if the end of file was reached, 0 elsewhere +*/ + +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read extra field from the current file (opened by unzOpenCurrentFile) + This is the local-header version of the extra field (sometimes, there is + more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ + +/***************************************************************************/ + +/* Get the current file offset */ +extern ZPOS64_T ZEXPORT unzGetOffset64 (unzFile file); +extern uLong ZEXPORT unzGetOffset (unzFile file); + +/* Set the current file offset */ +extern int ZEXPORT unzSetOffset64 (unzFile file, ZPOS64_T pos); +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos); + + + +#ifdef __cplusplus +} +#endif + +#endif /* _unz64_H */ diff --git a/storage/connect/zip.c b/storage/connect/zip.c new file mode 100644 index 00000000000..ea54853e858 --- /dev/null +++ b/storage/connect/zip.c @@ -0,0 +1,2007 @@ +/* zip.c -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + Oct-2009 - Mathias Svensson - Remove old C style function prototypes + Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives + Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. + Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data + It is used when recreting zip archive with RAW when deleting items from a zip. + ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + +*/ + + +#include +#include +#include +#include +#include "zlib.h" +#include "zip.h" + +#ifdef STDC +# include +# include +# include +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +#ifndef VERSIONMADEBY +# define VERSIONMADEBY (0x0) /* platform depedent */ +#endif + +#ifndef Z_BUFSIZE +#define Z_BUFSIZE (64*1024) //(16384) +#endif + +#ifndef Z_MAXFILENAMEINZIP +#define Z_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +/* +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) +*/ + +/* I've found an old Unix (a SunOS 4.1.3_U1) without all SEEK_* defined.... */ + + +// NOT sure that this work on ALL platform +#define MAKEULONG64(a, b) ((ZPOS64_T)(((unsigned long)(a)) | ((ZPOS64_T)((unsigned long)(b))) << 32)) + +#ifndef SEEK_CUR +#define SEEK_CUR 1 +#endif + +#ifndef SEEK_END +#define SEEK_END 2 +#endif + +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif + +#ifndef DEF_MEM_LEVEL +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +#endif +const char zip_copyright[] =" zip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + + +#define SIZEDATA_INDATABLOCK (4096-(4*4)) + +#define LOCALHEADERMAGIC (0x04034b50) +#define CENTRALHEADERMAGIC (0x02014b50) +#define ENDHEADERMAGIC (0x06054b50) +#define ZIP64ENDHEADERMAGIC (0x6064b50) +#define ZIP64ENDLOCHEADERMAGIC (0x7064b50) + +#define FLAG_LOCALHEADER_OFFSET (0x06) +#define CRC_LOCALHEADER_OFFSET (0x0e) + +#define SIZECENTRALHEADER (0x2e) /* 46 */ + +typedef struct linkedlist_datablock_internal_s +{ + struct linkedlist_datablock_internal_s* next_datablock; + uLong avail_in_this_block; + uLong filled_in_this_block; + uLong unused; /* for future use and alignement */ + unsigned char data[SIZEDATA_INDATABLOCK]; +} linkedlist_datablock_internal; + +typedef struct linkedlist_data_s +{ + linkedlist_datablock_internal* first_block; + linkedlist_datablock_internal* last_block; +} linkedlist_data; + + +typedef struct +{ + z_stream stream; /* zLib stream structure for inflate */ +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + int stream_initialised; /* 1 is stream is initialised */ + uInt pos_in_buffered_data; /* last written byte in buffered_data */ + + ZPOS64_T pos_local_header; /* offset of the local header of the file + currenty writing */ + char* central_header; /* central header data for the current file */ + uLong size_centralExtra; + uLong size_centralheader; /* size of the central header for cur file */ + uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ + uLong flag; /* flag of the file currently writing */ + + int method; /* compression method of file currenty wr.*/ + int raw; /* 1 for directly writing raw data */ + Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ + uLong dosDate; + uLong crc32; + int encrypt; + int zip64; /* Add ZIP64 extened information in the extra field */ + ZPOS64_T pos_zip64extrainfo; + ZPOS64_T totalCompressedData; + ZPOS64_T totalUncompressedData; +#ifndef NOCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; + int crypt_header_size; +#endif +} curfile64_info; + +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + linkedlist_data central_dir;/* datablock with central dir in construction*/ + int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ + curfile64_info ci; /* info on the file curretly writing */ + + ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ + ZPOS64_T add_position_when_writting_offset; + ZPOS64_T number_entry; + +#ifndef NO_ADDFILEINEXISTINGZIP + char *globalcomment; +#endif + +} zip64_internal; + + +#ifndef NOCRYPT +#define INCLUDECRYPTINGCODE_IFCRYPTALLOWED +#include "crypt.h" +#endif + +local linkedlist_datablock_internal* allocate_new_datablock() +{ + linkedlist_datablock_internal* ldi; + ldi = (linkedlist_datablock_internal*) + ALLOC(sizeof(linkedlist_datablock_internal)); + if (ldi!=NULL) + { + ldi->next_datablock = NULL ; + ldi->filled_in_this_block = 0 ; + ldi->avail_in_this_block = SIZEDATA_INDATABLOCK ; + } + return ldi; +} + +local void free_datablock(linkedlist_datablock_internal* ldi) +{ + while (ldi!=NULL) + { + linkedlist_datablock_internal* ldinext = ldi->next_datablock; + TRYFREE(ldi); + ldi = ldinext; + } +} + +local void init_linkedlist(linkedlist_data* ll) +{ + ll->first_block = ll->last_block = NULL; +} + +local void free_linkedlist(linkedlist_data* ll) +{ + free_datablock(ll->first_block); + ll->first_block = ll->last_block = NULL; +} + + +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) +{ + linkedlist_datablock_internal* ldi; + const unsigned char* from_copy; + + if (ll==NULL) + return ZIP_INTERNALERROR; + + if (ll->last_block == NULL) + { + ll->first_block = ll->last_block = allocate_new_datablock(); + if (ll->first_block == NULL) + return ZIP_INTERNALERROR; + } + + ldi = ll->last_block; + from_copy = (unsigned char*)buf; + + while (len>0) + { + uInt copy_this; + uInt i; + unsigned char* to_copy; + + if (ldi->avail_in_this_block==0) + { + ldi->next_datablock = allocate_new_datablock(); + if (ldi->next_datablock == NULL) + return ZIP_INTERNALERROR; + ldi = ldi->next_datablock ; + ll->last_block = ldi; + } + + if (ldi->avail_in_this_block < len) + copy_this = (uInt)ldi->avail_in_this_block; + else + copy_this = (uInt)len; + + to_copy = &(ldi->data[ldi->filled_in_this_block]); + + for (i=0;ifilled_in_this_block += copy_this; + ldi->avail_in_this_block -= copy_this; + from_copy += copy_this ; + len -= copy_this; + } + return ZIP_OK; +} + + + +/****************************************************************************/ + +#ifndef NO_ADDFILEINEXISTINGZIP +/* =========================================================================== + Inputs a long in LSB order to the given file + nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) +*/ + +local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); +local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) +{ + unsigned char buf[8]; + int n; + for (n = 0; n < nbByte; n++) + { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + if (x != 0) + { /* data overflow - hack for ZIP64 (X Roche) */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } + + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + return ZIP_ERRNO; + else + return ZIP_OK; +} + +local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) +{ + unsigned char* buf=(unsigned char*)dest; + int n; + for (n = 0; n < nbByte; n++) { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + + if (x != 0) + { /* data overflow - hack for ZIP64 */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } +} + +/****************************************************************************/ + + +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) +{ + uLong year = (uLong)ptm->tm_year; + if (year>=1980) + year-=1980; + else if (year>=80) + year-=80; + return + (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | + ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); +} + + +/****************************************************************************/ + +local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); + +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return ZIP_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return ZIP_ERRNO; + else + return ZIP_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); + + +local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) +{ + ZPOS64_T x; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<24; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<32; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<40; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<48; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<56; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + + return err; +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackReaduMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + +/* +Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before +the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackReaduMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + { + // Signature "0x07064b50" Zip64 end of central directory locater + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + } + + if (uPosFound!=0) + break; + } + + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (zip64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=ZIP_OK) + return 0; + + /* total number of disks */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto Zip64 end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + if (uL != 0x06064b50) // signature of 'Zip64 end of central directory' + return 0; + + return relativeOffset; +} + +int LoadCentralDirectoryRecord(zip64_internal* pziinit) +{ + int err=ZIP_OK; + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory */ + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry; + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + uLong VersionMadeBy; + uLong VersionNeeded; + uLong size_comment; + + int hasZIP64Record = 0; + + // check first if we find a ZIP64 record + central_pos = zip64local_SearchCentralDir64(&pziinit->z_filefunc,pziinit->filestream); + if(central_pos > 0) + { + hasZIP64Record = 1; + } + else if(central_pos == 0) + { + central_pos = zip64local_SearchCentralDir(&pziinit->z_filefunc,pziinit->filestream); + } + +/* disable to allow appending to empty ZIP archive + if (central_pos==0) + err=ZIP_ERRNO; +*/ + + if(hasZIP64Record) + { + ZPOS64_T sizeEndOfCentralDirectory; + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* size of zip64 end of central directory record */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &sizeEndOfCentralDirectory)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version made by */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionMadeBy)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version needed to extract */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionNeeded)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &number_entry)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&number_entry_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&size_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&offset_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + // TODO.. + // read the comment from the standard central header. + size_comment = 0; + } + else + { + // Read End of central Directory info + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central dir on this disk */ + number_entry = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry = uL; + + /* total number of entries in the central dir */ + number_entry_CD = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry_CD = uL; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + size_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + size_central_dir = uL; + + /* offset of start of central directory with respect to the starting disk number */ + offset_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + offset_central_dir = uL; + + + /* zipfile global comment length */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &size_comment)!=ZIP_OK) + err=ZIP_ERRNO; + } + + if ((central_posz_filefunc, pziinit->filestream); + return ZIP_ERRNO; + } + + if (size_comment>0) + { + pziinit->globalcomment = (char*)ALLOC(size_comment+1); + if (pziinit->globalcomment) + { + size_comment = ZREAD64(pziinit->z_filefunc, pziinit->filestream, pziinit->globalcomment,size_comment); + pziinit->globalcomment[size_comment]=0; + } + } + + byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); + pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + + { + ZPOS64_T size_central_dir_to_read = size_central_dir; + size_t buf_size = SIZEDATA_INDATABLOCK; + void* buf_read = (void*)ALLOC(buf_size); + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir + byte_before_the_zipfile, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + while ((size_central_dir_to_read>0) && (err==ZIP_OK)) + { + ZPOS64_T read_this = SIZEDATA_INDATABLOCK; + if (read_this > size_central_dir_to_read) + read_this = size_central_dir_to_read; + + if (ZREAD64(pziinit->z_filefunc, pziinit->filestream,buf_read,(uLong)read_this) != read_this) + err=ZIP_ERRNO; + + if (err==ZIP_OK) + err = add_data_in_datablock(&pziinit->central_dir,buf_read, (uLong)read_this); + + size_central_dir_to_read-=read_this; + } + TRYFREE(buf_read); + } + pziinit->begin_pos = byte_before_the_zipfile; + pziinit->number_entry = number_entry_CD; + + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir+byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + return err; +} + + +#endif /* !NO_ADDFILEINEXISTINGZIP*/ + + +/************************************************************/ +extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +{ + zip64_internal ziinit; + zip64_internal* zi; + int err=ZIP_OK; + + ziinit.z_filefunc.zseek32_file = NULL; + ziinit.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + else + ziinit.z_filefunc = *pzlib_filefunc64_32_def; + + ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + pathname, + (append == APPEND_STATUS_CREATE) ? + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); + + if (ziinit.filestream == NULL) + return NULL; + + if (append == APPEND_STATUS_CREATEAFTER) + ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + + ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); + ziinit.in_opened_file_inzip = 0; + ziinit.ci.stream_initialised = 0; + ziinit.number_entry = 0; + ziinit.add_position_when_writting_offset = 0; + init_linkedlist(&(ziinit.central_dir)); + + + + zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); + if (zi==NULL) + { + ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); + return NULL; + } + + /* now we add file in a zipfile */ +# ifndef NO_ADDFILEINEXISTINGZIP + ziinit.globalcomment = NULL; + if (append == APPEND_STATUS_ADDINZIP) + { + // Read and Cache Central Directory Records + err = LoadCentralDirectoryRecord(&ziinit); + } + + if (globalcomment) + { + *globalcomment = ziinit.globalcomment; + } +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + + if (err != ZIP_OK) + { +# ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(ziinit.globalcomment); +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + TRYFREE(zi); + return NULL; + } + else + { + *zi = ziinit; + return (zipFile)zi; + } +} + +extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + +extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + + + +extern zipFile ZEXPORT zipOpen (const char* pathname, int append) +{ + return zipOpen3((const void*)pathname,append,NULL,NULL); +} + +extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) +{ + return zipOpen3(pathname,append,NULL,NULL); +} + +int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +{ + /* write the local header */ + int err; + uInt size_filename = (uInt)strlen(filename); + uInt size_extrafield = size_extrafield_local; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)LOCALHEADERMAGIC, 4); + + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2);/* version needed to extract */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)20,2);/* version needed to extract */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.flag,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.method,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.dosDate,4); + + // CRC / Compressed size / Uncompressed size will be filled in later and rewritten later + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* crc 32, unknown */ + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* compressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* compressed size, unknown */ + } + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* uncompressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* uncompressed size, unknown */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_filename,2); + + if(zi->ci.zip64) + { + size_extrafield += 20; + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_extrafield,2); + + if ((err==ZIP_OK) && (size_filename > 0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream,filename,size_filename)!=size_filename) + err = ZIP_ERRNO; + } + + if ((err==ZIP_OK) && (size_extrafield_local > 0)) + { + if (ZWRITE64(zi->z_filefunc, zi->filestream, extrafield_local, size_extrafield_local) != size_extrafield_local) + err = ZIP_ERRNO; + } + + + if ((err==ZIP_OK) && (zi->ci.zip64)) + { + // write the Zip64 extended info + short HeaderID = 1; + short DataSize = 16; + ZPOS64_T CompressedSize = 0; + ZPOS64_T UncompressedSize = 0; + + // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) + zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); + } + + return err; +} + +/* + NOTE. + When writing RAW the ZIP64 extended information in extrafield_local and extrafield_global needs to be stripped + before calling this function it can be done with zipRemoveExtraInfoBlock + + It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize + unnecessary allocations. + */ +extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) +{ + zip64_internal* zi; + uInt size_filename; + uInt size_comment; + uInt i; + int err = ZIP_OK; + +# ifdef NOCRYPT + (crcForCrypting); + if (password != NULL) + return ZIP_PARAMERROR; +# endif + + if (file == NULL) + return ZIP_PARAMERROR; + +#ifdef HAVE_BZIP2 + if ((method!=0) && (method!=Z_DEFLATED) && (method!=Z_BZIP2ED)) + return ZIP_PARAMERROR; +#else + if ((method!=0) && (method!=Z_DEFLATED)) + return ZIP_PARAMERROR; +#endif + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + if (err != ZIP_OK) + return err; + } + + if (filename==NULL) + filename="-"; + + if (comment==NULL) + size_comment = 0; + else + size_comment = (uInt)strlen(comment); + + size_filename = (uInt)strlen(filename); + + if (zipfi == NULL) + zi->ci.dosDate = 0; + else + { + if (zipfi->dosDate != 0) + zi->ci.dosDate = zipfi->dosDate; + else + zi->ci.dosDate = zip64local_TmzDateToDosDate(&zipfi->tmz_date); + } + + zi->ci.flag = flagBase; + if ((level==8) || (level==9)) + zi->ci.flag |= 2; + if (level==2) + zi->ci.flag |= 4; + if (level==1) + zi->ci.flag |= 6; + if (password != NULL) + zi->ci.flag |= 1; + + zi->ci.crc32 = 0; + zi->ci.method = method; + zi->ci.encrypt = 0; + zi->ci.stream_initialised = 0; + zi->ci.pos_in_buffered_data = 0; + zi->ci.raw = raw; + zi->ci.pos_local_header = ZTELL64(zi->z_filefunc,zi->filestream); + + zi->ci.size_centralheader = SIZECENTRALHEADER + size_filename + size_extrafield_global + size_comment; + zi->ci.size_centralExtraFree = 32; // Extra space we have reserved in case we need to add ZIP64 extra info data + + zi->ci.central_header = (char*)ALLOC((uInt)zi->ci.size_centralheader + zi->ci.size_centralExtraFree); + + zi->ci.size_centralExtra = size_extrafield_global; + zip64local_putValue_inmemory(zi->ci.central_header,(uLong)CENTRALHEADERMAGIC,4); + /* version info */ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)versionMadeBy,2); + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)20,2); + zip64local_putValue_inmemory(zi->ci.central_header+8,(uLong)zi->ci.flag,2); + zip64local_putValue_inmemory(zi->ci.central_header+10,(uLong)zi->ci.method,2); + zip64local_putValue_inmemory(zi->ci.central_header+12,(uLong)zi->ci.dosDate,4); + zip64local_putValue_inmemory(zi->ci.central_header+16,(uLong)0,4); /*crc*/ + zip64local_putValue_inmemory(zi->ci.central_header+20,(uLong)0,4); /*compr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+24,(uLong)0,4); /*uncompr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+28,(uLong)size_filename,2); + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)size_extrafield_global,2); + zip64local_putValue_inmemory(zi->ci.central_header+32,(uLong)size_comment,2); + zip64local_putValue_inmemory(zi->ci.central_header+34,(uLong)0,2); /*disk nm start*/ + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)0,2); + else + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)zipfi->internal_fa,2); + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)0,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)zipfi->external_fa,4); + + if(zi->ci.pos_local_header >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + + for (i=0;ici.central_header+SIZECENTRALHEADER+i) = *(filename+i); + + for (i=0;ici.central_header+SIZECENTRALHEADER+size_filename+i) = + *(((const char*)extrafield_global)+i); + + for (i=0;ici.central_header+SIZECENTRALHEADER+size_filename+ + size_extrafield_global+i) = *(comment+i); + if (zi->ci.central_header == NULL) + return ZIP_INTERNALERROR; + + zi->ci.zip64 = zip64; + zi->ci.totalCompressedData = 0; + zi->ci.totalUncompressedData = 0; + zi->ci.pos_zip64extrainfo = 0; + + err = Write_LocalFileHeader(zi, filename, size_extrafield_local, extrafield_local); + +#ifdef HAVE_BZIP2 + zi->ci.bstream.avail_in = (uInt)0; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + zi->ci.bstream.total_in_hi32 = 0; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_out_hi32 = 0; + zi->ci.bstream.total_out_lo32 = 0; +#endif + + zi->ci.stream.avail_in = (uInt)0; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + zi->ci.stream.total_in = 0; + zi->ci.stream.total_out = 0; + zi->ci.stream.data_type = Z_BINARY; + +#ifdef HAVE_BZIP2 + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED || zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) +#else + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) +#endif + { + if(zi->ci.method == Z_DEFLATED) + { + zi->ci.stream.zalloc = (alloc_func)0; + zi->ci.stream.zfree = (free_func)0; + zi->ci.stream.opaque = (voidpf)0; + + if (windowBits>0) + windowBits = -windowBits; + + err = deflateInit2(&zi->ci.stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + + if (err==Z_OK) + zi->ci.stream_initialised = Z_DEFLATED; + } + else if(zi->ci.method == Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + // Init BZip stuff here + zi->ci.bstream.bzalloc = 0; + zi->ci.bstream.bzfree = 0; + zi->ci.bstream.opaque = (voidpf)0; + + err = BZ2_bzCompressInit(&zi->ci.bstream, level, 0,35); + if(err == BZ_OK) + zi->ci.stream_initialised = Z_BZIP2ED; +#endif + } + + } + +# ifndef NOCRYPT + zi->ci.crypt_header_size = 0; + if ((err==Z_OK) && (password != NULL)) + { + unsigned char bufHead[RAND_HEAD_LEN]; + unsigned int sizeHead; + zi->ci.encrypt = 1; + zi->ci.pcrc_32_tab = get_crc_table(); + /*init_keys(password,zi->ci.keys,zi->ci.pcrc_32_tab);*/ + + sizeHead=crypthead(password,bufHead,RAND_HEAD_LEN,zi->ci.keys,zi->ci.pcrc_32_tab,crcForCrypting); + zi->ci.crypt_header_size = sizeHead; + + if (ZWRITE64(zi->z_filefunc,zi->filestream,bufHead,sizeHead) != sizeHead) + err = ZIP_ERRNO; + } +# endif + + if (err==Z_OK) + zi->in_opened_file_inzip = 1; + return err; +} + +extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +local int zip64FlushWriteBuffer(zip64_internal* zi) +{ + int err=ZIP_OK; + + if (zi->ci.encrypt != 0) + { +#ifndef NOCRYPT + uInt i; + int t; + for (i=0;ici.pos_in_buffered_data;i++) + zi->ci.buffered_data[i] = zencode(zi->ci.keys, zi->ci.pcrc_32_tab, zi->ci.buffered_data[i],t); +#endif + } + + if (ZWRITE64(zi->z_filefunc,zi->filestream,zi->ci.buffered_data,zi->ci.pos_in_buffered_data) != zi->ci.pos_in_buffered_data) + err = ZIP_ERRNO; + + zi->ci.totalCompressedData += zi->ci.pos_in_buffered_data; + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED) + { + zi->ci.totalUncompressedData += zi->ci.bstream.total_in_lo32; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_in_hi32 = 0; + } + else +#endif + { + zi->ci.totalUncompressedData += zi->ci.stream.total_in; + zi->ci.stream.total_in = 0; + } + + + zi->ci.pos_in_buffered_data = 0; + + return err; +} + +extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) +{ + zip64_internal* zi; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + + zi->ci.crc32 = crc32(zi->ci.crc32,buf,(uInt)len); + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED && (!zi->ci.raw)) + { + zi->ci.bstream.next_in = (void*)buf; + zi->ci.bstream.avail_in = len; + err = BZ_RUN_OK; + + while ((err==BZ_RUN_OK) && (zi->ci.bstream.avail_in>0)) + { + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + + + if(err != BZ_RUN_OK) + break; + + if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore_lo = zi->ci.bstream.total_out_lo32; +// uLong uTotalOutBefore_hi = zi->ci.bstream.total_out_hi32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_RUN); + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore_lo) ; + } + } + + if(err == BZ_RUN_OK) + err = ZIP_OK; + } + else +#endif + { + zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.avail_in = len; + + while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) + { + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + + + if(err != ZIP_OK) + break; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_NO_FLUSH); + if(uTotalOutBefore > zi->ci.stream.total_out) + { + int bBreak = 0; + bBreak++; + } + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + else + { + uInt copy_this,i; + if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) + copy_this = zi->ci.stream.avail_in; + else + copy_this = zi->ci.stream.avail_out; + + for (i = 0; i < copy_this; i++) + *(((char*)zi->ci.stream.next_out)+i) = + *(((const char*)zi->ci.stream.next_in)+i); + { + zi->ci.stream.avail_in -= copy_this; + zi->ci.stream.avail_out-= copy_this; + zi->ci.stream.next_in+= copy_this; + zi->ci.stream.next_out+= copy_this; + zi->ci.stream.total_in+= copy_this; + zi->ci.stream.total_out+= copy_this; + zi->ci.pos_in_buffered_data += copy_this; + } + } + }// while(...) + } + + return err; +} + +extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) +{ + return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); +} + +extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) +{ + zip64_internal* zi; + ZPOS64_T compressed_size; + uLong invalidValue = 0xffffffff; + short datasize = 0; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + zi->ci.stream.avail_in = 0; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + while (err==ZIP_OK) + { + uLong uTotalOutBefore; + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_FINISH); + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + } + else if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { +#ifdef HAVE_BZIP2 + err = BZ_FINISH_OK; + while (err==BZ_FINISH_OK) + { + uLong uTotalOutBefore; + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.bstream.total_out_lo32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_FINISH); + if(err == BZ_STREAM_END) + err = Z_STREAM_END; + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore); + } + + if(err == BZ_FINISH_OK) + err = ZIP_OK; +#endif + } + + if (err==Z_STREAM_END) + err=ZIP_OK; /* this is normal */ + + if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) + { + if (zip64FlushWriteBuffer(zi)==ZIP_ERRNO) + err = ZIP_ERRNO; + } + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + int tmp_err = deflateEnd(&zi->ci.stream); + if (err == ZIP_OK) + err = tmp_err; + zi->ci.stream_initialised = 0; + } +#ifdef HAVE_BZIP2 + else if((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + int tmperr = BZ2_bzCompressEnd(&zi->ci.bstream); + if (err==ZIP_OK) + err = tmperr; + zi->ci.stream_initialised = 0; + } +#endif + + if (!zi->ci.raw) + { + crc32 = (uLong)zi->ci.crc32; + uncompressed_size = zi->ci.totalUncompressedData; + } + compressed_size = zi->ci.totalCompressedData; + +# ifndef NOCRYPT + compressed_size += zi->ci.crypt_header_size; +# endif + + // update Current Item crc and sizes, + if(compressed_size >= 0xffffffff || uncompressed_size >= 0xffffffff || zi->ci.pos_local_header >= 0xffffffff) + { + /*version Made by*/ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)45,2); + /*version needed*/ + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)45,2); + + } + + zip64local_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ + + + if(compressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+20, invalidValue,4); /*compr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ + + /// set internal file attributes field + if (zi->ci.stream.data_type == Z_ASCII) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); + + if(uncompressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+24, invalidValue,4); /*uncompr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ + + // Add ZIP64 extra info field for uncompressed size + if(uncompressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for compressed size + if(compressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for relative offset to local file header of current file + if(zi->ci.pos_local_header >= 0xffffffff) + datasize += 8; + + if(datasize > 0) + { + char* p = NULL; + + if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) + { + // we can not write more data to the buffer that we have room for. + return ZIP_BADZIPFILE; + } + + p = zi->ci.central_header + zi->ci.size_centralheader; + + // Add Extra Information Header for 'ZIP64 information' + zip64local_putValue_inmemory(p, 0x0001, 2); // HeaderID + p += 2; + zip64local_putValue_inmemory(p, datasize, 2); // DataSize + p += 2; + + if(uncompressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, uncompressed_size, 8); + p += 8; + } + + if(compressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, compressed_size, 8); + p += 8; + } + + if(zi->ci.pos_local_header >= 0xffffffff) + { + zip64local_putValue_inmemory(p, zi->ci.pos_local_header, 8); + p += 8; + } + + // Update how much extra free space we got in the memory buffer + // and increase the centralheader size so the new ZIP64 fields are included + // ( 4 below is the size of HeaderID and DataSize field ) + zi->ci.size_centralExtraFree -= datasize + 4; + zi->ci.size_centralheader += datasize + 4; + + // Update the extra info size field + zi->ci.size_centralExtra += datasize + 4; + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)zi->ci.size_centralExtra,2); + } + + if (err==ZIP_OK) + err = add_data_in_datablock(&zi->central_dir, zi->ci.central_header, (uLong)zi->ci.size_centralheader); + + free(zi->ci.central_header); + + if (err==ZIP_OK) + { + // Update the LocalFileHeader with the new values. + + ZPOS64_T cur_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ + + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) + { + if(zi->ci.pos_zip64extrainfo > 0) + { + // Update the size in the ZIP64 extended field. + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_zip64extrainfo + 4,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, uncompressed_size, 8); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); + } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal + } + else + { + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); + } + + if (ZSEEK64(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + } + + zi->number_entry ++; + zi->in_opened_file_inzip = 0; + + return err; +} + +extern int ZEXPORT zipCloseFileInZip (zipFile file) +{ + return zipCloseFileInZipRaw (file,0,0); +} + +int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +{ + int err = ZIP_OK; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); + + /*num disks*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + /*relative offset*/ + if (err==ZIP_OK) /* Relative offset to the Zip64EndOfCentralDirectory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, pos,8); + + /*total disks*/ /* Do not support spawning of disk so always say 1 here*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)1,4); + + return err; +} + +int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + uLong Zip64DataSize = 44; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* size of this 'zip64 end of central directory' */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)Zip64DataSize,8); // why ZPOS64_T of this ? + + if (err==ZIP_OK) /* version made by */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* version needed */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* total number of entries in the central dir */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)size_centraldir,8); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); + } + return err; +} +int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + /*signature*/ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + { + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + } + + if (err==ZIP_OK) /* total number of entries in the central dir */ + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_centraldir,4); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff) + { + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); + } + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + } + + return err; +} + +int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +{ + int err = ZIP_OK; + uInt size_global_comment = 0; + + if(global_comment != NULL) + size_global_comment = (uInt)strlen(global_comment); + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_global_comment,2); + + if (err == ZIP_OK && size_global_comment > 0) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, global_comment, size_global_comment) != size_global_comment) + err = ZIP_ERRNO; + } + return err; +} + +extern int ZEXPORT zipClose (zipFile file, const char* global_comment) +{ + zip64_internal* zi; + int err = 0; + uLong size_centraldir = 0; + ZPOS64_T centraldir_pos_inzip; + ZPOS64_T pos; + + if (file == NULL) + return ZIP_PARAMERROR; + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + } + +#ifndef NO_ADDFILEINEXISTINGZIP + if (global_comment==NULL) + global_comment = zi->globalcomment; +#endif + + centraldir_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (err==ZIP_OK) + { + linkedlist_datablock_internal* ldi = zi->central_dir.first_block; + while (ldi!=NULL) + { + if ((err==ZIP_OK) && (ldi->filled_in_this_block>0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, ldi->data, ldi->filled_in_this_block) != ldi->filled_in_this_block) + err = ZIP_ERRNO; + } + + size_centraldir += ldi->filled_in_this_block; + ldi = ldi->next_datablock; + } + } + free_linkedlist(&(zi->central_dir)); + + pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + { + ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); + Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + Write_Zip64EndOfCentralDirectoryLocator(zi, Zip64EOCDpos); + } + + if (err==ZIP_OK) + err = Write_EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + if(err == ZIP_OK) + err = Write_GlobalComment(zi, global_comment); + + if (ZCLOSE64(zi->z_filefunc,zi->filestream) != 0) + if (err == ZIP_OK) + err = ZIP_ERRNO; + +#ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(zi->globalcomment); +#endif + TRYFREE(zi); + + return err; +} + +extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) +{ + char* p = pData; + int size = 0; + char* pNewHeader; + char* pTmp; + short header; + short dataSize; + + int retVal = ZIP_OK; + + if(pData == NULL || *dataLen < 4) + return ZIP_PARAMERROR; + + pNewHeader = (char*)ALLOC(*dataLen); + pTmp = pNewHeader; + + while(p < (pData + *dataLen)) + { + header = *(short*)p; + dataSize = *(((short*)p)+1); + + if( header == sHeader ) // Header found. + { + p += dataSize + 4; // skip it. do not copy to temp buffer + } + else + { + // Extra Info block should not be removed, So copy it to the temp buffer. + memcpy(pTmp, p, dataSize + 4); + p += dataSize + 4; + size += dataSize + 4; + } + + } + + if(size < *dataLen) + { + // clean old extra info block. + memset(pData,0, *dataLen); + + // copy the new extra info block over the old + if(size > 0) + memcpy(pData, pNewHeader, size); + + // set the new extra info size + *dataLen = size; + + retVal = ZIP_OK; + } + else + retVal = ZIP_ERRNO; + + TRYFREE(pNewHeader); + + return retVal; +} diff --git a/storage/connect/zip.h b/storage/connect/zip.h new file mode 100644 index 00000000000..8aaebb62343 --- /dev/null +++ b/storage/connect/zip.h @@ -0,0 +1,362 @@ +/* zip.h -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------- + + Changes + + See header of zip.h + +*/ + +#ifndef _zip12_H +#define _zip12_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#define HAVE_BZIP2 + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagzipFile__ { int unused; } zipFile__; +typedef zipFile__ *zipFile; +#else +typedef voidp zipFile; +#endif + +#define ZIP_OK (0) +#define ZIP_EOF (0) +#define ZIP_ERRNO (Z_ERRNO) +#define ZIP_PARAMERROR (-102) +#define ZIP_BADZIPFILE (-103) +#define ZIP_INTERNALERROR (-104) + +#ifndef DEF_MEM_LEVEL +# if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +# else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +# endif +#endif +/* default memLevel */ + +/* tm_zip contain date/time info */ +typedef struct tm_zip_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_zip; + +typedef struct +{ + tm_zip tmz_date; /* date in understandable format */ + uLong dosDate; /* if dos_date == 0, tmu_date is used */ +/* uLong flag; */ /* general purpose bit flag 2 bytes */ + + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ +} zip_fileinfo; + +typedef const char* zipcharpc; + + +#define APPEND_STATUS_CREATE (0) +#define APPEND_STATUS_CREATEAFTER (1) +#define APPEND_STATUS_ADDINZIP (2) + +extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); +extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +/* + Create a zipfile. + pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on + an Unix computer "zlib/zlib113.zip". + if the file pathname exist and append==APPEND_STATUS_CREATEAFTER, the zip + will be created at the end of the file. + (useful if the file contain a self extractor code) + if the file pathname exist and append==APPEND_STATUS_ADDINZIP, we will + add files in existing zip (be sure you don't add file that doesn't exist) + If the zipfile cannot be opened, the return value is NULL. + Else, the return value is a zipFile Handle, usable with other function + of this zip package. +*/ + +/* Note : there is no delete function into a zipfile. + If you want delete file into a zipfile, you must open a zipfile, and create another + Of couse, you can use RAW reading and writing to copy the file you did not want delte +*/ + +extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def)); + +extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_def* pzlib_filefunc_def)); + +extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level)); + +extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64)); + +/* + Open a file in the ZIP for writing. + filename : the filename in zip (if NULL, '-' without quote will be used + *zipfi contain supplemental information + if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local + contains the extrafield data the the local header + if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global + contains the extrafield data the the local header + if comment != NULL, comment contain the comment string + method contain the compression method (0 for store, Z_DEFLATED for deflate) + level contain the level of compression (can be Z_DEFAULT_COMPRESSION) + zip64 is set to 1 if a zip64 extended information block should be added to the local file header. + this MUST be '1' if the uncompressed size is >= 0xffffffff. + +*/ + + +extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw)); + + +extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64)); +/* + Same than zipOpenNewFileInZip, except if raw=1, we write raw file + */ + +extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting)); + +extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64 + )); + +/* + Same than zipOpenNewFileInZip2, except + windowBits,memLevel,,strategy : see parameter strategy in deflateInit2 + password : crypting password (NULL for no crypting) + crcForCrypting : crc of file to compress (needed for crypting) + */ + +extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase + )); + + +extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64 + )); +/* + Same than zipOpenNewFileInZip4, except + versionMadeBy : value for Version made by field + flag : value for flag field (compression level info will be added) + */ + + +extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, + const void* buf, + unsigned len)); +/* + Write data in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +/* + Close the current file in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, + uLong uncompressed_size, + uLong crc32)); + +extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32)); + +/* + Close the current file in the zipfile, for file opened with + parameter raw=1 in zipOpenNewFileInZip2 + uncompressed_size and crc32 are value for the uncompressed size +*/ + +extern int ZEXPORT zipClose OF((zipFile file, + const char* global_comment)); +/* + Close the zipfile +*/ + + +extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +/* + zipRemoveExtraInfoBlock - Added by Mathias Svensson + + Remove extra information block from a extra information data for the local file header or central directory header + + It is needed to remove ZIP64 extra information blocks when before data is written if using RAW mode. + + 0x0001 is the signature header for the ZIP64 extra information blocks + + usage. + Remove ZIP64 Extra information from a central director extra field data + zipRemoveExtraInfoBlock(pCenDirExtraFieldData, &nCenDirExtraFieldDataLen, 0x0001); + + Remove ZIP64 Extra information from a Local File Header extra field data + zipRemoveExtraInfoBlock(pLocalHeaderExtraFieldData, &nLocalHeaderExtraFieldDataLen, 0x0001); +*/ + +#ifdef __cplusplus +} +#endif + +#endif /* _zip64_H */ -- cgit v1.2.1 From 952306502ebf1b26c627c5dc8b141581eeb30671 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Wed, 14 Dec 2016 14:20:23 +0100 Subject: - MDEV-11295: developing handling files contained in ZIP file. Enable using multiple zip files modified: storage/connect/filamzip.cpp modified: storage/connect/ha_connect.cc modified: storage/connect/ha_connect.h modified: storage/connect/mycat.h modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfmt.cpp modified: storage/connect/tabjson.cpp --- storage/connect/filamzip.cpp | 19 ++++++++++--------- storage/connect/filamzip.h | 2 +- storage/connect/ha_connect.cc | 14 +++++++------- storage/connect/ha_connect.h | 39 +++------------------------------------ storage/connect/mycat.h | 1 + storage/connect/tabdos.cpp | 23 +++++++---------------- storage/connect/tabdos.h | 7 ++++--- storage/connect/tabfmt.cpp | 26 +++++++++++--------------- storage/connect/tabjson.cpp | 23 ++++++++++++++--------- storage/connect/tabxml.cpp | 7 ++++++- 10 files changed, 64 insertions(+), 97 deletions(-) (limited to 'storage') diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index ea8b827974b..8386e5be481 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -48,11 +48,11 @@ ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) { zipfile = NULL; - zfn = tdp->Zipfn; - target = tdp->Fn; +//zfn = tdp->Fn; + target = tdp->Entry; //*fn = 0; entryopen = false; - multiple = tdp->Multiple; + multiple = (target && !(strchr(target, '*') || strchr(target, '?'))) ? 0 : 1; // Init the case mapping table. #if defined(__WIN__) @@ -65,7 +65,7 @@ ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) { zipfile = txfp->zipfile; - zfn = txfp->zfn; +//zfn = txfp->zfn; target = txfp->target; //strcpy(fn, txfp->fn); finfo = txfp->finfo; @@ -129,7 +129,7 @@ int ZIPFAM::GetFileLength(PGLOBAL g) bool ZIPFAM::open(PGLOBAL g, const char *filename) { if (!zipfile && !(zipfile = unzOpen64(filename))) - sprintf(g->Message, "Zipfile open error"); + sprintf(g->Message, "Zipfile open error on %s", filename); return (zipfile == NULL); } // end of open @@ -205,7 +205,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) /*********************************************************************/ if (mode == MODE_READ) { // We used the file name relative to recorded datapath - PlugSetPath(filename, zfn, Tdbp->GetPath()); + PlugSetPath(filename, To_File, Tdbp->GetPath()); bool b = open(g, filename); @@ -218,7 +218,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) if (rc == UNZ_END_OF_LIST_OF_FILE) { sprintf(g->Message, "Target file %s not in %s", target, filename); - return true; + return false; } else if (rc != UNZ_OK) { sprintf(g->Message, "unzLocateFile rc=%d", rc); return true; @@ -229,7 +229,7 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) return true; else if (rc == RC_NF) { sprintf(g->Message, "No match of %s in %s", target, filename); - return true; + return false; } // endif rc } // endif multiple @@ -258,7 +258,8 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) } // endif fp To_Fb = fp; // Useful when closing - } // endif b + } else + return true; } else { strcpy(g->Message, "Only READ mode supported for ZIP files"); diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index 85c1f907d20..c3c04b2b3bb 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -51,7 +51,7 @@ protected: // Members unzFile zipfile; // The ZIP container file - PSZ zfn; // The ZIP file name +//PSZ zfn; // The ZIP file name PSZ target; // The target file name unz_file_info finfo; // The current file info //char fn[FILENAME_MAX]; // The current file name diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index b690dff24f4..45ca546ad4e 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,9 +171,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0009 December 09, 2016"; + char version[]= "Version 1.05.0001 December 13, 2016"; #if defined(__WIN__) - char compver[]= "Version 1.04.0009 " __DATE__ " " __TIME__; + char compver[]= "Version 1.05.0001 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -512,13 +512,13 @@ ha_create_table_option connect_table_option_list[]= HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1), HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1), -//HA_TOPTION_BOOL("COMPRESS", compressed, 0), HA_TOPTION_BOOL("MAPPED", mapped, 0), HA_TOPTION_BOOL("HUGE", huge, 0), HA_TOPTION_BOOL("SPLIT", split, 0), HA_TOPTION_BOOL("READONLY", readonly, 0), HA_TOPTION_BOOL("SEPINDEX", sepindex, 0), - HA_TOPTION_END + HA_TOPTION_BOOL("ZIPPED", zipped, 0), + HA_TOPTION_END }; @@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]= { HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1), HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX -//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), @@ -678,7 +677,6 @@ static int connect_init_func(void *p) connect_hton= (handlerton *)p; connect_hton->state= SHOW_OPTION_YES; connect_hton->create= connect_create_handler; -//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION; connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED; connect_hton->table_options= connect_table_option_list; connect_hton->field_options= connect_field_option_list; @@ -1135,7 +1133,9 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef) opval= options->sepindex; else if (!stricmp(opname, "Header")) opval= (options->header != 0); // Is Boolean for some table types - else if (options->oplist) + else if (!stricmp(opname, "Zipped")) + opval = options->zipped; + else if (options->oplist) if ((pv= GetListOption(g, opname, options->oplist))) opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 60194ac0e3c..3d9ff967618 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -83,42 +83,9 @@ extern handlerton *connect_hton; These can be specified in the CREATE TABLE: CREATE TABLE ( ... ) {...here...} -*/ -#if 0 // moved to mycat.h -typedef struct ha_table_option_struct TOS, *PTOS; - -struct ha_table_option_struct { - const char *type; - const char *filename; - const char *optname; - const char *tabname; - const char *tablist; - const char *dbname; - const char *separator; -//const char *connect; - const char *qchar; - const char *module; - const char *subtype; - const char *catfunc; - const char *srcdef; - const char *colist; - const char *oplist; - const char *data_charset; - ulonglong lrecl; - ulonglong elements; -//ulonglong estimate; - ulonglong multiple; - ulonglong header; - ulonglong quoted; - ulonglong ending; - ulonglong compressed; - bool mapped; - bool huge; - bool split; - bool readonly; - bool sepindex; - }; -#endif // 0 + + ------ Was moved to mycat.h ------ + */ /** structure for CREATE TABLE options (field options) diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index 05163f08f1b..663b68fd4b9 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -62,6 +62,7 @@ struct ha_table_option_struct { bool split; bool readonly; bool sepindex; + bool zipped; }; // Possible value for catalog functions diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 9bcac0b5f1a..f47e66b014b 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -96,11 +96,12 @@ DOSDEF::DOSDEF(void) Pseudo = 3; Fn = NULL; Ofn = NULL; - Zipfn = NULL; + Entry = NULL; To_Indx = NULL; Recfm = RECFM_VAR; Mapped = false; - Padded = false; + Zipped = false; + Padded = false; Huge = false; Accept = false; Eof = false; @@ -131,20 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && !stricmp(am, "DBF")) ? "D" : "V"; if (*dfm != 'D') - Zipfn = GetStringCatInfo(g, "Zipfile", NULL); - - if (Zipfn && Multiple) { - // Prevent Fn to default to table name - Desc = GetStringCatInfo(g, "Filename", NULL); - Fn = GetStringCatInfo(g, "Filename", "<%>"); - - if (!strcmp(Fn, "<%>")) - Fn = NULL; - - } else - Desc = Fn = GetStringCatInfo(g, "Filename", NULL); + Zipped = GetBoolCatInfo("Zipped", false); + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); + Entry = GetStringCatInfo(g, "Entry", NULL); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : (toupper(*buf) == 'B') ? RECFM_BIN : @@ -350,7 +342,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Zipfn) { + if (Zipped) { #if defined(ZIP_SUPPORT) if (Recfm == RECFM_VAR) txfp = new(g) ZIPFAM(this); @@ -358,7 +350,6 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) ZPXFAM(this); tdbp = new(g) TDBDOS(this, txfp); - return tdbp; #else // !ZIP_SUPPORT strcpy(g->Message, "ZIP not supported"); return NULL; diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 501ddbc2e0b..623adcfed0d 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -59,7 +59,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Methods virtual int Indexable(void) - {return (!Multiple && !Zipfn && Compressed != 1) ? 1 : 0;} + {return (!Multiple && !Zipped && Compressed != 1) ? 1 : 0;} virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); @@ -73,11 +73,12 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Members PSZ Fn; /* Path/Name of corresponding file */ PSZ Ofn; /* Base Path/Name of matching index files*/ - PSZ Zipfn; /* Zip container name */ + PSZ Entry; /* Zip entry name or pattern */ PIXDEF To_Indx; /* To index definitions blocks */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ - bool Padded; /* true for padded table file */ + bool Zipped; /* true for zipped table file */ + bool Padded; /* true for padded table file */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted */ bool Eof; /* true if an EOF (0xA) character exists */ diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index d6649a0093b..2c4d605e66c 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -108,6 +108,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) goto skipit; } // endif info + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + // num_max = atoi(p+1); // Max num of record to test imax = hmax = nerr = 0; @@ -123,18 +128,16 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) /*********************************************************************/ tdp = new(g) CSVDEF; #if defined(ZIP_SUPPORT) - tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL); - tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); #endif // ZIP_SUPPORT - tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); - if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) { + if (!tdp->Fn) { strcpy(g->Message, MSG(MISSING_FNAME)); return NULL; } // endif Fn - fn = (tdp->Fn) ? tdp->Fn : "unnamed"; - if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) tdp->Lrecl = 4096; @@ -174,7 +177,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); - if (tdp->Zipfn) + if (tdp->Zipped) tdbp = new(g) TDBCSV(tdp, new(g) ZIPFAM(tdp)); else tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); @@ -493,16 +496,9 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) /*******************************************************************/ /* Allocate a file processing class of the proper type. */ /*******************************************************************/ - if (Zipfn) { + if (Zipped) { #if defined(ZIP_SUPPORT) txfp = new(g) ZIPFAM(this); - - if (!Fmtd) - tdbp = new(g) TDBCSV(this, txfp); - else - tdbp = new(g) TDBFMT(this, txfp); - - return tdbp; #else // !ZIP_SUPPORT strcpy(g->Message, "ZIP not supported"); return NULL; diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 73c6a6d85a4..eff95445a3a 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -94,7 +94,12 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ lvl = GetIntegerTableOption(g, topt, "Level", 0); @@ -102,12 +107,12 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp = new(g) JSONDEF; #if defined(ZIP_SUPPORT) - tdp->Zipfn = GetStringTableOption(g, topt, "Zipfile", NULL); - tdp->Multiple = GetIntegerTableOption(g, topt, "Multiple", 0); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); #endif // ZIP_SUPPORT tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); - if (!tdp->Fn && !tdp->Zipfn && !tdp->Multiple) { + if (!tdp->Fn) { strcpy(g->Message, MSG(MISSING_FNAME)); return NULL; } // endif Fn @@ -122,7 +127,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - if (tdp->Zipfn) + if (tdp->Zipped) tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); else tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); @@ -139,7 +144,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - if (tdp->Zipfn) + if (tdp->Zipped) tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp)); else tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); @@ -424,7 +429,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Zipfn) { + if (Zipped) { #if defined(ZIP_SUPPORT) txfp = new(g) ZIPFAM(this); #else // !ZIP_SUPPORT @@ -462,7 +467,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - if (Zipfn) + if (Zipped) txfp = new(g) ZIPFAM(this); else txfp = new(g) MAPFAM(this); @@ -471,7 +476,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSON*)tdbp)->G = g; } // endif Pretty - if (Multiple && !Zipfn) + if (Multiple) tdbp = new(g) TDBMUL(tdbp); return tdbp; diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 57d204a4286..1993b07eb7a 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -136,7 +136,12 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { -- cgit v1.2.1 From e6b563f8be68d57df2a4c9b8e2b6c130855b18e4 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Fri, 23 Dec 2016 16:58:32 +0100 Subject: Fix some XML table type bugs: - in DOMNODELIST::DropItem if (Listp == NULL || Listp->length <= n) return true; is wrong, should be: if (Listp == NULL || Listp->length < n) return true; - Crash in discovery with libxml2 in XMLColumns because: if (!tdp->Usedom) // nl was destroyed vp->nl = vp->pn->GetChildElements(g); is executed with vp->pn uninitialized. Fixed by adding: vp->pn = node; line 264. -In discovery with libxml2 some columns are not found. Because list was not recovered properly, nodes being modified and not reallocated. Fixed lines 214 and 277. modified: storage/connect/domdoc.cpp modified: storage/connect/tabxml.cpp Add support for zipped table files modified: storage/connect/domdoc.cpp modified: storage/connect/domdoc.h modified: storage/connect/filamap.cpp modified: storage/connect/filamap.h modified: storage/connect/filamzip.cpp modified: storage/connect/filamzip.h modified: storage/connect/ha_connect.cc modified: storage/connect/libdoc.cpp modified: storage/connect/plgdbutl.cpp modified: storage/connect/plgxml.cpp modified: storage/connect/plgxml.h modified: storage/connect/tabdos.cpp modified: storage/connect/tabdos.h modified: storage/connect/tabfmt.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp --- storage/connect/domdoc.cpp | 40 ++-- storage/connect/domdoc.h | 4 +- storage/connect/filamap.cpp | 49 +++-- storage/connect/filamap.h | 3 +- storage/connect/filamzip.cpp | 426 +++++++++++++++++++++++++++--------------- storage/connect/filamzip.h | 86 ++++++--- storage/connect/ha_connect.cc | 6 +- storage/connect/libdoc.cpp | 30 ++- storage/connect/plgdbutl.cpp | 2 +- storage/connect/plgxml.cpp | 55 +++++- storage/connect/plgxml.h | 16 +- storage/connect/tabdos.cpp | 34 ++-- storage/connect/tabdos.h | 13 +- storage/connect/tabfmt.cpp | 11 +- storage/connect/tabjson.cpp | 33 +++- storage/connect/tabxml.cpp | 56 ++++-- storage/connect/tabxml.h | 18 +- 17 files changed, 604 insertions(+), 278 deletions(-) (limited to 'storage') diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index 64a0a172956..eb9660b439d 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -89,30 +89,43 @@ DOMDOC::DOMDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool DOMDOC::Initialize(PGLOBAL g) - { - if (TestHr(g, CoInitialize(NULL))) +bool DOMDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + + if (TestHr(g, CoInitialize(NULL))) return true; if (TestHr(g, Docp.CreateInstance("msxml2.domdocument"))) return true; return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool DOMDOC::ParseFile(char *fn) - { - // Load the document +bool DOMDOC::ParseFile(PGLOBAL g, char *fn) +{ + bool b; + Docp->async = false; - if (!(bool)Docp->load((_bstr_t)fn)) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + b = (xdoc) ? (bool)Docp->loadXML((_bstr_t)xdoc) : false; + } else + // Load the document + b = (bool)Docp->load((_bstr_t)fn); + + if (!b) return true; return false; - } // end of ParseFile +} // end of ParseFile /******************************************************************/ /* Create or reuse an Xblock for this document. */ @@ -239,6 +252,7 @@ int DOMDOC::DumpDoc(PGLOBAL g, char *ofn) void DOMDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) { CloseXMLFile(g, xp, false); + CloseZip(); } // end of Close /* ----------------------- class DOMNODE ------------------------ */ @@ -616,13 +630,13 @@ PXNODE DOMNODELIST::GetItem(PGLOBAL g, int n, PXNODE np) /* Reset the pointer on the deleted item. */ /******************************************************************/ bool DOMNODELIST::DropItem(PGLOBAL g, int n) - { - if (Listp == NULL || Listp->length <= n) - return true; +{ + if (Listp == NULL || Listp->length < n) + return true; //Listp->item[n] = NULL; La propriété n'a pas de méthode 'set' return false; - } // end of DeleteItem +} // end of DeleteItem /* ----------------------- class DOMATTR ------------------------ */ diff --git a/storage/connect/domdoc.h b/storage/connect/domdoc.h index 2cffec499e2..cfec98a9422 100644 --- a/storage/connect/domdoc.h +++ b/storage/connect/domdoc.h @@ -37,8 +37,8 @@ class DOMDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {} // Only libxml2 // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index 3c5b3ae7592..94c562a9981 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -319,11 +319,13 @@ int MAPFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int MAPFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; // Are we at the end of the memory - if (Mempos >= Top) - return RC_EF; + if (Mempos >= Top) + if ((rc = GetNext(g)) != RC_OK) + return rc; + if (!Placed) { /*******************************************************************/ @@ -341,8 +343,10 @@ int MAPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + case RC_NF: // Skip this record if ((rc = SkipRecord(g, false)) != RC_OK) return rc; @@ -569,7 +573,7 @@ int MBKFAM::GetRowID(void) /***********************************************************************/ int MBKFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; /*********************************************************************/ /* Sequential block reading when Placed is not true. */ @@ -577,8 +581,10 @@ int MBKFAM::ReadBuffer(PGLOBAL g) if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -588,7 +594,8 @@ int MBKFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + if ((rc = GetNext(g)) != RC_OK) + return rc; /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -596,8 +603,11 @@ int MBKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc @@ -697,14 +707,18 @@ int MPXFAM::InitDelete(PGLOBAL, int fpos, int) /***********************************************************************/ int MPXFAM::ReadBuffer(PGLOBAL g) { + int rc; + /*********************************************************************/ /* Sequential block reading when Placed is not true. */ /*********************************************************************/ if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -714,7 +728,7 @@ int MPXFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + return GetNext(g); /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -722,8 +736,11 @@ int MPXFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc diff --git a/storage/connect/filamap.h b/storage/connect/filamap.h index b9c8ad965fd..774eb8b91b3 100644 --- a/storage/connect/filamap.h +++ b/storage/connect/filamap.h @@ -41,7 +41,8 @@ class DllExport MAPFAM : public TXTFAM { virtual int SkipRecord(PGLOBAL g, bool header); virtual bool OpenTableFile(PGLOBAL g); virtual bool DeferReading(void) {return false;} - virtual int ReadBuffer(PGLOBAL g); + virtual int GetNext(PGLOBAL g) {return RC_EF;} + virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); virtual int DeleteRecords(PGLOBAL g, int irc); virtual void CloseTableFile(PGLOBAL g, bool abort); diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index 8386e5be481..65013e170e4 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -40,19 +40,21 @@ //#include "tabzip.h" #include "filamzip.h" -/* -------------------------- class ZIPFAM --------------------------- */ +/* -------------------------- class ZIPUTIL -------------------------- */ /***********************************************************************/ /* Constructors. */ /***********************************************************************/ -ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +ZIPUTIL::ZIPUTIL(PSZ tgt, bool mul) { zipfile = NULL; -//zfn = tdp->Fn; - target = tdp->Entry; -//*fn = 0; + target = tgt; + fp = NULL; + memory = NULL; + size = 0; entryopen = false; - multiple = (target && !(strchr(target, '*') || strchr(target, '?'))) ? 0 : 1; + multiple = mul; + memset(fn, 0, sizeof(fn)); // Init the case mapping table. #if defined(__WIN__) @@ -60,29 +62,30 @@ ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) #else for (int i = 0; i < 256; ++i) mapCaseTable[i] = i; #endif -} // end of ZIPFAM standard constructor +} // end of ZIPUTIL standard constructor -ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +#if 0 +ZIPUTIL::ZIPUTIL(PZIPUTIL zutp) { - zipfile = txfp->zipfile; -//zfn = txfp->zfn; - target = txfp->target; -//strcpy(fn, txfp->fn); - finfo = txfp->finfo; - entryopen = txfp->entryopen; - multiple = txfp->multiple; - for (int i = 0; i < 256; ++i) mapCaseTable[i] = txfp->mapCaseTable[i]; -} // end of ZIPFAM copy constructor + zipfile = zutp->zipfile; + target = zutp->target; + fp = zutp->fp; + finfo = zutp->finfo; + entryopen = zutp->entryopen; + multiple = zutp->multiple; + for (int i = 0; i < 256; ++i) mapCaseTable[i] = zutp->mapCaseTable[i]; +} // end of ZIPUTIL copy constructor +#endif // 0 /***********************************************************************/ /* This code is the copyright property of Alessandro Felice Cantatore. */ /* http://xoomer.virgilio.it/acantato/dev/wildcard/wildmatch.html */ /***********************************************************************/ -bool ZIPFAM::WildMatch(PSZ pat, PSZ str) { +bool ZIPUTIL::WildMatch(PSZ pat, PSZ str) { PSZ s, p; bool star = FALSE; - loopStart: +loopStart: for (s = str, p = pat; *s; ++s, ++p) { switch (*p) { case '?': @@ -102,31 +105,18 @@ bool ZIPFAM::WildMatch(PSZ pat, PSZ str) { if (*p == '*') ++p; return (!*p); - starCheck: +starCheck: if (!star) return FALSE; str++; goto loopStart; } // end of WildMatch -/***********************************************************************/ -/* ZIP GetFileLength: returns file size in number of bytes. */ -/***********************************************************************/ -int ZIPFAM::GetFileLength(PGLOBAL g) -{ - int len = (entryopen) ? Top - Memory : 100; // not 0 to avoid ASSERT - - if (trace) - htrc("Zipped file length=%d\n", len); - - return len; -} // end of GetFileLength - /***********************************************************************/ /* open a zip file. */ /* param: filename path and the filename of the zip file to open. */ /* return: true if open, false otherwise. */ /***********************************************************************/ -bool ZIPFAM::open(PGLOBAL g, const char *filename) +bool ZIPUTIL::open(PGLOBAL g, char *filename) { if (!zipfile && !(zipfile = unzOpen64(filename))) sprintf(g->Message, "Zipfile open error on %s", filename); @@ -137,7 +127,7 @@ bool ZIPFAM::open(PGLOBAL g, const char *filename) /***********************************************************************/ /* Close the zip file. */ /***********************************************************************/ -void ZIPFAM::close() +void ZIPUTIL::close() { if (zipfile) { closeEntry(); @@ -150,10 +140,9 @@ void ZIPFAM::close() /***********************************************************************/ /* Find next entry matching target pattern. */ /***********************************************************************/ -int ZIPFAM::findEntry(PGLOBAL g, bool next) +int ZIPUTIL::findEntry(PGLOBAL g, bool next) { int rc; - char fn[FILENAME_MAX]; // The current entry file name do { if (next) { @@ -188,37 +177,53 @@ int ZIPFAM::findEntry(PGLOBAL g, bool next) strcpy(g->Message, "FindNext logical error"); return RC_FX; -} // end of FindNext +} // end of FindEntry + /***********************************************************************/ -/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/* Get the next used entry. */ /***********************************************************************/ -bool ZIPFAM::OpenTableFile(PGLOBAL g) +int ZIPUTIL::nextEntry(PGLOBAL g) { - char filename[_MAX_PATH]; - MODE mode = Tdbp->GetMode(); - PFBLOCK fp; - PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; + if (multiple) { + int rc; + + closeEntry(); + + if ((rc = findEntry(g, true)) != RC_OK) + return rc; + + if (openEntry(g)) + return RC_FX; + return RC_OK; + } else + return RC_EF; + +} // end of nextEntry + + +/***********************************************************************/ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/***********************************************************************/ +bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, char *fn) +{ /*********************************************************************/ /* The file will be decompressed into virtual memory. */ /*********************************************************************/ - if (mode == MODE_READ) { - // We used the file name relative to recorded datapath - PlugSetPath(filename, To_File, Tdbp->GetPath()); - - bool b = open(g, filename); + if (mode == MODE_READ || mode == MODE_ANY) { + bool b = open(g, fn); if (!b) { int rc; - + if (target && *target) { if (!multiple) { rc = unzLocateFile(zipfile, target, 0); if (rc == UNZ_END_OF_LIST_OF_FILE) { - sprintf(g->Message, "Target file %s not in %s", target, filename); - return false; + sprintf(g->Message, "Target file %s not in %s", target, fn); + return true; } else if (rc != UNZ_OK) { sprintf(g->Message, "unzLocateFile rc=%d", rc); return true; @@ -227,9 +232,9 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) } else { if ((rc = findEntry(g, false)) == RC_FX) return true; - else if (rc == RC_NF) { - sprintf(g->Message, "No match of %s in %s", target, filename); - return false; + else if (rc == RC_EF) { + sprintf(g->Message, "No match of %s in %s", target, fn); + return true; } // endif rc } // endif multiple @@ -239,25 +244,26 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) if (openEntry(g)) return true; - if (Top > Memory) { + if (size > 0) { /*******************************************************************/ /* Link a Fblock. This make possible to automatically close it */ /* in case of error g->jump. */ /*******************************************************************/ + PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; + fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK)); fp->Type = TYPE_FB_ZIP; - fp->Fname = PlugDup(g, filename); + fp->Fname = PlugDup(g, fn); fp->Next = dbuserp->Openlist; dbuserp->Openlist = fp; fp->Count = 1; - fp->Length = Top - Memory; - fp->Memory = Memory; + fp->Length = size; + fp->Memory = memory; fp->Mode = mode; fp->File = this; fp->Handle = NULL; } // endif fp - To_Fb = fp; // Useful when closing } else return true; @@ -267,65 +273,161 @@ bool ZIPFAM::OpenTableFile(PGLOBAL g) } // endif mode return false; - } // end of OpenTableFile +} // end of OpenTableFile /***********************************************************************/ /* Open target in zip file. */ /***********************************************************************/ -bool ZIPFAM::openEntry(PGLOBAL g) +bool ZIPUTIL::openEntry(PGLOBAL g) { - int rc; - uint size; + int rc; - rc = unzGetCurrentFileInfo(zipfile, &finfo, 0, 0, 0, 0, 0, 0); + rc = unzGetCurrentFileInfo(zipfile, &finfo, fn, sizeof(fn), + NULL, 0, NULL, 0); if (rc != UNZ_OK) { sprintf(g->Message, "unzGetCurrentFileInfo64 rc=%d", rc); return true; } else if ((rc = unzOpenCurrentFile(zipfile)) != UNZ_OK) { - sprintf(g->Message, "unzOpenCurrentFile rc=%d", rc); + sprintf(g->Message, "unzOpen fn=%s rc=%d", fn, rc); return true; } // endif rc size = finfo.uncompressed_size; - Memory = new char[size]; + memory = new char[size + 1]; - if ((rc = unzReadCurrentFile(zipfile, Memory, size)) < 0) { + if ((rc = unzReadCurrentFile(zipfile, memory, size)) < 0) { sprintf(g->Message, "unzReadCurrentFile rc = ", rc); unzCloseCurrentFile(zipfile); - free(Memory); + free(memory); + memory = NULL; entryopen = false; } else { - // The pseudo "buffer" is here the entire real buffer - Fpos = Mempos = Memory; - Top = Memory + size; - - if (trace) - htrc("Memory=%p size=%ud Top=%p\n", Memory, size, Top); - + memory[size] = 0; // Required by some table types (XML) entryopen = true; } // endif rc + if (trace) + htrc("Openning entry%s %s\n", fn, (entryopen) ? "oked" : "failed"); + return !entryopen; } // end of openEntry /***********************************************************************/ /* Close the zip file. */ /***********************************************************************/ -void ZIPFAM::closeEntry() +void ZIPUTIL::closeEntry() { if (entryopen) { unzCloseCurrentFile(zipfile); entryopen = false; } // endif entryopen - if (Memory) { - free(Memory); - Memory = NULL; - } // endif Memory + if (memory) { + free(memory); + memory = NULL; + } // endif memory } // end of closeEntry +/* -------------------------- class ZIPFAM --------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); +} // end of ZIPFAM standard constructor + +ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM copy constructor + +ZIPFAM::ZIPFAM(PDOSDEF tdp, PZPXFAM txfp) : MAPFAM(tdp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM constructor used in ResetTableOpt + +/***********************************************************************/ +/* ZIP GetFileLength: returns file size in number of bytes. */ +/***********************************************************************/ +int ZIPFAM::GetFileLength(PGLOBAL g) +{ + int len = (zutp && zutp->entryopen) ? Top - Memory + : TXTFAM::GetFileLength(g) * 3; + + if (trace) + htrc("Zipped file length=%d\n", len); + + return len; +} // end of GetFileLength + +/***********************************************************************/ +/* ZIP Cardinality: return the number of rows if possible. */ +/***********************************************************************/ +int ZIPFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + card = (len / (int)Lrecl) * 2; // Estimated ??? + return card; +} // end of Cardinality + +/***********************************************************************/ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/***********************************************************************/ +bool ZIPFAM::OpenTableFile(PGLOBAL g) +{ + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + zutp = new(g) ZIPUTIL(target, mul); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Fpos = Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + return false; + } // end of OpenTableFile + +/***********************************************************************/ +/* GetNext: go to next entry. */ +/***********************************************************************/ +int ZIPFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); + + if (rc != RC_OK) + return rc; + + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + return RC_OK; +} // end of GetNext + +#if 0 /***********************************************************************/ /* ReadBuffer: Read one line for a ZIP file. */ /***********************************************************************/ @@ -335,19 +437,12 @@ int ZIPFAM::ReadBuffer(PGLOBAL g) // Are we at the end of the memory if (Mempos >= Top) { - if (multiple) { - closeEntry(); - - if ((rc = findEntry(g, true)) != RC_OK) - return rc; - - if (openEntry(g)) - return RC_FX; + if ((rc = zutp->nextEntry(g)) != RC_OK) + return rc; - } else - return RC_EF; - - } // endif Mempos + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + } // endif Mempos #if 0 if (!Placed) { @@ -399,7 +494,6 @@ int ZIPFAM::ReadBuffer(PGLOBAL g) return RC_OK; } // end of ReadBuffer -#if 0 /***********************************************************************/ /* Table file close routine for MAP access method. */ /***********************************************************************/ @@ -414,89 +508,115 @@ void ZIPFAM::CloseTableFile(PGLOBAL g, bool) /***********************************************************************/ /* Constructors. */ /***********************************************************************/ -ZPXFAM::ZPXFAM(PDOSDEF tdp) : ZIPFAM(tdp) +ZPXFAM::ZPXFAM(PDOSDEF tdp) : MPXFAM(tdp) { - Lrecl = tdp->GetLrecl(); + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); } // end of ZPXFAM standard constructor -ZPXFAM::ZPXFAM(PZPXFAM txfp) : ZIPFAM(txfp) +ZPXFAM::ZPXFAM(PZPXFAM txfp) : MPXFAM(txfp) { - Lrecl = txfp->Lrecl; + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +//Lrecl = txfp->Lrecl; } // end of ZPXFAM copy constructor /***********************************************************************/ -/* ReadBuffer: Read one line for a fixed ZIP file. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -int ZPXFAM::ReadBuffer(PGLOBAL g) +int ZPXFAM::GetFileLength(PGLOBAL g) { - int rc, len; + int len; - // Are we at the end of the memory - if (Mempos >= Top) { - if (multiple) { - closeEntry(); + if (!zutp && OpenTableFile(g)) + return 0; - if ((rc = findEntry(g, true)) != RC_OK) - return rc; + if (zutp->entryopen) + len = zutp->size; + else + len = 0; - if (openEntry(g)) - return RC_FX; + return len; +} // end of GetFileLength - } else - return RC_EF; +/***********************************************************************/ +/* ZIP Cardinality: return the number of rows if possible. */ +/***********************************************************************/ +int ZPXFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; - } // endif Mempos + int card = -1; + int len = GetFileLength(g); -#if 0 - if (!Placed) { - /*******************************************************************/ - /* Record file position in case of UPDATE or DELETE. */ - /*******************************************************************/ - int rc; + if (!(len % Lrecl)) + card = len / (int)Lrecl; // Fixed length file + else + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); - next: - Fpos = Mempos; - CurBlk = (int)Rows++; + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality - /*******************************************************************/ - /* Check whether optimization on ROWID */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - // Skip this record - if ((rc = SkipRecord(g, false)) != RC_OK) - return rc; +/***********************************************************************/ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ +/***********************************************************************/ +bool ZPXFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); - goto next; - } // endswitch rc + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)ZIPUTIL(target, mul); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Fpos = Mempos = Memory + Headlen; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; } else - Placed = false; -#else - // Perhaps unuseful - Fpos = Mempos; - CurBlk = (int)Rows++; - Placed = false; -#endif + Reset(); - // Immediately calculate next position (Used by DeleteDB) - Mempos += Lrecl; + return false; +} // end of OpenTableFile - // Set caller line buffer - len = Lrecl; +/***********************************************************************/ +/* GetNext: go to next entry. */ +/***********************************************************************/ +int ZPXFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - // Don't rely on ENDING setting - if (len > 0 && *(Mempos - 1) == '\n') - len--; // Line ends by LF + if (rc != RC_OK) + return rc; - if (len > 0 && *(Mempos - 2) == '\r') - len--; // Line ends by CRLF + int len = zutp->size; - memcpy(Tdbp->GetLine(), Fpos, len); - Tdbp->GetLine()[len] = '\0'; + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size + + Memory = zutp->memory; + Top = Memory + len; + Rewind(); return RC_OK; -} // end of ReadBuffer +} // end of GetNext diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index c3c04b2b3bb..9312fb2f70e 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -17,67 +17,101 @@ typedef class ZIPFAM *PZIPFAM; typedef class ZPXFAM *PZPXFAM; +/***********************************************************************/ +/* This is the ZIP utility fonctions class. */ +/***********************************************************************/ +class DllExport ZIPUTIL : public BLOCK { +public: + // Constructor + ZIPUTIL(PSZ tgt, bool mul); +//ZIPUTIL(ZIPUTIL *zutp); + + // Implementation +//PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual bool OpenTable(PGLOBAL g, MODE mode, char *fn); + bool open(PGLOBAL g, char *fn); + bool openEntry(PGLOBAL g); + void close(void); + void closeEntry(void); + bool WildMatch(PSZ pat, PSZ str); + int findEntry(PGLOBAL g, bool next); + int nextEntry(PGLOBAL g); + + // Members + unzFile zipfile; // The ZIP container file + PSZ target; // The target file name + unz_file_info finfo; // The current file info + PFBLOCK fp; + char *memory; + uint size; + int multiple; // Multiple targets + bool entryopen; // True when open current entry + char fn[FILENAME_MAX]; // The current entry file name + char mapCaseTable[256]; +}; // end of ZIPFAM + /***********************************************************************/ /* This is the ZIP file access method. */ /***********************************************************************/ class DllExport ZIPFAM : public MAPFAM { + friend class ZPXFAM; public: - // Constructor + // Constructors ZIPFAM(PDOSDEF tdp); ZIPFAM(PZIPFAM txfp); + ZIPFAM(PDOSDEF tdp, PZPXFAM txfp); // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} - virtual PTXF Duplicate(PGLOBAL g) {return (PTXF) new(g) ZIPFAM(this);} + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } // Methods + virtual int Cardinality(PGLOBAL g); virtual int GetFileLength(PGLOBAL g); - virtual int Cardinality(PGLOBAL g) {return (g) ? 10 : 1;} //virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} virtual bool OpenTableFile(PGLOBAL g); - virtual bool DeferReading(void) {return false;} - virtual int ReadBuffer(PGLOBAL g); + virtual bool DeferReading(void) { return false; } + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); //virtual int WriteBuffer(PGLOBAL g); //virtual int DeleteRecords(PGLOBAL g, int irc); //virtual void CloseTableFile(PGLOBAL g, bool abort); - void close(void); protected: - bool open(PGLOBAL g, const char *filename); - bool openEntry(PGLOBAL g); - void closeEntry(void); - bool WildMatch(PSZ pat, PSZ str); - int findEntry(PGLOBAL g, bool next); - // Members - unzFile zipfile; // The ZIP container file -//PSZ zfn; // The ZIP file name - PSZ target; // The target file name - unz_file_info finfo; // The current file info -//char fn[FILENAME_MAX]; // The current file name - bool entryopen; // True when open current entry - int multiple; // Multiple targets - char mapCaseTable[256]; + ZIPUTIL *zutp; + PSZ target; + bool mul; }; // end of ZIPFAM /***********************************************************************/ /* This is the fixed ZIP file access method. */ /***********************************************************************/ -class DllExport ZPXFAM : public ZIPFAM { +class DllExport ZPXFAM : public MPXFAM { + friend class ZIPFAM; public: - // Constructor + // Constructors ZPXFAM(PDOSDEF tdp); ZPXFAM(PZPXFAM txfp); // Implementation - virtual PTXF Duplicate(PGLOBAL g) {return (PTXF) new(g) ZPXFAM(this);} + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZPXFAM(this); } // Methods - virtual int ReadBuffer(PGLOBAL g); + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); protected: // Members - int Lrecl; + ZIPUTIL *zutp; + PSZ target; + bool mul; }; // end of ZPXFAM #endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 45ca546ad4e..6590902bcd4 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1242,8 +1242,10 @@ char *ha_connect::GetStringOption(char *opname, char *sdef) if (opval && (!stricmp(opname, "connect") || !stricmp(opname, "tabname") - || !stricmp(opname, "filename"))) - opval = GetRealString(opval); + || !stricmp(opname, "filename") + || !stricmp(opname, "optname") + || !stricmp(opname, "entry"))) + opval = GetRealString(opval); if (!opval) { if (sdef && !strcmp(sdef, "*")) { diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index c2882fc0d7c..2470d37c353 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -1,6 +1,6 @@ /******************************************************************/ /* Implementation of XML document processing using libxml2 */ -/* Author: Olivier Bertrand 2007-2015 */ +/* Author: Olivier Bertrand 2007-2016 */ /******************************************************************/ #include "my_global.h" #include @@ -68,8 +68,8 @@ class LIBXMLDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {Nofreelist = b;} // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); @@ -373,22 +373,33 @@ LIBXMLDOC::LIBXMLDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool LIBXMLDOC::Initialize(PGLOBAL g) - { +bool LIBXMLDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + int n = xmlKeepBlanksDefault(1); return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool LIBXMLDOC::ParseFile(char *fn) +bool LIBXMLDOC::ParseFile(PGLOBAL g, char *fn) { if (trace) htrc("ParseFile\n"); - if ((Docp = xmlParseFile(fn))) { - if (Docp->encoding) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + Docp = (xdoc) ? xmlParseDoc((const xmlChar *)xdoc) : NULL; + } else + Docp = xmlParseFile(fn); + + if (Docp) { + if (Docp->encoding) Encoding = (char*)Docp->encoding; return false; @@ -609,6 +620,7 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) } // endif xp CloseXML2File(g, xp, false); + CloseZip(); } // end of Close /******************************************************************/ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index 31c040c6957..83975c6d8fa 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -939,7 +939,7 @@ int PlugCloseFile(PGLOBAL g __attribute__((unused)), PFBLOCK fp, bool all) #endif // LIBXML2_SUPPORT #ifdef ZIP_SUPPORT case TYPE_FB_ZIP: - ((PZIPFAM)fp->File)->close(); + ((ZIPUTIL*)fp->File)->close(); fp->Memory = NULL; fp->Mode = MODE_ANY; fp->Count = 0; diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index 3061a6d697e..71b72621b06 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -30,19 +30,51 @@ PXDOC GetLibxmlDoc(PGLOBAL g, char *nsl, char *nsdf, /* XMLDOCUMENT constructor. */ /******************************************************************/ XMLDOCUMENT::XMLDOCUMENT(char *nsl, char *nsdf, char *enc) - { - Namespaces = NULL; +{ +#if defined(ZIP_SUPPORT) + zip = NULL; +#else // !ZIP_SUPPORT + zip = false; +#endif // !ZIP_SUPPORT + Namespaces = NULL; Encoding = enc; Nslist = nsl; DefNs = nsdf; - } // end of XMLDOCUMENT constructor +} // end of XMLDOCUMENT constructor + +/******************************************************************/ +/* Initialize zipped file processing. */ +/******************************************************************/ +bool XMLDOCUMENT::InitZip(PGLOBAL g, char *entry) +{ +#if defined(ZIP_SUPPORT) + bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; + zip = new(g) ZIPUTIL(entry, mul); + return zip == NULL; +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return true; +#endif // !ZIP_SUPPORT +} // end of InitZip + +/******************************************************************/ +/* Make the namespace structure list. */ +/******************************************************************/ +char* XMLDOCUMENT::GetMemDoc(PGLOBAL g, char *fn) +{ +#if defined(ZIP_SUPPORT) + return (zip->OpenTable(g, MODE_ANY, fn)) ? NULL : zip->memory; +#else // !ZIP_SUPPORT + return NULL; +#endif // !ZIP_SUPPORT +} // end of GetMemDoc /******************************************************************/ /* Make the namespace structure list. */ /******************************************************************/ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) - { - char *prefix, *href, *next = Nslist; +{ + char *prefix, *href, *next = Nslist; PNS nsp, *ppns = &Namespaces; while (next) { @@ -83,6 +115,19 @@ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) return false; } // end of MakeNSlist +/******************************************************************/ +/* Close ZIP file. */ +/******************************************************************/ +void XMLDOCUMENT::CloseZip(void) +{ +#if defined(ZIP_SUPPORT) + if (zip) { + zip->close(); + zip = NULL; + } // endif zip +#endif // ZIP_SUPPORT +} // end of CloseZip + /******************************************************************/ /* XMLNODE constructor. */ /******************************************************************/ diff --git a/storage/connect/plgxml.h b/storage/connect/plgxml.h index b8e914e0bf1..db7dfa6bda5 100644 --- a/storage/connect/plgxml.h +++ b/storage/connect/plgxml.h @@ -1,3 +1,7 @@ +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT + /******************************************************************/ /* Dual XML implementation base classes defines. */ /******************************************************************/ @@ -72,8 +76,8 @@ class XMLDOCUMENT : public BLOCK { virtual void SetNofree(bool b) = 0; // Methods - virtual bool Initialize(PGLOBAL) = 0; - virtual bool ParseFile(char *) = 0; + virtual bool Initialize(PGLOBAL, char *, bool) = 0; + virtual bool ParseFile(PGLOBAL, char *) = 0; virtual bool NewDoc(PGLOBAL, char *) = 0; virtual void AddComment(PGLOBAL, char *) = 0; virtual PXNODE GetRoot(PGLOBAL) = 0; @@ -91,8 +95,16 @@ class XMLDOCUMENT : public BLOCK { // Utility bool MakeNSlist(PGLOBAL g); + bool InitZip(PGLOBAL g, char *entry); + char *GetMemDoc(PGLOBAL g, char *fn); + void CloseZip(void); // Members +#if defined(ZIP_SUPPORT) + ZIPUTIL *zip; /* Used for zipped file */ +#else // !ZIP_SUPPORT + bool zip; /* Always false */ +#endif // !ZIP_SUPPORT PNS Namespaces; /* To the namespaces */ char *Encoding; /* The document encoding */ char *Nslist; /* Namespace list */ diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index f47e66b014b..16cc6c33b44 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -101,6 +101,7 @@ DOSDEF::DOSDEF(void) Recfm = RECFM_VAR; Mapped = false; Zipped = false; + Mulentries = false; Padded = false; Huge = false; Accept = false; @@ -131,12 +132,13 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && !stricmp(am, "DBF")) ? "D" : "V"; - if (*dfm != 'D') - Zipped = GetBoolCatInfo("Zipped", false); + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); - Entry = GetStringCatInfo(g, "Entry", NULL); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); Recfm = (toupper(*buf) == 'F') ? RECFM_FIX : (toupper(*buf) == 'B') ? RECFM_BIN : @@ -344,14 +346,16 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (Zipped) { #if defined(ZIP_SUPPORT) - if (Recfm == RECFM_VAR) - txfp = new(g) ZIPFAM(this); - else - txfp = new(g) ZPXFAM(this); + if (Recfm == RECFM_VAR) { + txfp = new(g)ZIPFAM(this); + tdbp = new(g)TDBDOS(this, txfp); + } else { + txfp = new(g)ZPXFAM(this); + tdbp = new(g)TDBFIX(this, txfp); + } // endif Recfm - tdbp = new(g) TDBDOS(this, txfp); #else // !ZIP_SUPPORT - strcpy(g->Message, "ZIP not supported"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); return NULL; #endif // !ZIP_SUPPORT } else if (Recfm == RECFM_DBF) { @@ -559,7 +563,7 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); #endif // GZ_SUPPORT - } else if (Txfp->GetAmType() == TYPE_AM_BLK) + } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); @@ -630,7 +634,12 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) defp->SetOptimized(0); // Estimate the number of needed blocks - block = (int)((MaxSize + (int)nrec - 1) / (int)nrec); + if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { + // This may be wrong to do in some cases + defp->RemoveOptValues(g); + strcpy(g->Message, MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } // endif block // We have to use local variables because Txfp->CurBlk is set // to Rows+1 by unblocked variable length table access methods. @@ -973,13 +982,14 @@ bool TDBDOS::GetBlockValues(PGLOBAL g) PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PCATLG cat = defp->GetCat(); + PDBUSER dup = PlgGetUser(g); #if 0 if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) return false; #endif // __WIN__ - if (defp->Optimized) + if (defp->Optimized || !(dup->Check & CHK_OPT)) return false; // Already done or to be redone if (Ftype == RECFM_VAR || defp->Compressed == 2) { diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index 623adcfed0d..4c8eb438a26 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -28,7 +28,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class TDBFIX; friend class TXTFAM; friend class DBFBASE; - friend class ZIPFAM; + friend class ZIPUTIL; public: // Constructor DOSDEF(void); @@ -41,7 +41,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ virtual bool IsHuge(void) {return Huge;} PSZ GetFn(void) {return Fn;} PSZ GetOfn(void) {return Ofn;} - void SetBlock(int block) {Block = block;} + PSZ GetEntry(void) {return Entry;} + bool GetMul(void) {return Mulentries;} + void SetBlock(int block) {Block = block;} int GetBlock(void) {return Block;} int GetLast(void) {return Last;} void SetLast(int last) {Last = last;} @@ -58,9 +60,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int *GetTo_Pos(void) {return To_Pos;} // Methods - virtual int Indexable(void) - {return (!Multiple && !Zipped && Compressed != 1) ? 1 : 0;} - virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); + virtual int Indexable(void) + {return (!Multiple && !Mulentries && Compressed != 1) ? 1 : 0;} + virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); bool InvalidateIndex(PGLOBAL g); @@ -78,6 +80,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ bool Zipped; /* true for zipped table file */ + bool Mulentries; /* true for multiple entries */ bool Padded; /* true for padded table file */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted */ diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 2c4d605e66c..b24375443f6 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -177,9 +177,14 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); - if (tdp->Zipped) - tdbp = new(g) TDBCSV(tdp, new(g) ZIPFAM(tdp)); - else + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tdbp = new(g)TDBCSV(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); tdbp->SetMode(MODE_READ); diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index eff95445a3a..1b9ce8b64c9 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -127,9 +127,14 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - if (tdp->Zipped) - tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); - else + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) @@ -144,9 +149,14 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - if (tdp->Zipped) - tjnp = new(g) TDBJSN(tdp, new(g) ZIPFAM(tdp)); - else + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjnp = new(g)TDBJSN(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); tjnp->SetMode(MODE_READ); @@ -467,9 +477,14 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - if (Zipped) - txfp = new(g) ZIPFAM(this); - else + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g)ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else txfp = new(g) MAPFAM(this); tdbp = new(g) TDBJSON(this, txfp); diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 1993b07eb7a..3b8229fcf51 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -1,9 +1,9 @@ /************* Tabxml C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABXML */ /* ------------- */ -/* Version 2.8 */ +/* Version 2.9 */ /* */ -/* Author Olivier BERTRAND 2007 - 2015 */ +/* Author Olivier BERTRAND 2007 - 2016 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -159,6 +159,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) tdp->Fn = fn; tdp->Database = SetPath(g, db); tdp->Tabname = tab; + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); if (!(op = GetStringTableOption(g, topt, "Xmlsup", NULL))) #if defined(__WIN__) @@ -209,7 +211,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) while (true) { if (!vp->atp && - !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, node) : NULL)) + !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL) + : NULL)) if (j) { vp = lvlp[--j]; @@ -259,7 +262,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (j < lvl && ok) { vp = lvlp[j+1]; vp->k = 0; - vp->atp = node->GetAttribute(g, NULL); + vp->pn = node; + vp->atp = node->GetAttribute(g, NULL); vp->nl = node->GetChildElements(g); if (tdp->Usedom && vp->nl->GetLength() == 1) { @@ -270,7 +274,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (vp->atp || vp->b) { if (!vp->atp) - node = vp->nl->GetItem(g, vp->k++, node); + node = vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL); strncat(fmt, colname, XLEN(fmt)); strncat(fmt, "/", XLEN(fmt)); @@ -429,11 +433,14 @@ XMLDEF::XMLDEF(void) DefNs = NULL; Attrib = NULL; Hdattr = NULL; + Entry = NULL; Coltype = 1; Limit = 0; Header = 0; Xpand = false; Usedom = false; + Zipped = false; + Mulentries = false; } // end of XMLDEF constructor /***********************************************************************/ @@ -512,7 +519,14 @@ bool XMLDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // Get eventual table node attribute Attrib = GetStringCatInfo(g, "Attribute", NULL); Hdattr = GetStringCatInfo(g, "HeadAttr", NULL); - return false; + + // Specific for zipped files + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + + return false; } // end of DefineAM /***********************************************************************/ @@ -552,6 +566,7 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) Xfile = tdp->Fn; Enc = tdp->Encoding; Tabname = tdp->Tabname; +#if 0 // why all these? Rowname = (tdp->Rowname) ? tdp->Rowname : NULL; Colname = (tdp->Colname) ? tdp->Colname : NULL; Mulnode = (tdp->Mulnode) ? tdp->Mulnode : NULL; @@ -560,10 +575,22 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) DefNs = (tdp->DefNs) ? tdp->DefNs : NULL; Attrib = (tdp->Attrib) ? tdp->Attrib : NULL; Hdattr = (tdp->Hdattr) ? tdp->Hdattr : NULL; - Coltype = tdp->Coltype; +#endif // 0 + Rowname = tdp->Rowname; + Colname = tdp->Colname; + Mulnode = tdp->Mulnode; + XmlDB = tdp->XmlDB; + Nslist = tdp->Nslist; + DefNs = tdp->DefNs; + Attrib = tdp->Attrib; + Hdattr = tdp->Hdattr; + Entry = tdp->Entry; + Coltype = tdp->Coltype; Limit = tdp->Limit; Xpand = tdp->Xpand; - Changed = false; + Zipped = tdp->Zipped; + Mulentries = tdp->Mulentries; + Changed = false; Checked = false; NextSame = false; NewRow = false; @@ -605,10 +632,13 @@ TDBXML::TDBXML(PTDBXML tdbp) : TDBASE(tdbp) DefNs = tdbp->DefNs; Attrib = tdbp->Attrib; Hdattr = tdbp->Hdattr; - Coltype = tdbp->Coltype; + Entry = tdbp->Entry; + Coltype = tdbp->Coltype; Limit = tdbp->Limit; Xpand = tdbp->Xpand; - Changed = tdbp->Changed; + Zipped = tdbp->Zipped; + Mulentries = tdbp->Mulentries; + Changed = tdbp->Changed; Checked = tdbp->Checked; NextSame = tdbp->NextSame; NewRow = tdbp->NewRow; @@ -686,7 +716,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) /*********************************************************************/ /* Firstly we check whether this file have been already loaded. */ /*********************************************************************/ - if (Mode == MODE_READ || Mode == MODE_ANY) + if ((Mode == MODE_READ || Mode == MODE_ANY) && !Zipped) for (fp = dup->Openlist; fp; fp = fp->Next) if (fp->Type == type && fp->Length && fp->Count) if (!stricmp(fp->Fname, filename)) @@ -708,7 +738,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) return RC_FX; // Initialize the implementation - if (Docp->Initialize(g)) { + if (Docp->Initialize(g, Entry, Zipped)) { sprintf(g->Message, MSG(INIT_FAILED), (Usedom) ? "DOM" : "libxml2"); return RC_FX; } // endif init @@ -717,7 +747,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) htrc("TDBXML: parsing %s rc=%d\n", filename, rc); // Parse the XML file - if (Docp->ParseFile(filename)) { + if (Docp->ParseFile(g, filename)) { // Does the file exist? int h= global_open(g, MSGID_NONE, filename, _O_RDONLY); diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h index 7ba3166881d..6c586d79dec 100644 --- a/storage/connect/tabxml.h +++ b/storage/connect/tabxml.h @@ -1,7 +1,7 @@ /*************** Tabxml H Declares Source Code File (.H) ***************/ -/* Name: TABXML.H Version 1.6 */ +/* Name: TABXML.H Version 1.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2007-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2007-2016 */ /* */ /* This file contains the XML table classes declares. */ /***********************************************************************/ @@ -42,12 +42,15 @@ class DllExport XMLDEF : public TABDEF { /* Logical table description */ char *DefNs; /* Dummy name of default namespace */ char *Attrib; /* Table node attributes */ char *Hdattr; /* Header node attributes */ - int Coltype; /* Default column type */ + char *Entry; /* Zip entry name or pattern */ + int Coltype; /* Default column type */ int Limit; /* Limit of multiple values */ int Header; /* n first rows are header rows */ bool Xpand; /* Put multiple tags in several rows */ bool Usedom; /* True: DOM, False: libxml2 */ - }; // end of XMLDEF + bool Zipped; /* True: Zipped XML file(s) */ + bool Mulentries; /* True: multiple entries in zip file*/ +}; // end of XMLDEF #if defined(INCLUDE_TDBXML) /***********************************************************************/ @@ -122,7 +125,9 @@ class DllExport TDBXML : public TDBASE { bool Bufdone; // True when column buffers allocated bool Nodedone; // True when column nodes allocated bool Void; // True if the file does not exist - char *Xfile; // The XML file + bool Zipped; // True if Zipped XML file(s) + bool Mulentries; // True if multiple entries in zip file + char *Xfile; // The XML file char *Enc; // New XML table file encoding char *Tabname; // Name of Table node char *Rowname; // Name of first level nodes @@ -133,7 +138,8 @@ class DllExport TDBXML : public TDBASE { char *DefNs; // Dummy name of default namespace char *Attrib; // Table node attribut(s) char *Hdattr; // Header node attribut(s) - int Coltype; // Default column type + char *Entry; // Zip entry name or pattern + int Coltype; // Default column type int Limit; // Limit of multiple values int Header; // n first rows are header rows int Multiple; // If multiple files -- cgit v1.2.1 From 43147681503299ccdf31e23e84dd39aeff52b2df Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 25 Dec 2016 12:32:05 +0100 Subject: Modified version number --- storage/connect/ha_connect.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 6590902bcd4..33f02d8338c 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -6955,7 +6955,7 @@ maria_declare_plugin(connect) 0x0104, /* version number (1.04) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.04.0008", /* string version */ + "1.05.0001", /* string version */ MariaDB_PLUGIN_MATURITY_GAMMA /* maturity */ } maria_declare_plugin_end; -- cgit v1.2.1 From 6ac84d9824ec384c4489b68b8087369aef147ff9 Mon Sep 17 00:00:00 2001 From: vicentiu Date: Sat, 7 Jan 2017 14:24:42 +0200 Subject: 5.6.35 --- storage/innobase/dict/dict0stats.cc | 13 +++-- storage/innobase/fts/fts0opt.cc | 7 +-- storage/innobase/handler/ha_innodb.cc | 38 +++++++++++++++ storage/innobase/handler/handler0alter.cc | 79 +++++++++++++++++-------------- storage/innobase/include/os0thread.h | 13 ++++- storage/innobase/include/srv0srv.h | 1 + storage/innobase/mach/mach0data.cc | 53 ++++++++++++++++----- storage/innobase/os/os0thread.cc | 32 +++++++++++-- storage/innobase/row/row0ftsort.cc | 2 +- storage/innobase/row/row0merge.cc | 7 +++ storage/innobase/row/row0mysql.cc | 2 + storage/innobase/srv/srv0srv.cc | 1 + 12 files changed, 188 insertions(+), 60 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 9aa63caa579..b0ba98308be 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2009, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1095,7 +1095,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1281,8 +1282,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1558,6 +1563,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 0d45a195c95..19098dc00ef 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -578,9 +578,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -655,10 +652,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -669,7 +665,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index be5e74e1617..41c767a4bfc 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13359,6 +13359,37 @@ ha_innobase::get_auto_increment( ulonglong col_max_value = innobase_get_int_col_max_value( table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { @@ -15880,6 +15911,12 @@ static MYSQL_SYSVAR_BOOL(doublewrite, innobase_use_doublewrite, "Disable with --skip-innodb-doublewrite.", NULL, NULL, TRUE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Scan delete marked records for persistent stat", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -16681,6 +16718,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(data_file_path), MYSQL_SYSVAR(data_home_dir), MYSQL_SYSVAR(doublewrite), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 2261754a4f5..e772208ed72 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1709,6 +1709,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -1775,26 +1776,11 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); - - if (maybe_null) { - new_primary = false; - break; - } - } + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id @@ -1812,8 +1798,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -2900,6 +2892,8 @@ prepare_inplace_alter_table_dict( ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); if (!ctx->new_table->fts && innobase_fulltext_exist(altered_table)) { @@ -3892,6 +3886,27 @@ found_col: add_fts_doc_id_idx)); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} + /** Alter the table structure in-place with operations specified using Alter_inplace_info. The level of concurrency allowed during this operation depends @@ -4009,17 +4024,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; default: my_error_innodb(error, @@ -4829,7 +4840,6 @@ innobase_update_foreign_cache( "Foreign key constraints for table '%s'" " are loaded with charset check off", user_table->name); - } } @@ -4929,14 +4939,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); diff --git a/storage/innobase/include/os0thread.h b/storage/innobase/include/os0thread.h index 9a1ada8fa0d..54f3d7554bf 100644 --- a/storage/innobase/include/os0thread.h +++ b/storage/innobase/include/os0thread.h @@ -117,14 +117,25 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread); + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach = true) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** Returns the thread identifier of current thread. diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index aaea21b264b..042ced75d10 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -347,6 +347,7 @@ extern unsigned long long srv_stats_transient_sample_pages; extern my_bool srv_stats_persistent; extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; +extern my_bool srv_stats_include_delete_marked; extern ibool srv_use_doublewrite_buf; extern ulong srv_doublewrite_batch_size; diff --git a/storage/innobase/mach/mach0data.cc b/storage/innobase/mach/mach0data.cc index df68aab8a18..feeedb01609 100644 --- a/storage/innobase/mach/mach0data.cc +++ b/storage/innobase/mach/mach0data.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,8 +55,22 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xC0UL) { + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } @@ -64,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xE0UL) { + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -73,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -81,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index 772336215c9..d6f897ca46a 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -191,14 +191,38 @@ os_thread_create_func( #endif } +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread) +{ +#ifdef __WIN__ + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed. */ + ut_ad(ret == 0 || ret == ESRCH); +#endif /* __WIN__ */ +} + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ { #ifdef UNIV_DEBUG_THREAD_CREATION fprintf(stderr, "Thread exits, id %lu\n", @@ -216,7 +240,9 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - pthread_detach(pthread_self()); + if (detach) { + pthread_detach(pthread_self()); + } pthread_exit(exit_value); #endif } diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 4fd3a51040a..7d3fcf675ec 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -957,7 +957,7 @@ fts_parallel_merge( CloseHandle(psort_info->thread_hdl); #endif /*__WIN__ */ - os_thread_exit(NULL); + os_thread_exit(NULL, false); OS_THREAD_DUMMY_RETURN; } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index c094be8a23b..f8bea67906c 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3774,6 +3774,13 @@ wait_again: " exited when creating FTS" " index '%s'", indexes[i]->name); + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 11bef1064d6..09c20911ec9 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1362,6 +1362,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 7a7783b962c..a67f3a776c5 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -340,6 +340,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ UNIV_INTERN unsigned long long srv_stats_transient_sample_pages = 8; UNIV_INTERN my_bool srv_stats_persistent = TRUE; +UNIV_INTERN my_bool srv_stats_include_delete_marked = FALSE; UNIV_INTERN unsigned long long srv_stats_persistent_sample_pages = 20; UNIV_INTERN my_bool srv_stats_auto_recalc = TRUE; -- cgit v1.2.1 From c33db2cdc0ab70a874060d58710895f6dac3dea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sat, 7 Jan 2017 15:53:37 +0200 Subject: 5.6.35 --- storage/perfschema/pfs.cc | 34 +++++++-- storage/perfschema/pfs_digest.cc | 118 +++++++++++++++++------------ storage/perfschema/pfs_digest.h | 5 +- storage/perfschema/pfs_lock.h | 14 +++- storage/perfschema/table_esms_by_digest.cc | 22 ++++-- 5 files changed, 129 insertions(+), 64 deletions(-) (limited to 'storage') diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index cf0fa82ba3f..3bbc0a4602e 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2560,10 +2560,7 @@ start_table_io_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_io_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; - - DBUG_ASSERT(pfs_thread == - my_pthread_getspecific_ptr(PFS_thread*, THR_PFS)); + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); register uint flags; ulonglong timer_start= 0; @@ -2666,7 +2663,7 @@ start_table_lock_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_lock_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); PFS_TL_LOCK_TYPE lock_type; @@ -3068,7 +3065,12 @@ start_socket_wait_v1(PSI_socket_locker_state *state, if (flag_thread_instrumentation) { - PFS_thread *pfs_thread= pfs_socket->m_thread_owner; + /* + Do not use pfs_socket->m_thread_owner here, + as different threads may use concurrently the same socket, + for example during a KILL. + */ + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); if (unlikely(pfs_thread == NULL)) return NULL; @@ -3436,6 +3438,8 @@ static void end_idle_wait_v1(PSI_idle_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3517,6 +3521,8 @@ static void end_mutex_wait_v1(PSI_mutex_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3596,6 +3602,8 @@ static void end_rwlock_rdwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3668,6 +3676,8 @@ static void end_rwlock_wrwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3732,6 +3742,8 @@ static void end_cond_wait_v1(PSI_cond_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3826,6 +3838,8 @@ static void end_table_io_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3895,6 +3909,8 @@ static void end_table_lock_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -4143,6 +4159,8 @@ static void end_file_wait_v1(PSI_file_locker *locker, if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -5070,6 +5088,8 @@ static void end_socket_wait_v1(PSI_socket_locker *locker, size_t byte_count) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index 1053bd59571..5886c379b2f 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -volatile uint32 digest_index; +volatile uint32 PFS_ALIGNED digest_monotonic_index; bool digest_full= false; LF_HASH digest_hash; @@ -63,7 +63,7 @@ int init_digest(const PFS_global_param *param) */ digest_max= param->m_digest_sizing; digest_lost= 0; - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; if (digest_max == 0) @@ -105,6 +105,9 @@ int init_digest(const PFS_global_param *param) + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Set record[0] as allocated. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + return 0; } @@ -207,9 +210,10 @@ find_or_create_digest(PFS_thread *thread, memcpy(hash_key.m_schema_name, schema_name, schema_name_length); int res; - ulong safe_index; uint retry_count= 0; const uint retry_max= 3; + size_t safe_index; + size_t attempts= 0; PFS_statements_digest_stat **entry; PFS_statements_digest_stat *pfs= NULL; @@ -245,55 +249,70 @@ search: return & pfs->m_stat; } - safe_index= PFS_atomic::add_u32(& digest_index, 1); - if (safe_index >= digest_max) + while (++attempts <= digest_max) { - /* The digest array is now full. */ - digest_full= true; - pfs= &statements_digest_stat_array[0]; - - if (pfs->m_first_seen == 0) - pfs->m_first_seen= now; - pfs->m_last_seen= now; - return & pfs->m_stat; - } - - /* Add a new record in digest stat array. */ - pfs= &statements_digest_stat_array[safe_index]; - - /* Copy digest hash/LF Hash search key. */ - memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); - - /* - Copy digest storage to statement_digest_stat_array so that it could be - used later to generate digest text. - */ - pfs->m_digest_storage.copy(digest_storage); - - pfs->m_first_seen= now; - pfs->m_last_seen= now; + safe_index= PFS_atomic::add_u32(& digest_monotonic_index, 1) % digest_max; + if (safe_index == 0) + { + /* Record [0] is reserved. */ + safe_index= 1; + } - res= lf_hash_insert(&digest_hash, pins, &pfs); - if (likely(res == 0)) - { - return & pfs->m_stat; - } + /* Add a new record in digest stat array. */ + pfs= &statements_digest_stat_array[safe_index]; - if (res > 0) - { - /* Duplicate insert by another thread */ - if (++retry_count > retry_max) + if (pfs->m_lock.is_free()) { - /* Avoid infinite loops */ - digest_lost++; - return NULL; + if (pfs->m_lock.free_to_dirty()) + { + /* Copy digest hash/LF Hash search key. */ + memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); + + /* + Copy digest storage to statement_digest_stat_array so that it could be + used later to generate digest text. + */ + pfs->m_digest_storage.copy(digest_storage); + + pfs->m_first_seen= now; + pfs->m_last_seen= now; + + res= lf_hash_insert(&digest_hash, pins, &pfs); + if (likely(res == 0)) + { + pfs->m_lock.dirty_to_allocated(); + return & pfs->m_stat; + } + + pfs->m_lock.dirty_to_free(); + + if (res > 0) + { + /* Duplicate insert by another thread */ + if (++retry_count > retry_max) + { + /* Avoid infinite loops */ + digest_lost++; + return NULL; + } + goto search; + } + + /* OOM in lf_hash_insert */ + digest_lost++; + return NULL; + } } - goto search; } - /* OOM in lf_hash_insert */ - digest_lost++; - return NULL; + /* The digest array is now full. */ + digest_full= true; + pfs= &statements_digest_stat_array[0]; + + if (pfs->m_first_seen == 0) + pfs->m_first_seen= now; + pfs->m_last_seen= now; + return & pfs->m_stat; } void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) @@ -320,10 +339,12 @@ void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) void PFS_statements_digest_stat::reset_data(unsigned char *token_array, uint length) { + m_lock.set_dirty(); m_digest_storage.reset(token_array, length); m_stat.reset(); m_first_seen= 0; m_last_seen= 0; + m_lock.dirty_to_free(); } void PFS_statements_digest_stat::reset_index(PFS_thread *thread) @@ -351,11 +372,14 @@ void reset_esms_by_digest() statements_digest_stat_array[index].reset_data(statements_digest_token_array + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Mark record[0] as allocated again. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + /* Reset index which indicates where the next calculated digest information to be inserted in statements_digest_stat_array. */ - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; } diff --git a/storage/perfschema/pfs_digest.h b/storage/perfschema/pfs_digest.h index 76d6c33d984..429a9f4250a 100644 --- a/storage/perfschema/pfs_digest.h +++ b/storage/perfschema/pfs_digest.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ struct PFS_digest_key /** A statement digest stat record. */ struct PFS_ALIGNED PFS_statements_digest_stat { + /** Internal lock. */ + pfs_lock m_lock; + /** Digest Schema + MD5 Hash. */ PFS_digest_key m_digest_key; diff --git a/storage/perfschema/pfs_lock.h b/storage/perfschema/pfs_lock.h index c429d934702..339a893c833 100644 --- a/storage/perfschema/pfs_lock.h +++ b/storage/perfschema/pfs_lock.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -155,6 +155,18 @@ struct pfs_lock PFS_atomic::store_u32(&m_version_state, new_val); } + /** + Initialize a lock to dirty. + */ + void set_dirty(void) + { + /* Do not set the version to 0, read the previous value. */ + uint32 copy= PFS_atomic::load_u32(&m_version_state); + /* Increment the version, set the DIRTY state */ + uint32 new_val= (copy & VERSION_MASK) + VERSION_INC + PFS_LOCK_DIRTY; + PFS_atomic::store_u32(&m_version_state, new_val); + } + /** Execute a dirty to free transition. This transition should be executed by the writer that owns the record. diff --git a/storage/perfschema/table_esms_by_digest.cc b/storage/perfschema/table_esms_by_digest.cc index 99e24316cbb..002a7f0104b 100644 --- a/storage/perfschema/table_esms_by_digest.cc +++ b/storage/perfschema/table_esms_by_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -238,11 +238,14 @@ int table_esms_by_digest::rnd_next(void) m_pos.next()) { digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - m_next_pos.set_after(&m_pos); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + m_next_pos.set_after(&m_pos); + return 0; + } } } @@ -260,10 +263,13 @@ table_esms_by_digest::rnd_pos(const void *pos) set_position(pos); digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + return 0; + } } return HA_ERR_RECORD_DELETED; -- cgit v1.2.1 From ecdb39a9f51ebbdafae167889ec65ee817e07de6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 10 Jan 2017 12:08:36 +0200 Subject: Fix problems from 5.5 merge * Update mysqld_safe script to remove duplicated parameter --crash-script * Make --core-file-size accept underscores as well as dashes correctly. * Add mysqld_safe_helper to Debian and Ubuntu files. * Update innodb minor version to 35 --- storage/innobase/include/univ.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index e6ddf45faba..e4b20441749 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -44,7 +44,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 33 +#define INNODB_VERSION_BUGFIX 35 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; -- cgit v1.2.1 From 4799af092574e7957d7143c7751acef74a95a495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 10 Jan 2017 14:20:43 +0200 Subject: Fix unit test after merge from mysql 5.5.35 perfschema The problem in MariaDB is introduced by this merge commit: c33db2cdc0ab70a874060d58710895f6dac3dea3 The merge comes from mysql and the original author comes from this commit from MySQL: ------------------------------------------------ commit 160b823d146288d66638e4a740d6d2da72f9a689 Author: Marc Alff Date: Tue Aug 30 12:14:07 2016 +0200 Bug#22551677 SIGNAL 11 IN LF_PINBOX_PUT_PINS Backport to 5.6 ------------------------------------------------ The breaking change is in start_socket_wait_v1 where instead of using m_thread_owner, we make use of my_pthread_getspecific_ptr to fetch a thread local storage value. Unfortunately this invalidates the "m_thread_owner" member when a socket is created. The internals of the socket structure have m_thread_owner set to NULL, but when checking for ownership we actually look at the current thread's key store. This seems incorrect however it is not immediately apparent why. To not diverge from MySQL's reasoning as it is not described what the actual problem was that this commit is trying to fix, I have adjusted the unittest to account for this new behaviour. We destroy the current thread in the unit test, such that the newly created socket actually has no thread owner. The m_thread_owner is untouched in all this. --- storage/perfschema/unittest/pfs-t.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'storage') diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 6121fac098f..f76b1aa2e75 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1316,6 +1316,7 @@ void test_locker_disabled() /* Pretend the socket does not have a thread owner */ /* ---------------------------------------------- */ + psi->delete_current_thread(); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); ok(socket_A1 != NULL, "instrumented"); -- cgit v1.2.1 From a9d00db15559749e54fd9d55cf0ff90f83604e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sun, 15 Jan 2017 14:20:16 +0200 Subject: MDEV-11799 InnoDB can abort if the doublewrite buffer contains a bad and a good copy Clean up the InnoDB doublewrite buffer code. buf_dblwr_init_or_load_pages(): Do not add empty pages to the buffer. buf_dblwr_process(): Do consider changes to pages that are all zero. Do not abort when finding a corrupted copy of a page in the doublewrite buffer, because there could be multiple copies in the doublewrite buffer, and only one of them needs to be good. --- storage/innobase/buf/buf0dblwr.cc | 222 ++++++++++++++++---------------------- storage/xtradb/buf/buf0dblwr.cc | 221 ++++++++++++++++--------------------- 2 files changed, 188 insertions(+), 255 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 1cf856a731f..0b42e38a655 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -455,8 +455,11 @@ buf_dblwr_init_or_load_pages( os_file_write(path, file, page, source_page_no * UNIV_PAGE_SIZE, UNIV_PAGE_SIZE); - } else if (load_corrupt_pages) { - + } else if (load_corrupt_pages + && !buf_page_is_zeroes(page, FIL_PAGE_DATA)) { + /* Each valid page header must contain some + nonzero bytes, such as FIL_PAGE_OFFSET + or FIL_PAGE_LSN. */ recv_dblwr.add(page); } @@ -492,8 +495,6 @@ buf_dblwr_process() for (std::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { - bool is_compressed = false; - page = *i; page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); space_id = mach_read_from_4(page + FIL_PAGE_SPACE_ID); @@ -501,139 +502,104 @@ buf_dblwr_process() if (!fil_tablespace_exists_in_mem(space_id)) { /* Maybe we have dropped the single-table tablespace and this page once belonged to it: do nothing */ + continue; + } - } else if (!fil_check_adress_in_tablespace(space_id, - page_no)) { + if (!fil_check_adress_in_tablespace(space_id, page_no)) { ib_logf(IB_LOG_LEVEL_WARN, - "A page in the doublewrite buffer is not " - "within space bounds; space id %lu " - "page number %lu, page %lu in " - "doublewrite buf.", - (ulong) space_id, (ulong) page_no, - page_no_dblwr); + "A copy of page " ULINTPF ":" ULINTPF + " in the doublewrite buffer slot " ULINTPF + " is not within space bounds", + space_id, page_no, page_no_dblwr); + continue; + } + + ulint zip_size = fil_space_get_zip_size(space_id); + ut_ad(!buf_page_is_zeroes(page, zip_size)); + + /* Read in the actual page from the file */ + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0); + + const bool is_all_zero = buf_page_is_zeroes( + read_buf, zip_size); + + if (is_all_zero) { + /* We will check if the copy in the + doublewrite buffer is valid. If not, we will + ignore this page (there should be redo log + records to initialize it). */ } else { - ulint zip_size = fil_space_get_zip_size(space_id); - - /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, - NULL, - 0); - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(read_buf) | - fil_page_is_compressed(read_buf); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true); + if (fil_page_is_compressed_encrypted(read_buf) || + fil_page_is_compressed(read_buf)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, read_buf, UNIV_PAGE_SIZE, + NULL, true); } - if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { - /* page is encrypted and checksum is OK */ - } else if (buf_page_is_corrupted(true, read_buf, zip_size)) { - - fprintf(stderr, - "InnoDB: Warning: database page" - " corruption or a failed\n" - "InnoDB: file read of" - " space %lu page %lu.\n" - "InnoDB: Trying to recover it from" - " the doublewrite buffer.\n", - (ulong) space_id, (ulong) page_no); - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(page) | - fil_page_is_compressed(page); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, NULL, true); - } + if (fil_space_verify_crypt_checksum( + read_buf, zip_size) + || !buf_page_is_corrupted( + true, read_buf, zip_size)) { + /* The page is good; there is no need + to consult the doublewrite buffer. */ + continue; + } - if (fil_space_verify_crypt_checksum(page, zip_size)) { - /* the doublewrite buffer page is encrypted and OK */ - } else if (buf_page_is_corrupted(true, - page, - zip_size)) { - fprintf(stderr, - "InnoDB: Dump of the page:\n"); - buf_page_print( - read_buf, zip_size, - BUF_PAGE_PRINT_NO_CRASH); - fprintf(stderr, - "InnoDB: Dump of" - " corresponding page" - " in doublewrite buffer:\n"); - buf_page_print( - page, zip_size, - BUF_PAGE_PRINT_NO_CRASH); - - fprintf(stderr, - "InnoDB: Also the page in the" - " doublewrite buffer" - " is corrupt.\n" - "InnoDB: Cannot continue" - " operation.\n" - "InnoDB: You can try to" - " recover the database" - " with the my.cnf\n" - "InnoDB: option:\n" - "InnoDB:" - " innodb_force_recovery=6\n"); - ut_error; - } + /* We intentionally skip this message for + is_all_zero pages. */ + ib_logf(IB_LOG_LEVEL_INFO, + "Trying to recover page " ULINTPF ":" ULINTPF + " from the doublewrite buffer.", + space_id, page_no); + } - /* Write the good page from the - doublewrite buffer to the intended - position */ - - fil_io(OS_FILE_WRITE, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, - NULL, - 0); - - ib_logf(IB_LOG_LEVEL_INFO, - "Recovered the page from" - " the doublewrite buffer."); - - } else if (buf_page_is_zeroes(read_buf, zip_size)) { - - if (!buf_page_is_zeroes(page, zip_size) - && !buf_page_is_corrupted(true, page, - zip_size)) { - - /* Database page contained only - zeroes, while a valid copy is - available in dblwr buffer. */ - - fil_io(OS_FILE_WRITE, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, - NULL, - 0); - } + /* Next, validate the doublewrite page. */ + if (fil_page_is_compressed_encrypted(page) || + fil_page_is_compressed(page)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, page, UNIV_PAGE_SIZE, NULL, true); + } + + if (!fil_space_verify_crypt_checksum(page, zip_size) + && buf_page_is_corrupted(true, page, zip_size)) { + if (!is_all_zero) { + ib_logf(IB_LOG_LEVEL_WARN, + "A doublewrite copy of page " + ULINTPF ":" ULINTPF " is corrupted.", + space_id, page_no); } + /* Theoretically we could have another good + copy for this page in the doublewrite + buffer. If not, we will report a fatal error + for a corrupted page somewhere else if that + page was truly needed. */ + continue; } + + /* Write the good page from the doublewrite buffer to + the intended position. */ + + fil_io(OS_FILE_WRITE, true, space_id, zip_size, page_no, 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, NULL, 0); + + ib_logf(IB_LOG_LEVEL_INFO, + "Recovered page " ULINTPF ":" ULINTPF " from" + " the doublewrite buffer.", + space_id, page_no); } ut_free(unaligned_read_buf); diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 7f6b6caee9d..9c7eea6410a 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -455,8 +455,11 @@ buf_dblwr_init_or_load_pages( os_file_write(path, file, page, source_page_no * UNIV_PAGE_SIZE, UNIV_PAGE_SIZE); - } else if (load_corrupt_pages) { - + } else if (load_corrupt_pages + && !buf_page_is_zeroes(page, FIL_PAGE_DATA)) { + /* Each valid page header must contain some + nonzero bytes, such as FIL_PAGE_OFFSET + or FIL_PAGE_LSN. */ recv_dblwr.add(page); } @@ -492,8 +495,6 @@ buf_dblwr_process() for (std::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { - bool is_compressed = false; - page = *i; page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); space_id = mach_read_from_4(page + FIL_PAGE_SPACE_ID); @@ -501,138 +502,104 @@ buf_dblwr_process() if (!fil_tablespace_exists_in_mem(space_id)) { /* Maybe we have dropped the single-table tablespace and this page once belonged to it: do nothing */ + continue; + } - } else if (!fil_check_adress_in_tablespace(space_id, - page_no)) { + if (!fil_check_adress_in_tablespace(space_id, page_no)) { ib_logf(IB_LOG_LEVEL_WARN, - "A page in the doublewrite buffer is not " - "within space bounds; space id %lu " - "page number %lu, page %lu in " - "doublewrite buf.", - (ulong) space_id, (ulong) page_no, - page_no_dblwr); + "A copy of page " ULINTPF ":" ULINTPF + " in the doublewrite buffer slot " ULINTPF + " is not within space bounds", + space_id, page_no, page_no_dblwr); + continue; + } + + ulint zip_size = fil_space_get_zip_size(space_id); + ut_ad(!buf_page_is_zeroes(page, zip_size)); + + /* Read in the actual page from the file */ + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0); + + const bool is_all_zero = buf_page_is_zeroes( + read_buf, zip_size); + + if (is_all_zero) { + /* We will check if the copy in the + doublewrite buffer is valid. If not, we will + ignore this page (there should be redo log + records to initialize it). */ } else { - ulint zip_size = fil_space_get_zip_size(space_id); - - /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, - NULL, - 0); - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(read_buf) | - fil_page_is_compressed(read_buf); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true); + if (fil_page_is_compressed_encrypted(read_buf) || + fil_page_is_compressed(read_buf)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, read_buf, UNIV_PAGE_SIZE, + NULL, true); } - if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { - /* page is encrypted and checksum is OK */ - } else if (buf_page_is_corrupted(true, read_buf, zip_size)) { - - fprintf(stderr, - "InnoDB: Database page" - " corruption or a failed\n" - "InnoDB: file read of" - " space %lu page %lu.\n" - "InnoDB: Trying to recover it from" - " the doublewrite buffer.\n", - (ulong) space_id, (ulong) page_no); - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(page) | - fil_page_is_compressed(page); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, NULL, true); - } + if (fil_space_verify_crypt_checksum( + read_buf, zip_size) + || !buf_page_is_corrupted( + true, read_buf, zip_size)) { + /* The page is good; there is no need + to consult the doublewrite buffer. */ + continue; + } - if (fil_space_verify_crypt_checksum(page, zip_size)) { - /* the doublewrite buffer page is encrypted and OK */ - } else if (buf_page_is_corrupted(true, - page, - zip_size)) { - fprintf(stderr, - "InnoDB: Dump of the page:\n"); - buf_page_print( - read_buf, zip_size, - BUF_PAGE_PRINT_NO_CRASH); - fprintf(stderr, - "InnoDB: Dump of" - " corresponding page" - " in doublewrite buffer:\n"); - buf_page_print( - page, zip_size, - BUF_PAGE_PRINT_NO_CRASH); - - fprintf(stderr, - "InnoDB: Also the page in the" - " doublewrite buffer" - " is corrupt.\n" - "InnoDB: Cannot continue" - " operation.\n" - "InnoDB: You can try to" - " recover the database" - " with the my.cnf\n" - "InnoDB: option:\n" - "InnoDB:" - " innodb_force_recovery=6\n"); - ut_error; - } + /* We intentionally skip this message for + is_all_zero pages. */ + ib_logf(IB_LOG_LEVEL_INFO, + "Trying to recover page " ULINTPF ":" ULINTPF + " from the doublewrite buffer.", + space_id, page_no); + } - /* Write the good page from the - doublewrite buffer to the intended - position */ - - fil_io(OS_FILE_WRITE, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, - NULL, - 0); - - ib_logf(IB_LOG_LEVEL_INFO, - "Recovered the page from" - " the doublewrite buffer."); - - } else if (buf_page_is_zeroes(read_buf, zip_size)) { - - if (!buf_page_is_zeroes(page, zip_size) - && !buf_page_is_corrupted(true, page, - zip_size)) { - - /* Database page contained only - zeroes, while a valid copy is - available in dblwr buffer. */ - - fil_io(OS_FILE_WRITE, - true, - space_id, - zip_size, - page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, - NULL, - 0); - } + /* Next, validate the doublewrite page. */ + if (fil_page_is_compressed_encrypted(page) || + fil_page_is_compressed(page)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, page, UNIV_PAGE_SIZE, NULL, true); + } + + if (!fil_space_verify_crypt_checksum(page, zip_size) + && buf_page_is_corrupted(true, page, zip_size)) { + if (!is_all_zero) { + ib_logf(IB_LOG_LEVEL_WARN, + "A doublewrite copy of page " + ULINTPF ":" ULINTPF " is corrupted.", + space_id, page_no); } + /* Theoretically we could have another good + copy for this page in the doublewrite + buffer. If not, we will report a fatal error + for a corrupted page somewhere else if that + page was truly needed. */ + continue; } + + /* Write the good page from the doublewrite buffer to + the intended position. */ + + fil_io(OS_FILE_WRITE, true, space_id, zip_size, page_no, 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, NULL, 0); + + ib_logf(IB_LOG_LEVEL_INFO, + "Recovered page " ULINTPF ":" ULINTPF " from" + " the doublewrite buffer.", + space_id, page_no); } ut_free(unaligned_read_buf); -- cgit v1.2.1 From ab1e6fefd869242d962cb91a006f37bb9ad534a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 14 Jan 2017 00:13:16 +0200 Subject: MDEV-11623 MariaDB 10.1 fails to start datadir created with MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K The storage format of FSP_SPACE_FLAGS was accidentally broken already in MariaDB 10.1.0. This fix is bringing the format in line with other MySQL and MariaDB release series. Please refer to the comments that were added to fsp0fsp.h for details. This is an INCOMPATIBLE CHANGE that affects users of page_compression and non-default innodb_page_size. Upgrading to this release will correct the flags in the data files. If you want to downgrade to earlier MariaDB 10.1.x, please refer to the test innodb.101_compatibility how to reset the FSP_SPACE_FLAGS in the files. NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret uncompressed data files with innodb_page_size=4k or 64k as compressed innodb_page_size=16k files, and then probably fail when trying to access the pages. See the comments in the function fsp_flags_convert_from_101() for detailed analysis. Move PAGE_COMPRESSION to FSP_SPACE_FLAGS bit position 16. In this way, compressed innodb_page_size=16k tablespaces will not be mistaken for uncompressed ones by MariaDB 10.1.0 to 10.1.20. Derive PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR from the dict_table_t::flags when the table is available, in fil_space_for_table_exists_in_mem() or fil_open_single_table_tablespace(). During crash recovery, fil_load_single_table_tablespace() will use innodb_compression_level for the PAGE_COMPRESSION_LEVEL. FSP_FLAGS_MEM_MASK: A bitmap of the memory-only fil_space_t::flags that are not to be written to FSP_SPACE_FLAGS. Currently, these will include PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES and DATA_DIR. Introduce the macro FSP_FLAGS_PAGE_SSIZE(). We only support one innodb_page_size for the whole instance. When creating a dummy tablespace for the redo log, use fil_space_t::flags=0. The flags are never written to the redo log files. Remove many FSP_FLAGS_SET_ macros. dict_tf_verify_flags(): Remove. This is basically only duplicating the logic of dict_tf_to_fsp_flags(), used in a debug assertion. fil_space_t::mark: Remove. This flag was not used for anything. fil_space_for_table_exists_in_mem(): Remove the unnecessary parameter mark_space, and add a parameter for table flags. Check that fil_space_t::flags match the table flags, and adjust the (memory-only) flags based on the table flags. fil_node_open_file(): Remove some redundant or unreachable conditions, do not use stderr for output, and avoid unnecessary server aborts. fil_user_tablespace_restore_page(): Convert the flags, so that the correct page_size will be used when restoring a page from the doublewrite buffer. fil_space_get_page_compressed(), fsp_flags_is_page_compressed(): Remove. It suffices to have fil_space_is_page_compressed(). FSP_FLAGS_WIDTH_DATA_DIR, FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL, FSP_FLAGS_WIDTH_ATOMIC_WRITES: Remove, because these flags do not exist in the FSP_SPACE_FLAGS but only in memory. fsp_flags_try_adjust(): New function, to adjust the FSP_SPACE_FLAGS in page 0. Called by fil_open_single_table_tablespace(), fil_space_for_table_exists_in_mem(), innobase_start_or_create_for_mysql() except if --innodb-read-only is active. fsp_flags_is_valid(ulint): Reimplement from the scratch, with accurate comments. Do not display any details of detected inconsistencies, because the output could be confusing when dealing with MariaDB 10.1.x data files. fsp_flags_convert_from_101(ulint): Convert flags from buggy MariaDB 10.1.x format, or return ULINT_UNDEFINED if the flags cannot be in MariaDB 10.1.x format. fsp_flags_match(): Check the flags when probing files. Implemented based on fsp_flags_is_valid() and fsp_flags_convert_from_101(). dict_check_tablespaces_and_store_max_id(): Do not access the page after committing the mini-transaction. IMPORT TABLESPACE fixes: AbstractCallback::init(): Convert the flags. FetchIndexRootPages::operator(): Check that the tablespace flags match the table flags. Do not attempt to convert tablespace flags to table flags, because the conversion would necessarily be lossy. PageConverter::update_header(): Write back the correct flags. This takes care of the flags in IMPORT TABLESPACE. --- storage/innobase/buf/buf0dblwr.cc | 15 + storage/innobase/dict/dict0load.cc | 20 +- storage/innobase/fil/fil0fil.cc | 396 +++++++++++------------ storage/innobase/fsp/fsp0fsp.cc | 3 +- storage/innobase/include/dict0dict.ic | 40 ++- storage/innobase/include/dict0pagecompress.h | 13 +- storage/innobase/include/dict0pagecompress.ic | 88 +----- storage/innobase/include/fil0fil.h | 41 ++- storage/innobase/include/fil0pagecompress.h | 13 +- storage/innobase/include/fsp0fsp.h | 430 +++++++++++++++++++------ storage/innobase/include/fsp0fsp.ic | 153 +-------- storage/innobase/include/fsp0pagecompress.h | 11 +- storage/innobase/include/fsp0pagecompress.ic | 28 +- storage/innobase/row/row0import.cc | 94 ++---- storage/innobase/row/row0mysql.cc | 11 +- storage/innobase/srv/srv0start.cc | 18 +- storage/xtradb/buf/buf0dblwr.cc | 15 + storage/xtradb/dict/dict0load.cc | 20 +- storage/xtradb/fil/fil0fil.cc | 395 +++++++++++------------ storage/xtradb/fsp/fsp0fsp.cc | 3 +- storage/xtradb/include/dict0dict.ic | 40 ++- storage/xtradb/include/dict0pagecompress.h | 13 +- storage/xtradb/include/dict0pagecompress.ic | 88 +----- storage/xtradb/include/fil0fil.h | 41 ++- storage/xtradb/include/fil0pagecompress.h | 13 +- storage/xtradb/include/fsp0fsp.h | 437 +++++++++++++++++++------- storage/xtradb/include/fsp0fsp.ic | 153 +-------- storage/xtradb/include/fsp0pagecompress.h | 11 +- storage/xtradb/include/fsp0pagecompress.ic | 28 +- storage/xtradb/row/row0import.cc | 94 ++---- storage/xtradb/row/row0mysql.cc | 11 +- storage/xtradb/srv/srv0start.cc | 18 +- 32 files changed, 1264 insertions(+), 1490 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 0b42e38a655..e49185f6a1f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -589,6 +589,21 @@ buf_dblwr_process() continue; } + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":0 due to invalid flags 0x%x", + space_id, int(flags)); + continue; + } + /* The flags on the page should be converted later. */ + } + /* Write the good page from the doublewrite buffer to the intended position. */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 0ab25ee1eac..0f9f4f153f5 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1052,8 +1052,6 @@ loop: btr_pcur_store_position(&pcur, &mtr); - mtr_commit(&mtr); - /* For tables created with old versions of InnoDB, SYS_TABLES.MIX_LEN may contain garbage. Such tables would always be in ROW_FORMAT=REDUNDANT. Pretend that @@ -1087,16 +1085,19 @@ loop: if (space_id == 0) { /* The system tablespace always exists. */ ut_ad(!discarded); - goto next_tablespace; + mem_free(name); + goto loop; } + mtr_commit(&mtr); + switch (dict_check) { case DICT_CHECK_ALL_LOADED: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, TRUE, !(is_temp || discarded), - false, NULL, 0) + space_id, name, !(is_temp || discarded), + false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1128,8 +1129,8 @@ loop: /* Some tablespaces may have been opened in trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, FALSE, FALSE, - false, NULL, 0)) { + space_id, name, false, + false, NULL, 0, flags)) { break; } /* fall through */ @@ -1191,7 +1192,6 @@ loop: max_space_id = space_id; } -next_tablespace: mem_free(name); mtr_start(&mtr); @@ -2382,8 +2382,8 @@ err_exit: table->ibd_file_missing = TRUE; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, FALSE, FALSE, true, heap, - table->id)) { + table->space, name, false, true, heap, + table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 1e8b2be6f01..fe6a2922b35 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -596,10 +596,7 @@ fil_node_open_file( ibool success; byte* buf2; byte* page; - ulint space_id; - ulint flags=0; ulint page_size; - ulint atomic_writes=0; ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); @@ -621,8 +618,6 @@ fil_node_open_file( /* The following call prints an error message */ os_file_get_last_error(true); - ut_print_timestamp(stderr); - ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Error: cannot " "open %s\n. InnoDB: Have you deleted .ibd " "files under a running mysqld server?\n", @@ -648,17 +643,13 @@ fil_node_open_file( ut_a(fil_is_user_tablespace_id(space->id)); if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) { - fprintf(stderr, - "InnoDB: Error: the size of single-table" - " tablespace file %s\n" - "InnoDB: is only " UINT64PF "," - " should be at least %lu!\n", - node->name, - size_bytes, - (ulong) (FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - - ut_a(0); + ib_logf(IB_LOG_LEVEL_ERROR, + "The size of the file %s is only " UINT64PF + " bytes, should be at least " ULINTPF, + node->name, size_bytes, + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + os_file_close(node->handle); + return(false); } /* Read the first page of the tablespace */ @@ -671,78 +662,34 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); srv_stats.page0_read.add(1); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - page_size = fsp_flags_get_page_size(flags); - atomic_writes = fsp_flags_get_atomic_writes(flags); - + const ulint space_id = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); ut_free(buf2); - - /* Close the file now that we have read the space id from it */ - os_file_close(node->handle); - if (UNIV_UNLIKELY(space_id != space->id)) { - fprintf(stderr, - "InnoDB: Error: tablespace id is %lu" - " in the data dictionary\n" - "InnoDB: but in file %s it is %lu!\n", - space->id, node->name, space_id); - - ut_error; - } - - if (UNIV_UNLIKELY(space_id == ULINT_UNDEFINED - || space_id == 0)) { - fprintf(stderr, - "InnoDB: Error: tablespace id %lu" - " in file %s is not sensible\n", - (ulong) space_id, node->name); - - ut_error; - } - - if (UNIV_UNLIKELY(fsp_flags_get_page_size(space->flags) - != page_size)) { - fprintf(stderr, - "InnoDB: Error: tablespace file %s" - " has page size 0x%lx\n" - "InnoDB: but the data dictionary" - " expects page size 0x%lx!\n", - node->name, flags, - fsp_flags_get_page_size(space->flags)); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Expected tablespace flags 0x%x" + " but found 0x%x in the file %s", + int(space->flags), int(flags), + node->name); + return(false); + } - ut_error; + flags = cflags; } - if (UNIV_UNLIKELY(space->flags != flags)) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - fprintf(stderr, - "InnoDB: Warning: Table flags 0x%lx" - " in the data dictionary but in file %s are 0x%lx!\n" - " Temporally corrected because DATA_DIR option to 0x%lx.\n", - space->flags, node->name, flags, space->flags); - - flags = space->flags; - } + page_size = fsp_flags_get_page_size(flags); - if (!dict_tf_verify_flags(space->flags, flags)) { - fprintf(stderr, - "InnoDB: Error: table flags are 0x%lx" - " in the data dictionary\n" - "InnoDB: but the flags in file %s are 0x%lx!\n", - space->flags, node->name, flags); - ut_error; - } + if (UNIV_UNLIKELY(space_id != space->id)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "tablespace id is " ULINTPF " in the data dictionary" + " but in file %s it is " ULINTPF "!\n", + space->id, node->name, space_id); + return(false); } if (size_bytes >= (1024*1024)) { @@ -764,7 +711,7 @@ add_size: space->size += node->size; } - atomic_writes = fsp_flags_get_atomic_writes(space->flags); + ulint atomic_writes = fsp_flags_get_atomic_writes(space->flags); /* printf("Opening file %s\n", node->name); */ @@ -1534,7 +1481,6 @@ fil_space_create( fil_system->tablespace_version++; space->tablespace_version = fil_system->tablespace_version; - space->mark = FALSE; if (purpose == FIL_TABLESPACE && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -2273,27 +2219,21 @@ fil_write_flushed_lsn_to_data_files( return(DB_SUCCESS); } -/*******************************************************************//** -Checks the consistency of the first data page of a tablespace +/** Check the consistency of the first data page of a tablespace at database startup. +@param[in] page page frame +@param[in] space_id tablespace identifier +@param[in] flags tablespace flags @retval NULL on success, or if innodb_force_recovery is set @return pointer to an error message string */ static MY_ATTRIBUTE((warn_unused_result)) const char* -fil_check_first_page( -/*=================*/ - const page_t* page) /*!< in: data page */ +fil_check_first_page(const page_t* page, ulint space_id, ulint flags) { - ulint space_id; - ulint flags; - if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); } - space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { fprintf(stderr, "InnoDB: Error: Current page size %lu != " @@ -2342,7 +2282,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ #ifdef UNIV_LOG_ARCHIVE ulint* min_arch_log_no, /*!< out: min of archived @@ -2378,12 +2318,22 @@ fil_read_first_page( *flags and *space_id as they were read from the first file and do not validate the first page. */ if (!one_read_already) { - *flags = fsp_header_get_flags(page); *space_id = fsp_header_get_space_id(page); - } + *flags = fsp_header_get_flags(page); - if (!one_read_already) { - check_msg = fil_check_first_page(page); + if (!fsp_flags_is_valid(*flags)) { + ulint cflags = fsp_flags_convert_from_101(*flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid flags 0x%x in tablespace %u", + unsigned(*flags), unsigned(*space_id)); + return "invalid tablespace flags"; + } else { + *flags = cflags; + } + } + + check_msg = fil_check_first_page(page, *space_id, *flags); } flushed_lsn = mach_read_from_8(page + @@ -2577,6 +2527,7 @@ fil_op_write_log( ulint len; log_ptr = mlog_open(mtr, 11 + 2 + 1); + ut_ad(fsp_flags_is_valid(flags)); if (!log_ptr) { /* Logging in mtr is switched off during crash recovery: @@ -3783,7 +3734,7 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3791,7 +3742,7 @@ fil_create_new_single_table_tablespace( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); if (is_temp) { /* Temporary table filepath */ @@ -3884,12 +3835,9 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, UNIV_PAGE_SIZE); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - ut_ad(fsp_flags_is_valid(flags)); if (!(fsp_flags_is_compressed(flags))) { buf_flush_init_for_writing(page, NULL, 0); @@ -3968,7 +3916,8 @@ fil_create_new_single_table_tablespace( fil_op_write_log(flags ? MLOG_FILE_CREATE2 : MLOG_FILE_CREATE, - space_id, mlog_file_flag, flags, + space_id, mlog_file_flag, + flags & ~FSP_FLAGS_MEM_MASK, tablename, NULL, &mtr); mtr_commit(&mtr); @@ -4032,6 +3981,39 @@ fil_report_bad_tablespace( (ulong) expected_id, (ulong) expected_flags); } +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, + &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib_logf(IB_LOG_LEVEL_WARN, + "adjusting FSP_SPACE_FLAGS of tablespace " + ULINTPF " from 0x%x to 0x%x", + space_id, int(f), int(flags)); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + /********************************************************************//** Tries to open a single-table tablespace and optionally checks that the space id in it is correct. If this does not succeed, print an error message @@ -4061,7 +4043,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* path_in, /*!< in: tablespace filepath */ @@ -4086,20 +4068,13 @@ fil_open_single_table_tablespace( /* Table flags can be ULINT_UNDEFINED if dict_tf_to_fsp_flags_failure is set. */ - if (flags != ULINT_UNDEFINED) { - if (!fsp_flags_is_valid(flags)) { - return(DB_CORRUPTION); - } - } else { + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); atomic_writes = fsp_flags_get_atomic_writes(flags); - /* If the tablespace was relocated, we do not - compare the DATA_DIR flag */ - ulint mod_flags = flags & ~FSP_FLAGS_MASK_DATA_DIR; - memset(&def, 0, sizeof(def)); memset(&dict, 0, sizeof(dict)); memset(&remote, 0, sizeof(remote)); @@ -4180,31 +4155,17 @@ fil_open_single_table_tablespace( &space_arch_log_no, &space_arch_log_no, #endif /* UNIV_LOG_ARCHIVE */ &def.lsn, &def.lsn, &def.crypt_data); - def.valid = !def.check_msg; if (table) { table->crypt_data = def.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - - ulint newf = def.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (def.valid && def.id == id - && newf == mod_flags) { + def.valid = !def.check_msg && def.id == id + && fsp_flags_match(flags, def.flags); + if (def.valid) { valid_tablespaces_found++; } else { - def.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( def.filepath, def.check_msg, def.id, @@ -4220,30 +4181,18 @@ fil_open_single_table_tablespace( &remote.arch_log_no, &remote.arch_log_no, #endif /* UNIV_LOG_ARCHIVE */ &remote.lsn, &remote.lsn, &remote.crypt_data); - remote.valid = !remote.check_msg; if (table) { table->crypt_data = remote.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = remote.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (remote.valid && remote.id == id - && newf == mod_flags) { + /* Validate this single-table-tablespace with SYS_TABLES. */ + remote.valid = !remote.check_msg && remote.id == id + && fsp_flags_match(flags, remote.flags); + if (remote.valid) { valid_tablespaces_found++; } else { - remote.valid = false; /* Do not use this linked tablespace. */ fil_report_bad_tablespace( remote.filepath, remote.check_msg, remote.id, @@ -4260,30 +4209,19 @@ fil_open_single_table_tablespace( &dict.arch_log_no, &dict.arch_log_no, #endif /* UNIV_LOG_ARCHIVE */ &dict.lsn, &dict.lsn, &dict.crypt_data); - dict.valid = !dict.check_msg; if (table) { table->crypt_data = dict.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = dict.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } + /* Validate this single-table-tablespace with SYS_TABLES. */ + dict.valid = !dict.check_msg && dict.id == id + && fsp_flags_match(flags, dict.flags); - if (dict.valid && dict.id == id - && newf == mod_flags) { + if (dict.valid) { valid_tablespaces_found++; } else { - dict.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( dict.filepath, dict.check_msg, dict.id, @@ -4481,6 +4419,10 @@ cleanup_and_exit: mem_free(def.filepath); + if (err == DB_SUCCESS && !srv_read_only_mode) { + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); + } + return(err); } #endif /* !UNIV_HOTBACKUP */ @@ -4662,7 +4604,23 @@ fil_user_tablespace_restore_page( goto out; } - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":" ULINTPF + " due to invalid flags 0x%x", + fsp->id, page_no, int(flags)); + err = false; + goto out; + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + zip_size = fsp_flags_get_zip_size(flags); page_size = fsp_flags_get_page_size(flags); @@ -5051,6 +5009,16 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { + fsp->flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + } + remote.flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. + That will be adjusted in fil_space_for_table_exists_in_mem(). */ + ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5342,13 +5310,12 @@ fil_report_missing_tablespace( name, space_id); } -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ @@ -5356,13 +5323,7 @@ fil_space_for_table_exists_in_mem( fil_space_create(). Either the standard 'dbname/tablename' format or table->dir_path_of_temp_table */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -5370,12 +5331,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id) /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags) /*!< in: table flags */ { fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -5387,42 +5349,31 @@ fil_space_for_table_exists_in_mem( directory path from the datadir to the file */ fnamespace = fil_space_get_by_name(name); - if (space && space == fnamespace) { - /* Found */ - - if (mark_space) { - space->mark = TRUE; - } - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since it is - not transactional. If adjust_space is set, and the mismatching - space are between a user table and its temp table, we shall - adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - char* tmp_name; + char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); - - fil_rename_tablespace(fnamespace->name, fnamespace->id, - tmp_name, NULL); - } + fil_rename_tablespace(fnamespace->name, fnamespace->id, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -5435,16 +5386,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(TRUE); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (space == NULL) { @@ -5471,10 +5418,8 @@ error_exit: fputs("InnoDB: Please refer to\n" "InnoDB: " REFMAN "innodb-troubleshooting-datadict.html\n" "InnoDB: for how to resolve the issue.\n", stderr); - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -5501,9 +5446,19 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(FALSE); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } /*******************************************************************//** @@ -7331,9 +7286,8 @@ fil_space_get_crypt_data( byte *page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, NULL, NULL); - ulint flags = fsp_header_get_flags(page); ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags), NULL); + fsp_header_get_zip_size(page), NULL); space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 87aa5f7db5c..a71d9f95ac2 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -660,6 +660,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -710,7 +711,7 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 3d2f0dff0da..800065ddaa1 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -914,10 +914,13 @@ dict_tf_to_fsp_flags( ulint table_flags) /*!< in: dict_table_t::flags */ { ulint fsp_flags; - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", return(ULINT_UNDEFINED);); @@ -925,30 +928,23 @@ dict_tf_to_fsp_flags( fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ - fsp_flags |= table_flags & DICT_TF_MASK_ZIP_SSIZE; - fsp_flags |= table_flags & DICT_TF_MASK_ATOMIC_BLOBS; - - /* In addition, tablespace flags also contain the page size. */ - fsp_flags |= fsp_flags_set_page_size(fsp_flags, UNIV_PAGE_SIZE); + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); - /* The DATA_DIR flag is in a different position in fsp_flag */ - fsp_flags |= DICT_TF_HAS_DATA_DIR(table_flags) - ? FSP_FLAGS_MASK_DATA_DIR : 0; + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; + } - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); + ut_a(fsp_flags_is_valid(fsp_flags)); - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + } - ut_a(fsp_flags_is_valid(fsp_flags)); - ut_a(dict_tf_verify_flags(table_flags, fsp_flags)); + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; return(fsp_flags); } diff --git a/storage/innobase/include/dict0pagecompress.h b/storage/innobase/include/dict0pagecompress.h index 19a2a6c52f3..6503c86ffa2 100644 --- a/storage/innobase/include/dict0pagecompress.h +++ b/storage/innobase/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,17 +56,6 @@ dict_table_page_compression_level( const dict_table_t* table) /*!< in: table */ __attribute__((const)); -/********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - /********************************************************************//** Extract the atomic writes flag from table flags. @return true if atomic writes are used, false if not used */ diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index 811976434a8..13c2b46c51c 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -24,92 +24,6 @@ page compression and atomic writes information to dictionary. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - - return(TRUE); -} - /********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index cc67d918d5f..525a68bf5e6 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,7 +44,6 @@ Created 10/25/1995 Heikki Tuuri // Forward declaration struct trx_t; -struct fil_space_t; typedef std::list space_name_list_t; @@ -271,10 +270,6 @@ struct fil_space_t { an insert buffer merge request for a page because it actually was for the previous incarnation of the space */ - ibool mark; /*!< this is set to TRUE at database startup if - the space corresponds to a table in the InnoDB - data dictionary; so we can print a warning of - orphaned tablespaces */ ibool stop_ios;/*!< TRUE if we want to rename the .ibd file of tablespace and want to stop temporarily posting of new i/o @@ -303,7 +298,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ - ulint flags; /*!< tablespace flags; see + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0fsp.h, fsp_flags_is_valid(), fsp_flags_get_zip_size() */ ulint n_reserved_extents; @@ -455,6 +451,7 @@ fil_node_create( ibool is_raw) /*!< in: TRUE if a raw device or a raw disk partition */ MY_ATTRIBUTE((nonnull, warn_unused_result)); + #ifdef UNIV_LOG_ARCHIVE /****************************************************************//** Drops files from the start of a file space, so that its size is cut by @@ -618,7 +615,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ #ifdef UNIV_LOG_ARCHIVE ulint* min_arch_log_no, /*!< out: min of archived @@ -836,6 +833,14 @@ fil_create_new_single_table_tablespace( ulint key_id) /*!< in: encryption key_id */ __attribute__((nonnull, warn_unused_result)); #ifndef UNIV_HOTBACKUP +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -864,7 +869,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* filepath, /*!< in: tablespace filepath */ @@ -905,25 +910,18 @@ fil_tablespace_exists_in_mem( /*=========================*/ ulint id); /*!< in: space id */ #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ const char* name, /*!< in: table name in the standard 'databasename/tablename' format */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -931,7 +929,8 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id); /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags); /*!< in: table flags */ #else /* !UNIV_HOTBACKUP */ /********************************************************************//** Extends all tablespaces to the size stored in the space header. During the diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 10db59fb218..1fe5cb66bf6 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,20 +44,11 @@ Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -ibool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index abcd5721a47..24fe382b344 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -52,28 +52,67 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) + +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 @@ -83,29 +122,16 @@ is found in a remote location, not the default data directory. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Note that these need to be before the page size to be compatible with -dictionary */ -/** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR \ - + FSP_FLAGS_WIDTH_DATA_DIR) +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -123,26 +149,23 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) + /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ ((flags & FSP_FLAGS_MASK_POST_ANTELOPE) \ @@ -159,49 +182,78 @@ dictionary */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) -/** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Return the value of the PAGE_COMPRESSION_LEVEL field */ +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ #define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) - -/** Set a PAGE_SSIZE into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize) \ - (flags | (ssize << FSP_FLAGS_POS_PAGE_SSIZE)) + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) +/* Compatibility macros for MariaDB 10.1.20 or older 10.1 see +table above. */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) +/** Return the value of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Return the value of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) /* @} */ @@ -735,19 +787,193 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ - MY_ATTRIBUTE((warn_unused_result, const)); +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} + +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +ulint +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} + +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) +UNIV_INLINE +bool +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} + /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 9f09a9d53e1..59c732b7a29 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2012, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -46,107 +46,6 @@ fsp_descr_page( return((page_no & (zip_size - 1)) == FSP_XDES_OFFSET); } -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ -UNIV_INLINE -bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ -{ - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - ulint atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* fsp_flags is zero unless atomic_blobs is set. */ - /* Make sure there are no bits that we do not know about. */ - if (unused != 0 || flags == 1) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted unused %lu\n", - flags, unused); - return(false); - } else if (post_antelope) { - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so this flag and the entire - 4-byte field is zero for Antelope row formats. */ - - if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_blobs %lu\n", - flags, atomic_blobs); - return(false); - } - } - - if (!atomic_blobs) { - /* Barracuda row formats COMPRESSED and DYNAMIC build on - the page structure introduced for the COMPACT row format - by allowing long fields to be broken into prefix and - externally stored parts. */ - - if (post_antelope || zip_ssize != 0) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu atomic_blobs %lu\n", - flags, zip_ssize, atomic_blobs); - return(false); - } - - } else if (!post_antelope || zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu max %d\n", - flags, zip_ssize, PAGE_ZIP_SSIZE_MAX); - return(false); - } else if (page_ssize > UNIV_PAGE_SSIZE_MAX) { - - /* The page size field can be used for any row type, or it may - be zero for an original 16k page size. - Validate the page shift size is within allowed range. */ - - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu\n", - flags, page_ssize, UNIV_PAGE_SSIZE_MAX); - return(false); - - } else if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_ORIG && !page_ssize) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu:%d\n", - flags, page_ssize, UNIV_PAGE_SIZE, UNIV_PAGE_SIZE_ORIG); - return(false); - } - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_compression %lu\n" - "InnoDB: Error: page_compression_level %lu atomic_blobs %lu\n", - flags, page_compression, page_compression_level, atomic_blobs); - return(false); - } - } - - if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_writes %lu\n", - flags, atomic_writes); - return (false); - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error "UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations." -#endif - - /* The DATA_DIR field can be used for any row type so there is - nothing here to validate. */ - - return(true); -} - /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ @@ -191,10 +90,10 @@ UNIV_INLINE ulint fsp_flags_get_page_size( /*====================*/ - ulint flags) /*!< in: tablespace flags */ + ulint flags) /*!< in: tablespace flags */ { - ulint page_size = 0; - ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + ulint page_size = 0; + ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); /* Convert from a 'log2 minus 9' to a page size in bytes. */ if (UNIV_UNLIKELY(ssize)) { @@ -211,50 +110,6 @@ fsp_flags_get_page_size( } #ifndef UNIV_INNOCHECKSUM - -/********************************************************************//** -Add the page size to the tablespace flags. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( -/*====================*/ - ulint flags, /*!< in: tablespace flags */ - ulint page_size) /*!< in: page size in bytes */ -{ - ulint ssize = 0; - ulint shift; - - /* Page size should be > UNIV_PAGE_SIZE_MIN */ - ut_ad(page_size >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size <= UNIV_PAGE_SIZE_MAX); - - if (page_size == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - return(flags); - } - - for (shift = UNIV_PAGE_SIZE_SHIFT_MAX; - shift >= UNIV_PAGE_SIZE_SHIFT_MIN; - shift--) { - ulint mask = (1 << shift); - if (page_size & mask) { - ut_ad(!(page_size & ~mask)); - ssize = shift - UNIV_ZIP_SIZE_SHIFT_MIN + 1; - break; - } - } - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags = FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /********************************************************************//** Calculates the descriptor index within a descriptor page. @return descriptor index */ diff --git a/storage/innobase/include/fsp0pagecompress.h b/storage/innobase/include/fsp0pagecompress.h index 5f943ee2b83..c623d11c326 100644 --- a/storage/innobase/include/fsp0pagecompress.h +++ b/storage/innobase/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,15 +47,6 @@ fsp_header_get_compression_level( /*=============================*/ const page_t* page); /*!< in: first page of a tablespace */ -/********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - /********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index e879aa2c16e..48163277feb 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,18 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ -/********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - /********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ @@ -125,21 +113,15 @@ Extract the page compression from space. @return true if space is page compressed, false if space is not found or space is not page compressed. */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 3a47e3fc6aa..f54c9d589c2 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -371,8 +371,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ @@ -535,10 +534,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -553,6 +548,19 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid FSP_SPACE_FLAGS=0x%x", + int(m_space_flags)); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } + + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); /* Since we don't know whether it is a compressed table or not, the data is always read into the block->frame. */ @@ -640,46 +648,6 @@ struct FetchIndexRootPages : public AbstractCallback { return(m_space); } - /** - Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags - determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invlad table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd " - "file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - /** Called for each block as it is read from the file. @param offset - physical offset in the file @@ -743,12 +711,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, page_no)); if (m_indexes.size() == 1) { - - m_table_flags = dict_sys_tables_type_to_tf( - m_space_flags, - page_is_comp(page) ? DICT_N_COLS_COMPACT : 0); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -1965,21 +1938,14 @@ PageConverter::update_header( "- ignored"); } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unsupported tablespace format %lu", - (ulong) space_flags); - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 1c02590933f..0e539f53641 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -4335,6 +4335,7 @@ row_drop_table_for_mysql( switch (err) { ibool is_temp; + ulint table_flags; case DB_SUCCESS: /* Clone the name, in case it has been allocated @@ -4343,6 +4344,7 @@ row_drop_table_for_mysql( space_id = table->space; ibd_file_missing = table->ibd_file_missing; + table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); /* If there is a temp path then the temp flag is set. @@ -4358,9 +4360,9 @@ row_drop_table_for_mysql( } /* We do not allow temporary tables with a remote path. */ - ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table->flags))); + ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table_flags))); - if (space_id && DICT_TF_HAS_DATA_DIR(table->flags)) { + if (space_id && DICT_TF_HAS_DATA_DIR(table_flags)) { dict_get_and_save_data_dir_path(table, true); ut_a(table->data_dir_path); @@ -4426,8 +4428,9 @@ row_drop_table_for_mysql( if (err == DB_SUCCESS && space_id > TRX_SYS_SPACE) { if (!is_temp && !fil_space_for_table_exists_in_mem( - space_id, tablename, FALSE, - print_msg, false, NULL, 0)) { + space_id, tablename, + print_msg, false, NULL, 0, + table_flags)) { /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 40154c11ce1..2fec8b7bc2f 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -673,8 +673,7 @@ create_log_files( sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); fil_space_create( - logfilename, SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), + logfilename, SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* this is create */); @@ -1159,7 +1158,7 @@ check_first_page: crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, 0, flags, FIL_TABLESPACE, crypt_data, (*create_new_db) == true); @@ -1307,7 +1306,7 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space); /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, space, flags, FIL_TABLESPACE, NULL /* no encryption */, true /* create */); @@ -2296,9 +2295,7 @@ innobase_start_or_create_for_mysql(void) sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); fil_space_create(logfilename, - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), - FIL_LOG, + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); @@ -2714,6 +2711,13 @@ files_checked: } if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 9c7eea6410a..68bb83e4903 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -589,6 +589,21 @@ buf_dblwr_process() continue; } + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":0 due to invalid flags 0x%x", + space_id, int(flags)); + continue; + } + /* The flags on the page should be converted later. */ + } + /* Write the good page from the doublewrite buffer to the intended position. */ diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 39a6ae993c8..776bb873c5c 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1052,8 +1052,6 @@ loop: btr_pcur_store_position(&pcur, &mtr); - mtr_commit(&mtr); - /* For tables created with old versions of InnoDB, SYS_TABLES.MIX_LEN may contain garbage. Such tables would always be in ROW_FORMAT=REDUNDANT. Pretend that @@ -1087,16 +1085,19 @@ loop: if (space_id == 0) { /* The system tablespace always exists. */ ut_ad(!discarded); - goto next_tablespace; + mem_free(name); + goto loop; } + mtr_commit(&mtr); + switch (dict_check) { case DICT_CHECK_ALL_LOADED: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, TRUE, !(is_temp || discarded), - false, NULL, 0) + space_id, name, !(is_temp || discarded), + false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1128,8 +1129,8 @@ loop: /* Some tablespaces may have been opened in trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, FALSE, FALSE, - false, NULL, 0)) { + space_id, name, false, + false, NULL, 0, flags)) { break; } /* fall through */ @@ -1191,7 +1192,6 @@ loop: max_space_id = space_id; } -next_tablespace: mem_free(name); mtr_start(&mtr); @@ -2383,8 +2383,8 @@ err_exit: table->ibd_file_missing = TRUE; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, FALSE, FALSE, true, heap, - table->id)) { + table->space, name, false, true, heap, + table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 133960ae8b4..f4301d47028 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -601,10 +601,7 @@ fil_node_open_file( ibool success; byte* buf2; byte* page; - ulint space_id; - ulint flags=0; ulint page_size; - ulint atomic_writes=0; ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); @@ -626,8 +623,6 @@ fil_node_open_file( /* The following call prints an error message */ os_file_get_last_error(true); - ut_print_timestamp(stderr); - ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Error: cannot " "open %s\n. InnoDB: Have you deleted .ibd " "files under a running mysqld server?\n", @@ -653,17 +648,13 @@ fil_node_open_file( ut_a(fil_is_user_tablespace_id(space->id)); if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) { - fprintf(stderr, - "InnoDB: Error: the size of single-table" - " tablespace file %s\n" - "InnoDB: is only " UINT64PF "," - " should be at least %lu!\n", - node->name, - size_bytes, - (ulong) (FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - - ut_a(0); + ib_logf(IB_LOG_LEVEL_ERROR, + "The size of the file %s is only " UINT64PF + " bytes, should be at least " ULINTPF, + node->name, size_bytes, + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + os_file_close(node->handle); + return(false); } /* Read the first page of the tablespace */ @@ -676,77 +667,34 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); srv_stats.page0_read.add(1); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - page_size = fsp_flags_get_page_size(flags); - atomic_writes = fsp_flags_get_atomic_writes(flags); + const ulint space_id = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); ut_free(buf2); - - /* Close the file now that we have read the space id from it */ - os_file_close(node->handle); - if (UNIV_UNLIKELY(space_id != space->id)) { - fprintf(stderr, - "InnoDB: Error: tablespace id is %lu" - " in the data dictionary\n" - "InnoDB: but in file %s it is %lu!\n", - space->id, node->name, space_id); - - ut_error; - } - - if (UNIV_UNLIKELY(space_id == ULINT_UNDEFINED - || space_id == 0)) { - fprintf(stderr, - "InnoDB: Error: tablespace id %lu" - " in file %s is not sensible\n", - (ulong) space_id, node->name); - - ut_error; - } - - if (UNIV_UNLIKELY(fsp_flags_get_page_size(space->flags) - != page_size)) { - fprintf(stderr, - "InnoDB: Error: tablespace file %s" - " has page size 0x%lx\n" - "InnoDB: but the data dictionary" - " expects page size 0x%lx!\n", - node->name, flags, - fsp_flags_get_page_size(space->flags)); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Expected tablespace flags 0x%x" + " but found 0x%x in the file %s", + int(space->flags), int(flags), + node->name); + return(false); + } - ut_error; + flags = cflags; } - if (UNIV_UNLIKELY(space->flags != flags)) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - fprintf(stderr, - "InnoDB: Warning: Table flags 0x%lx" - " in the data dictionary but in file %s are 0x%lx!\n" - " Temporally corrected because DATA_DIR option to 0x%lx.\n", - space->flags, node->name, flags, space->flags); - - flags = space->flags; - } + page_size = fsp_flags_get_page_size(flags); - if (!dict_tf_verify_flags(space->flags, flags)) { - fprintf(stderr, - "InnoDB: Error: table flags are 0x%lx" - " in the data dictionary\n" - "InnoDB: but the flags in file %s are 0x%lx!\n", - space->flags, node->name, flags); - ut_error; - } + if (UNIV_UNLIKELY(space_id != space->id)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "tablespace id is " ULINTPF " in the data dictionary" + " but in file %s it is " ULINTPF "!\n", + space->id, node->name, space_id); + return(false); } if (size_bytes >= (1024*1024)) { @@ -768,7 +716,7 @@ add_size: space->size += node->size; } - atomic_writes = fsp_flags_get_atomic_writes(space->flags); + ulint atomic_writes = fsp_flags_get_atomic_writes(space->flags); /* printf("Opening file %s\n", node->name); */ @@ -1572,7 +1520,6 @@ fil_space_create( fil_system->tablespace_version++; space->tablespace_version = fil_system->tablespace_version; - space->mark = FALSE; if (purpose == FIL_TABLESPACE && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -2324,27 +2271,21 @@ fil_write_flushed_lsn_to_data_files( return(DB_SUCCESS); } -/*******************************************************************//** -Checks the consistency of the first data page of a tablespace +/** Check the consistency of the first data page of a tablespace at database startup. +@param[in] page page frame +@param[in] space_id tablespace identifier +@param[in] flags tablespace flags @retval NULL on success, or if innodb_force_recovery is set @return pointer to an error message string */ static MY_ATTRIBUTE((warn_unused_result)) const char* -fil_check_first_page( -/*=================*/ - const page_t* page) /*!< in: data page */ +fil_check_first_page(const page_t* page, ulint space_id, ulint flags) { - ulint space_id; - ulint flags; - if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); } - space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { fprintf(stderr, "InnoDB: Error: Current page size %lu != " @@ -2393,7 +2334,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ lsn_t* min_flushed_lsn, /*!< out: min of flushed lsn values in data files */ @@ -2423,12 +2364,22 @@ fil_read_first_page( *flags and *space_id as they were read from the first file and do not validate the first page. */ if (!one_read_already) { - *flags = fsp_header_get_flags(page); *space_id = fsp_header_get_space_id(page); - } + *flags = fsp_header_get_flags(page); - if (!one_read_already) { - check_msg = fil_check_first_page(page); + if (!fsp_flags_is_valid(*flags)) { + ulint cflags = fsp_flags_convert_from_101(*flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid flags 0x%x in tablespace %u", + unsigned(*flags), unsigned(*space_id)); + return "invalid tablespace flags"; + } else { + *flags = cflags; + } + } + + check_msg = fil_check_first_page(page, *space_id, *flags); } flushed_lsn = mach_read_from_8(page + @@ -2612,6 +2563,7 @@ fil_op_write_log( ulint len; log_ptr = mlog_open(mtr, 11 + 2 + 1); + ut_ad(fsp_flags_is_valid(flags)); if (!log_ptr) { /* Logging in mtr is switched off during crash recovery: @@ -3823,7 +3775,7 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3831,7 +3783,7 @@ fil_create_new_single_table_tablespace( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); if (is_temp) { /* Temporary table filepath */ @@ -3924,12 +3876,9 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, UNIV_PAGE_SIZE); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - ut_ad(fsp_flags_is_valid(flags)); if (!(fsp_flags_is_compressed(flags))) { buf_flush_init_for_writing(page, NULL, 0); @@ -4008,7 +3957,8 @@ fil_create_new_single_table_tablespace( fil_op_write_log(flags ? MLOG_FILE_CREATE2 : MLOG_FILE_CREATE, - space_id, mlog_file_flag, flags, + space_id, mlog_file_flag, + flags & ~FSP_FLAGS_MEM_MASK, tablename, NULL, &mtr); mtr_commit(&mtr); @@ -4072,6 +4022,39 @@ fil_report_bad_tablespace( (ulong) expected_id, (ulong) expected_flags); } +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, + &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib_logf(IB_LOG_LEVEL_WARN, + "adjusting FSP_SPACE_FLAGS of tablespace " + ULINTPF " from 0x%x to 0x%x", + space_id, int(f), int(flags)); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + /********************************************************************//** Tries to open a single-table tablespace and optionally checks that the space id in it is correct. If this does not succeed, print an error message @@ -4101,7 +4084,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* path_in, /*!< in: tablespace filepath */ @@ -4126,20 +4109,13 @@ fil_open_single_table_tablespace( /* Table flags can be ULINT_UNDEFINED if dict_tf_to_fsp_flags_failure is set. */ - if (flags != ULINT_UNDEFINED) { - if (!fsp_flags_is_valid(flags)) { - return(DB_CORRUPTION); - } - } else { + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); atomic_writes = fsp_flags_get_atomic_writes(flags); - /* If the tablespace was relocated, we do not - compare the DATA_DIR flag */ - ulint mod_flags = flags & ~FSP_FLAGS_MASK_DATA_DIR; - memset(&def, 0, sizeof(def)); memset(&dict, 0, sizeof(dict)); memset(&remote, 0, sizeof(remote)); @@ -4217,31 +4193,17 @@ fil_open_single_table_tablespace( def.check_msg = fil_read_first_page( def.file, FALSE, &def.flags, &def.id, &def.lsn, &def.lsn, &def.crypt_data); - def.valid = !def.check_msg; if (table) { table->crypt_data = def.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - - ulint newf = def.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (def.valid && def.id == id - && newf == mod_flags) { + def.valid = !def.check_msg && def.id == id + && fsp_flags_match(flags, def.flags); + if (def.valid) { valid_tablespaces_found++; } else { - def.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( def.filepath, def.check_msg, def.id, @@ -4254,30 +4216,18 @@ fil_open_single_table_tablespace( remote.check_msg = fil_read_first_page( remote.file, FALSE, &remote.flags, &remote.id, &remote.lsn, &remote.lsn, &remote.crypt_data); - remote.valid = !remote.check_msg; if (table) { table->crypt_data = remote.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = remote.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (remote.valid && remote.id == id - && newf == mod_flags) { + /* Validate this single-table-tablespace with SYS_TABLES. */ + remote.valid = !remote.check_msg && remote.id == id + && fsp_flags_match(flags, remote.flags); + if (remote.valid) { valid_tablespaces_found++; } else { - remote.valid = false; /* Do not use this linked tablespace. */ fil_report_bad_tablespace( remote.filepath, remote.check_msg, remote.id, @@ -4291,30 +4241,19 @@ fil_open_single_table_tablespace( dict.check_msg = fil_read_first_page( dict.file, FALSE, &dict.flags, &dict.id, &dict.lsn, &dict.lsn, &dict.crypt_data); - dict.valid = !dict.check_msg; if (table) { table->crypt_data = dict.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = dict.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } + /* Validate this single-table-tablespace with SYS_TABLES. */ + dict.valid = !dict.check_msg && dict.id == id + && fsp_flags_match(flags, dict.flags); - if (dict.valid && dict.id == id - && newf == mod_flags) { + if (dict.valid) { valid_tablespaces_found++; } else { - dict.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( dict.filepath, dict.check_msg, dict.id, @@ -4512,6 +4451,10 @@ cleanup_and_exit: mem_free(def.filepath); + if (err == DB_SUCCESS && !srv_read_only_mode) { + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); + } + return(err); } #endif /* !UNIV_HOTBACKUP */ @@ -4693,7 +4636,23 @@ fil_user_tablespace_restore_page( goto out; } - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":" ULINTPF + " due to invalid flags 0x%x", + fsp->id, page_no, int(flags)); + err = false; + goto out; + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + zip_size = fsp_flags_get_zip_size(flags); page_size = fsp_flags_get_page_size(flags); @@ -5071,6 +5030,16 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { + fsp->flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + } + remote.flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. + That will be adjusted in fil_space_for_table_exists_in_mem(). */ + ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5380,13 +5349,12 @@ fil_report_missing_tablespace( name, space_id); } -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ @@ -5394,13 +5362,7 @@ fil_space_for_table_exists_in_mem( fil_space_create(). Either the standard 'dbname/tablename' format or table->dir_path_of_temp_table */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -5408,12 +5370,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id) /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags) /*!< in: table flags */ { fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -5425,42 +5388,31 @@ fil_space_for_table_exists_in_mem( directory path from the datadir to the file */ fnamespace = fil_space_get_by_name(name); - if (space && space == fnamespace) { - /* Found */ - - if (mark_space) { - space->mark = TRUE; - } - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since it is - not transactional. If adjust_space is set, and the mismatching - space are between a user table and its temp table, we shall - adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - char* tmp_name; + char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); - - fil_rename_tablespace(fnamespace->name, fnamespace->id, - tmp_name, NULL); - } + fil_rename_tablespace(fnamespace->name, fnamespace->id, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -5473,16 +5425,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(TRUE); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (space == NULL) { @@ -5509,10 +5457,8 @@ error_exit: fputs("InnoDB: Please refer to\n" "InnoDB: " REFMAN "innodb-troubleshooting-datadict.html\n" "InnoDB: for how to resolve the issue.\n", stderr); - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -5539,9 +5485,19 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(FALSE); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } /*******************************************************************//** @@ -7444,9 +7400,8 @@ fil_space_get_crypt_data( byte *page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, NULL, NULL); - ulint flags = fsp_header_get_flags(page); ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags), NULL); + fsp_header_get_zip_size(page), NULL); space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 4acfd134c1f..c32fddaabbe 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -663,6 +663,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -713,7 +714,7 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index d9d40b809d1..2b63ddea51d 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -918,10 +918,13 @@ dict_tf_to_fsp_flags( ulint table_flags) /*!< in: dict_table_t::flags */ { ulint fsp_flags; - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", return(ULINT_UNDEFINED);); @@ -929,30 +932,23 @@ dict_tf_to_fsp_flags( fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ - fsp_flags |= table_flags & DICT_TF_MASK_ZIP_SSIZE; - fsp_flags |= table_flags & DICT_TF_MASK_ATOMIC_BLOBS; - - /* In addition, tablespace flags also contain the page size. */ - fsp_flags |= fsp_flags_set_page_size(fsp_flags, UNIV_PAGE_SIZE); + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); - /* The DATA_DIR flag is in a different position in fsp_flag */ - fsp_flags |= DICT_TF_HAS_DATA_DIR(table_flags) - ? FSP_FLAGS_MASK_DATA_DIR : 0; + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; + } - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); + ut_a(fsp_flags_is_valid(fsp_flags)); - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + } - ut_a(fsp_flags_is_valid(fsp_flags)); - ut_a(dict_tf_verify_flags(table_flags, fsp_flags)); + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; return(fsp_flags); } diff --git a/storage/xtradb/include/dict0pagecompress.h b/storage/xtradb/include/dict0pagecompress.h index 19a2a6c52f3..6503c86ffa2 100644 --- a/storage/xtradb/include/dict0pagecompress.h +++ b/storage/xtradb/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,17 +56,6 @@ dict_table_page_compression_level( const dict_table_t* table) /*!< in: table */ __attribute__((const)); -/********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - /********************************************************************//** Extract the atomic writes flag from table flags. @return true if atomic writes are used, false if not used */ diff --git a/storage/xtradb/include/dict0pagecompress.ic b/storage/xtradb/include/dict0pagecompress.ic index 811976434a8..13c2b46c51c 100644 --- a/storage/xtradb/include/dict0pagecompress.ic +++ b/storage/xtradb/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -24,92 +24,6 @@ page compression and atomic writes information to dictionary. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - - return(TRUE); -} - /********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 38cc09bced3..41e38794ea9 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,7 +45,6 @@ Created 10/25/1995 Heikki Tuuri // Forward declaration struct trx_t; -struct fil_space_t; typedef std::list space_name_list_t; @@ -264,10 +263,6 @@ struct fil_space_t { an insert buffer merge request for a page because it actually was for the previous incarnation of the space */ - ibool mark; /*!< this is set to TRUE at database startup if - the space corresponds to a table in the InnoDB - data dictionary; so we can print a warning of - orphaned tablespaces */ ibool stop_ios;/*!< TRUE if we want to rename the .ibd file of tablespace and want to stop temporarily posting of new i/o @@ -296,7 +291,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ - ulint flags; /*!< tablespace flags; see + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0fsp.h, fsp_flags_is_valid(), fsp_flags_get_zip_size() */ ulint n_reserved_extents; @@ -449,6 +445,7 @@ fil_node_create( ibool is_raw) /*!< in: TRUE if a raw device or a raw disk partition */ MY_ATTRIBUTE((nonnull, warn_unused_result)); + #ifdef UNIV_LOG_ARCHIVE /****************************************************************//** Drops files from the start of a file space, so that its size is cut by @@ -620,7 +617,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ lsn_t* min_flushed_lsn, /*!< out: min of flushed lsn values in data files */ @@ -832,6 +829,14 @@ fil_create_new_single_table_tablespace( ulint key_id) /*!< in: encryption key_id */ __attribute__((nonnull, warn_unused_result)); #ifndef UNIV_HOTBACKUP +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -860,7 +865,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* filepath, /*!< in: tablespace filepath */ @@ -901,25 +906,18 @@ fil_tablespace_exists_in_mem( /*=========================*/ ulint id); /*!< in: space id */ #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ const char* name, /*!< in: table name in the standard 'databasename/tablename' format */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -927,7 +925,8 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id); /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags); /*!< in: table flags */ #else /* !UNIV_HOTBACKUP */ /********************************************************************//** Extends all tablespaces to the size stored in the space header. During the diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 10db59fb218..1fe5cb66bf6 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,20 +44,11 @@ Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -ibool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 551a8c9ef97..93f98eb0b0b 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -51,28 +51,67 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) + +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 @@ -82,29 +121,16 @@ is found in a remote location, not the default data directory. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Note that these need to be before the page size to be compatible with -dictionary */ -/** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the start of the DATA DIR bits */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR\ - + FSP_FLAGS_WIDTH_DATA_DIR) +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -122,26 +148,23 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) + /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ ((flags & FSP_FLAGS_MASK_POST_ANTELOPE) \ @@ -158,48 +181,78 @@ dictionary */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) + +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ +#define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) + +/* Compatibility macros for MariaDB 10.1.20 or older 10.1 see +table above. */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) + +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) + /** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) /** Return the value of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ -#define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) - -/** Set a PAGE_SSIZE into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize) \ - (flags | (ssize << FSP_FLAGS_POS_PAGE_SSIZE)) - -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) - -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) - -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) /* @} */ @@ -733,19 +786,193 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ - MY_ATTRIBUTE((warn_unused_result, const)); +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} + +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +ulint +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} + +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) +UNIV_INLINE +bool +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} + /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ diff --git a/storage/xtradb/include/fsp0fsp.ic b/storage/xtradb/include/fsp0fsp.ic index ddcb87b0e57..ee4cb1f32c7 100644 --- a/storage/xtradb/include/fsp0fsp.ic +++ b/storage/xtradb/include/fsp0fsp.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2012, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -46,107 +46,6 @@ fsp_descr_page( return((page_no & (zip_size - 1)) == FSP_XDES_OFFSET); } -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ -UNIV_INLINE -bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ -{ - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - ulint atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* fsp_flags is zero unless atomic_blobs is set. */ - /* Make sure there are no bits that we do not know about. */ - if (unused != 0 || flags == 1) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted unused %lu\n", - flags, unused); - return(false); - } else if (post_antelope) { - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so this flag and the entire - 4-byte field is zero for Antelope row formats. */ - - if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_blobs %lu\n", - flags, atomic_blobs); - return(false); - } - } - - if (!atomic_blobs) { - /* Barracuda row formats COMPRESSED and DYNAMIC build on - the page structure introduced for the COMPACT row format - by allowing long fields to be broken into prefix and - externally stored parts. */ - - if (post_antelope || zip_ssize != 0) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu atomic_blobs %lu\n", - flags, zip_ssize, atomic_blobs); - return(false); - } - - } else if (!post_antelope || zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu max %d\n", - flags, zip_ssize, PAGE_ZIP_SSIZE_MAX); - return(false); - } else if (page_ssize > UNIV_PAGE_SSIZE_MAX) { - - /* The page size field can be used for any row type, or it may - be zero for an original 16k page size. - Validate the page shift size is within allowed range. */ - - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu\n", - flags, page_ssize, UNIV_PAGE_SSIZE_MAX); - return(false); - - } else if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_ORIG && !page_ssize) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu:%d\n", - flags, page_ssize, UNIV_PAGE_SIZE, UNIV_PAGE_SIZE_ORIG); - return(false); - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error "UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations." -#endif - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_compression %lu\n" - "InnoDB: Error: page_compression_level %lu atomic_blobs %lu\n", - flags, page_compression, page_compression_level, atomic_blobs); - return(false); - } - } - - if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_writes %lu\n", - flags, atomic_writes); - return (false); - } - - /* The DATA_DIR field can be used for any row type so there is - nothing here to validate. */ - - return(true); -} - /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ @@ -191,10 +90,10 @@ UNIV_INLINE ulint fsp_flags_get_page_size( /*====================*/ - ulint flags) /*!< in: tablespace flags */ + ulint flags) /*!< in: tablespace flags */ { - ulint page_size = 0; - ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + ulint page_size = 0; + ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); /* Convert from a 'log2 minus 9' to a page size in bytes. */ if (UNIV_UNLIKELY(ssize)) { @@ -211,50 +110,6 @@ fsp_flags_get_page_size( } #ifndef UNIV_INNOCHECKSUM - -/********************************************************************//** -Add the page size to the tablespace flags. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( -/*====================*/ - ulint flags, /*!< in: tablespace flags */ - ulint page_size) /*!< in: page size in bytes */ -{ - ulint ssize = 0; - ulint shift; - - /* Page size should be > UNIV_PAGE_SIZE_MIN */ - ut_ad(page_size >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size <= UNIV_PAGE_SIZE_MAX); - - if (page_size == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - return(flags); - } - - for (shift = UNIV_PAGE_SIZE_SHIFT_MAX; - shift >= UNIV_PAGE_SIZE_SHIFT_MIN; - shift--) { - ulint mask = (1 << shift); - if (page_size & mask) { - ut_ad(!(page_size & ~mask)); - ssize = shift - UNIV_ZIP_SIZE_SHIFT_MIN + 1; - break; - } - } - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags = FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /********************************************************************//** Calculates the descriptor index within a descriptor page. @return descriptor index */ diff --git a/storage/xtradb/include/fsp0pagecompress.h b/storage/xtradb/include/fsp0pagecompress.h index 5f943ee2b83..c623d11c326 100644 --- a/storage/xtradb/include/fsp0pagecompress.h +++ b/storage/xtradb/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,15 +47,6 @@ fsp_header_get_compression_level( /*=============================*/ const page_t* page); /*!< in: first page of a tablespace */ -/********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - /********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level diff --git a/storage/xtradb/include/fsp0pagecompress.ic b/storage/xtradb/include/fsp0pagecompress.ic index e879aa2c16e..48163277feb 100644 --- a/storage/xtradb/include/fsp0pagecompress.ic +++ b/storage/xtradb/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,18 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ -/********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - /********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ @@ -125,21 +113,15 @@ Extract the page compression from space. @return true if space is page compressed, false if space is not found or space is not page compressed. */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index d45ce907304..6170eb66195 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -371,8 +371,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ @@ -535,10 +534,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -553,6 +548,19 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid FSP_SPACE_FLAGS=0x%x", + int(m_space_flags)); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } + + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); /* Since we don't know whether it is a compressed table or not, the data is always read into the block->frame. */ @@ -640,46 +648,6 @@ struct FetchIndexRootPages : public AbstractCallback { return(m_space); } - /** - Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags - determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invlad table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd " - "file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - /** Called for each block as it is read from the file. @param offset - physical offset in the file @@ -743,12 +711,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, page_no)); if (m_indexes.size() == 1) { - - m_table_flags = dict_sys_tables_type_to_tf( - m_space_flags, - page_is_comp(page) ? DICT_N_COLS_COMPACT : 0); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -1968,21 +1941,14 @@ PageConverter::update_header( "- ignored"); } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unsupported tablespace format %lu", - (ulong) space_flags); - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index aff7b758249..c81b10b93f1 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4362,6 +4362,7 @@ row_drop_table_for_mysql( switch (err) { ibool is_temp; + ulint table_flags; case DB_SUCCESS: /* Clone the name, in case it has been allocated @@ -4370,6 +4371,7 @@ row_drop_table_for_mysql( space_id = table->space; ibd_file_missing = table->ibd_file_missing; + table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); /* If there is a temp path then the temp flag is set. @@ -4385,9 +4387,9 @@ row_drop_table_for_mysql( } /* We do not allow temporary tables with a remote path. */ - ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table->flags))); + ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table_flags))); - if (space_id && DICT_TF_HAS_DATA_DIR(table->flags)) { + if (space_id && DICT_TF_HAS_DATA_DIR(table_flags)) { dict_get_and_save_data_dir_path(table, true); ut_a(table->data_dir_path); @@ -4453,8 +4455,9 @@ row_drop_table_for_mysql( if (err == DB_SUCCESS && space_id > TRX_SYS_SPACE) { if (!is_temp && !fil_space_for_table_exists_in_mem( - space_id, tablename, FALSE, - print_msg, false, NULL, 0)) { + space_id, tablename, + print_msg, false, NULL, 0, + table_flags)) { /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 77cdfe1affb..b933171e683 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -703,8 +703,7 @@ create_log_files( sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); fil_space_create( - logfilename, SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), + logfilename, SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* this is create */); @@ -1193,7 +1192,7 @@ check_first_page: crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, 0, flags, FIL_TABLESPACE, crypt_data, (*create_new_db) == true); @@ -1341,7 +1340,7 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space); /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, space, flags, FIL_TABLESPACE, NULL /* no encryption */, true /* create */); @@ -2374,9 +2373,7 @@ innobase_start_or_create_for_mysql(void) sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); fil_space_create(logfilename, - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), - FIL_LOG, + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); @@ -2771,6 +2768,13 @@ files_checked: } if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( -- cgit v1.2.1 From e4e801d4789d3992c1f04b260de3af4c9e5e6b0c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 17 Jan 2017 11:15:21 +0100 Subject: connect: compilation errors and few obvious bugs --- storage/connect/filamzip.cpp | 8 ++++---- storage/connect/ioapi.h | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'storage') diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index 65013e170e4..22fe95aebad 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -97,7 +97,7 @@ loopStart: if (!*++pat) return TRUE; goto loopStart; default: - if (mapCaseTable[*s] != mapCaseTable[*p]) + if (mapCaseTable[(unsigned)*s] != mapCaseTable[(unsigned)*p]) goto starCheck; break; } /* endswitch */ @@ -151,7 +151,7 @@ int ZIPUTIL::findEntry(PGLOBAL g, bool next) if (rc == UNZ_END_OF_LIST_OF_FILE) return RC_EF; else if (rc != UNZ_OK) { - sprintf(g->Message, "unzGoToNextFile rc = ", rc); + sprintf(g->Message, "unzGoToNextFile rc = %d", rc); return RC_FX; } // endif rc @@ -261,7 +261,7 @@ bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, char *fn) fp->Memory = memory; fp->Mode = mode; fp->File = this; - fp->Handle = NULL; + fp->Handle = 0; } // endif fp } else @@ -297,7 +297,7 @@ bool ZIPUTIL::openEntry(PGLOBAL g) memory = new char[size + 1]; if ((rc = unzReadCurrentFile(zipfile, memory, size)) < 0) { - sprintf(g->Message, "unzReadCurrentFile rc = ", rc); + sprintf(g->Message, "unzReadCurrentFile rc = %d", rc); unzCloseCurrentFile(zipfile); free(memory); memory = NULL; diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h index 8dcbdb06e35..4fa73002053 100644 --- a/storage/connect/ioapi.h +++ b/storage/connect/ioapi.h @@ -129,8 +129,9 @@ extern "C" { #endif #endif - - +#ifndef OF +#define OF(args) args +#endif typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); -- cgit v1.2.1