diff options
66 files changed, 2197 insertions, 507 deletions
diff --git a/Docs/Makefile.am b/Docs/Makefile.am index 6f8c51e79f0..b1f69381774 100644 --- a/Docs/Makefile.am +++ b/Docs/Makefile.am @@ -36,22 +36,22 @@ CLEAN_FILES: $(txt_files) GT = $(srcdir)/Support/generate-text-files.pl ../INSTALL-SOURCE: mysql.info $(GT) - perl -w $(GT) mysql.info "Installing" "Tutorial" > $@ + perl -w $(GT) mysql.info "installing-source" "windows-source-build" > $@ ../INSTALL-WIN-SOURCE: mysql.info $(GT) - perl -w $(GT) mysql.info "Windows source build" "Post-installation" > $@ + perl -w $(GT) mysql.info "windows-source-build" "post-installation" > $@ # We put the description for the binary installation here so that # people who download source wont have to see it. It is moved up to # the toplevel by the script that makes the binary tar files. INSTALL-BINARY: mysql.info $(GT) - perl -w $(GT) mysql.info "Installing binary" "Installing source" > $@ + perl -w $(GT) mysql.info "installing-binary" "installing-source" > $@ ../EXCEPTIONS-CLIENT: mysql.info $(GT) - perl -w $(GT) mysql.info "MySQL FLOSS License Exception" "Function Index" > $@ + perl -w $(GT) mysql.info "mysql-floss-license-exception" "function-index" > $@ ../support-files/MacOSX/ReadMe.txt: mysql.info $(GT) - perl -w $(GT) mysql.info "Mac OS X installation" "NetWare installation" > $@ + perl -w $(GT) mysql.info "mac-os-x-installation" "netware-installation" > $@ # Don't update the files from bitkeeper %::SCCS/s.% diff --git a/Docs/Support/generate-flag-images b/Docs/Support/generate-flag-images deleted file mode 100755 index 21140388012..00000000000 --- a/Docs/Support/generate-flag-images +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -flags=`grep @image mirrors.texi | cut -d" " -f1 | cut -d/ -f2 | tr -d "}" | sort | uniq` - -set -x -cd Flags - -for c in $flags -do - # For PNM, to be used later - giftopnm ../Raw-Flags/$c.gif | pnmscale -xsize 30 > $c-tmp.pnm - pnmpaste $c-tmp.pnm 1 1 ../Images/flag-background.pnm > $c.pnm - rm -f $c-tmp.pnm - - # For GIF version - ppmtogif $c.pnm > $c.gif - # or cjpeg -optimize -quality 70 -outfile $c.jpg - - # For EPS version - pnmtops -noturn $c.pnm > $c.eps - - # For PDF version - ps2pdf $c.eps $c.pdf - - # For text version - echo -n "" > $c.txt - - # PNM isn't really needed - rm -f $c.pnm - -done diff --git a/Docs/Support/generate-text-files.pl b/Docs/Support/generate-text-files.pl index 6470baaa6e9..0829525f679 100755 --- a/Docs/Support/generate-text-files.pl +++ b/Docs/Support/generate-text-files.pl @@ -13,7 +13,7 @@ while (<IN>) { if ($in) { - if (/Node: $tnode,/) + if (/Node: $tnode,/ || /\[index/) { $in = 0; } diff --git a/Makefile.am b/Makefile.am index 9a6fcf3c95a..d7059c6adaf 100644 --- a/Makefile.am +++ b/Makefile.am @@ -101,12 +101,12 @@ tags: test: cd mysql-test; \ - perl mysql-test-run.pl && perl mysql-test-run.pl --ps-protocol + ./mysql-test-run && ./mysql-test-run --ps-protocol test-force: cd mysql-test; \ - perl mysql-test-run.pl --force ;\ - perl mysql-test-run.pl --ps-protocol --force + ./mysql-test-run --force ;\ + ./mysql-test-run --ps-protocol --force # Don't update the files from bitkeeper %::SCCS/s.% diff --git a/configure.in b/configure.in index ae26a0c44df..9e294a35223 100644 --- a/configure.in +++ b/configure.in @@ -6,7 +6,7 @@ AC_PREREQ(2.50)dnl Minimum Autoconf version required. AC_INIT(sql/mysqld.cc) AC_CANONICAL_SYSTEM # Don't forget to also update the NDB lines below. -AM_INIT_AUTOMAKE(mysql, 5.0.8-beta) +AM_INIT_AUTOMAKE(mysql, 5.0.9-beta) AM_CONFIG_HEADER(config.h) PROTOCOL_VERSION=10 @@ -17,7 +17,7 @@ SHARED_LIB_VERSION=14:0:0 # ndb version NDB_VERSION_MAJOR=5 NDB_VERSION_MINOR=0 -NDB_VERSION_BUILD=8 +NDB_VERSION_BUILD=9 NDB_VERSION_STATUS="beta" # Set all version vars based on $VERSION. How do we do this more elegant ? @@ -342,7 +342,7 @@ AC_SUBST(CXXFLAGS) AC_SUBST(LD) AC_SUBST(INSTALL_SCRIPT) -export CC CXX CFLAGS LD LDFLAGS AR +export CC CXX CFLAGS LD LDFLAGS AR ARFLAGS if test "$GCC" = "yes" then diff --git a/extra/yassl/taocrypt/include/runtime.hpp b/extra/yassl/taocrypt/include/runtime.hpp index 70768bb01d1..f506040f0d8 100644 --- a/extra/yassl/taocrypt/include/runtime.hpp +++ b/extra/yassl/taocrypt/include/runtime.hpp @@ -25,7 +25,7 @@ -#if !defined(yaSSL_NEW_HPP) && defined(__GNUC__) +#if !defined(yaSSL_NEW_HPP) && defined(__GNUC__) && !defined(__ICC) #define yaSSL_NEW_HPP diff --git a/extra/yassl/taocrypt/src/integer.cpp b/extra/yassl/taocrypt/src/integer.cpp index 0f06bb4e044..460b2d31426 100644 --- a/extra/yassl/taocrypt/src/integer.cpp +++ b/extra/yassl/taocrypt/src/integer.cpp @@ -35,7 +35,8 @@ #endif -#if defined(_MSC_VER) && defined(_WIN64) // 64 bit X overflow intrinsic +#if defined(_MSC_VER) && defined(_WIN64) && \ + !defined(__INTEL_COMPILER) // 64 bit X overflow intrinsic #ifdef __ia64__ #define myUMULH __UMULH #elif __x86_64__ diff --git a/include/my_base.h b/include/my_base.h index 25fa683744e..c76cf8c604e 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -367,6 +367,7 @@ enum ha_base_keytype { #define HA_STATE_EXTEND_BLOCK 2048 #define HA_STATE_RNEXT_SAME 4096 /* rnext_same was called */ +/* myisampack expects no more than 32 field types. */ enum en_fieldtype { FIELD_LAST=-1,FIELD_NORMAL,FIELD_SKIP_ENDSPACE,FIELD_SKIP_PRESPACE, FIELD_SKIP_ZERO,FIELD_BLOB,FIELD_CONSTANT,FIELD_INTERVALL,FIELD_ZERO, diff --git a/include/my_global.h b/include/my_global.h index 04d33be38b4..22ec6dfffd3 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -295,10 +295,8 @@ C_MODE_END #include <alloca.h> #endif #ifdef HAVE_ATOMIC_ADD -#if defined(__ia64__) #define new my_arg_new #define need_to_restore_new 1 -#endif C_MODE_START #include <asm/atomic.h> C_MODE_END diff --git a/innobase/configure.in b/innobase/configure.in index baf11272ab9..c56bd8274c4 100644 --- a/innobase/configure.in +++ b/innobase/configure.in @@ -117,6 +117,13 @@ case "$target" in CFLAGS="$CFLAGS -DUNIV_MUST_NOT_INLINE";; esac +# must go in pair with AR as set by MYSQL_CHECK_AR +if test -z "$ARFLAGS" +then + ARFLAGS="cru" +fi +AC_SUBST(ARFLAGS) + AC_OUTPUT(Makefile os/Makefile ut/Makefile btr/Makefile dnl buf/Makefile data/Makefile dnl dict/Makefile dyn/Makefile dnl diff --git a/myisam/mi_packrec.c b/myisam/mi_packrec.c index 4b512dd89dd..c251e4dda4a 100644 --- a/myisam/mi_packrec.c +++ b/myisam/mi_packrec.c @@ -416,8 +416,19 @@ static uint find_longest_bitstream(uint16 *table, uint16 *end) } - /* Read record from datafile */ - /* Returns length of packed record, -1 if error */ +/* + Read record from datafile. + + SYNOPSIS + _mi_read_pack_record() + info A pointer to MI_INFO. + filepos File offset of the record. + buf RETURN The buffer to receive the record. + + RETURN + 0 on success + HA_ERR_WRONG_IN_RECORD or -1 on error +*/ int _mi_read_pack_record(MI_INFO *info, my_off_t filepos, byte *buf) { diff --git a/myisam/myisampack.c b/myisam/myisampack.c index 74bb541b220..70a32902da6 100644 --- a/myisam/myisampack.c +++ b/myisam/myisampack.c @@ -33,10 +33,10 @@ #include <my_getopt.h> #include <assert.h> -#if INT_MAX > 32767 -#define BITS_SAVED 32 +#if SIZEOF_LONG_LONG > 4 +#define BITS_SAVED 64 #else -#define BITS_SAVED 16 +#define BITS_SAVED 32 #endif #define IS_OFFSET ((uint) 32768) /* Bit if offset or char in tree */ @@ -49,10 +49,10 @@ struct st_file_buffer { File file; - char *buffer,*pos,*end; + uchar *buffer,*pos,*end; my_off_t pos_in_file; int bits; - uint current_byte; + ulonglong bitbucket; }; struct st_huff_tree; @@ -69,13 +69,17 @@ typedef struct st_huff_counts { my_off_t end_space[8]; my_off_t pre_space[8]; my_off_t tot_end_space,tot_pre_space,zero_fields,empty_fields,bytes_packed; - TREE int_tree; - byte *tree_buff; - byte *tree_pos; + TREE int_tree; /* Tree for detecting distinct column values. */ + byte *tree_buff; /* Column values, 'field_length' each. */ + byte *tree_pos; /* Points to end of column values in 'tree_buff'. */ } HUFF_COUNTS; typedef struct st_huff_element HUFF_ELEMENT; +/* + WARNING: It is crucial for the optimizations in calc_packed_length() + that 'count' is the first element of 'HUFF_ELEMENT'. +*/ struct st_huff_element { my_off_t count; union un_element { @@ -98,7 +102,7 @@ typedef struct st_huff_tree { my_off_t bytes_packed; uint tree_pack_length; uint min_chr,max_chr,char_bits,offset_bits,max_offset,height; - ulong *code; + ulonglong *code; uchar *code_len; } HUFF_TREE; @@ -146,7 +150,7 @@ static uint join_same_trees(HUFF_COUNTS *huff_counts,uint trees); static int make_huff_decode_table(HUFF_TREE *huff_tree,uint trees); static void make_traverse_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element,uint size, - ulong code); + ulonglong code); static int write_header(PACK_MRG_INFO *isam_file, uint header_length,uint trees, my_off_t tot_elements,my_off_t filelength); static void write_field_info(HUFF_COUNTS *counts, uint fields,uint trees); @@ -161,7 +165,7 @@ static char *make_old_name(char *new_name,char *old_name); static void init_file_buffer(File file,pbool read_buffer); static int flush_buffer(ulong neaded_length); static void end_file_buffer(void); -static void write_bits(ulong value,uint bits); +static void write_bits(ulonglong value, uint bits); static void flush_bits(void); static int save_state(MI_INFO *isam_file,PACK_MRG_INFO *mrg,my_off_t new_length, ha_checksum crc); @@ -170,13 +174,23 @@ static int save_state_mrg(File file,PACK_MRG_INFO *isam_file,my_off_t new_length static int mrg_close(PACK_MRG_INFO *mrg); static int mrg_rrnd(PACK_MRG_INFO *info,byte *buf); static void mrg_reset(PACK_MRG_INFO *mrg); +#if !defined(DBUG_OFF) +static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count); +static int fakecmp(my_off_t **count1, my_off_t **count2); +#endif static int error_on_write=0,test_only=0,verbose=0,silent=0, write_loop=0,force_pack=0, isamchk_neaded=0; static int tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL; static my_bool backup, opt_wait; -static uint tree_buff_length=8196-MALLOC_OVERHEAD; +/* + tree_buff_length is somewhat arbitrary. The bigger it is the better + the chance to win in terms of compression factor. On the other hand, + this table becomes part of the compressed file header. And its length + is coded with 16 bits in the header. Hence the limit is 2**16 - 1. +*/ +static uint tree_buff_length= 65536 - MALLOC_OVERHEAD; static char tmp_dir[FN_REFLEN]={0},*join_table; static my_off_t intervall_length; static ha_checksum glob_crc; @@ -225,7 +239,8 @@ int main(int argc, char **argv) } if (ok && isamchk_neaded && !silent) puts("Remember to run myisamchk -rq on compressed tables"); - VOID(fflush(stdout)); VOID(fflush(stderr)); + VOID(fflush(stdout)); + VOID(fflush(stderr)); free_defaults(default_argv); my_end(verbose ? MY_CHECK_ERROR | MY_GIVE_INFO : MY_CHECK_ERROR); exit(error ? 2 : 0); @@ -260,7 +275,7 @@ static struct my_option my_long_options[] = 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"test", 't', "Don't pack table, only test packing it.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"verbose", 'v', "Write info about progress and packing result.", + {"verbose", 'v', "Write info about progress and packing result. Use many -v for more verbosity!", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"version", 'V', "Output version information and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -273,7 +288,8 @@ static struct my_option my_long_options[] = static void print_version(void) { - printf("%s Ver 1.22 for %s on %s\n", my_progname, SYSTEM_TYPE, MACHINE_TYPE); + VOID(printf("%s Ver 1.22 for %s on %s\n", + my_progname, SYSTEM_TYPE, MACHINE_TYPE)); NETWARE_SET_SCREEN_MODE(1); } @@ -290,7 +306,7 @@ static void usage(void) puts("afterwards to update the keys."); puts("You should give the .MYI file as the filename argument."); - printf("\nUsage: %s [OPTIONS] filename...\n", my_progname); + VOID(printf("\nUsage: %s [OPTIONS] filename...\n", my_progname)); my_print_help(my_long_options); print_defaults("my", load_default_groups); my_print_variables(my_long_options); @@ -314,7 +330,10 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), silent= 1; break; case 't': - test_only= verbose= 1; + test_only= 1; + /* Avoid to reset 'verbose' if it was already set > 1. */ + if (! verbose) + verbose= 1; break; case 'T': length= (uint) (strmov(tmp_dir, argument) - tmp_dir); @@ -325,7 +344,7 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), } break; case 'v': - verbose= 1; + verbose++; /* Allow for selecting the level of verbosity. */ silent= 0; break; case '#': @@ -380,7 +399,7 @@ static MI_INFO *open_isam_file(char *name,int mode) (opt_wait ? HA_OPEN_WAIT_IF_LOCKED : HA_OPEN_ABORT_IF_LOCKED)))) { - VOID(fprintf(stderr,"%s gave error %d on open\n",name,my_errno)); + VOID(fprintf(stderr, "%s gave error %d on open\n", name, my_errno)); DBUG_RETURN(0); } share=isam_file->s; @@ -388,7 +407,7 @@ static MI_INFO *open_isam_file(char *name,int mode) { if (!force_pack) { - VOID(fprintf(stderr,"%s is already compressed\n",name)); + VOID(fprintf(stderr, "%s is already compressed\n", name)); VOID(mi_close(isam_file)); DBUG_RETURN(0); } @@ -400,7 +419,7 @@ static MI_INFO *open_isam_file(char *name,int mode) (share->state.state.records <= 1 || share->state.state.data_file_length < 1024)) { - VOID(fprintf(stderr,"%s is too small to compress\n",name)); + VOID(fprintf(stderr, "%s is too small to compress\n", name)); VOID(mi_close(isam_file)); DBUG_RETURN(0); } @@ -446,8 +465,8 @@ static bool open_isam_files(PACK_MRG_INFO *mrg,char **names,uint count) return 0; diff_file: - fprintf(stderr,"%s: Tables '%s' and '%s' are not identical\n", - my_progname,names[j],names[j+1]); + VOID(fprintf(stderr, "%s: Tables '%s' and '%s' are not identical\n", + my_progname, names[j], names[j+1])); error: while (i--) mi_close(mrg->file[i]); @@ -518,16 +537,25 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) mrg->records=0; for (i=0 ; i < mrg->count ; i++) mrg->records+=mrg->file[i]->s->state.state.records; + + DBUG_PRINT("info", ("Compressing %s: (%lu records)", + result_table ? new_name : org_name, + (ulong) mrg->records)); if (write_loop || verbose) { - printf("Compressing %s: (%lu records)\n", - result_table ? new_name : org_name,(ulong) mrg->records); + VOID(printf("Compressing %s: (%lu records)\n", + result_table ? new_name : org_name, (ulong) mrg->records)); } trees=fields=share->base.fields; huff_counts=init_huff_count(isam_file,mrg->records); QUICK_SAFEMALLOC; + + /* + Read the whole data file(s) for statistics. + */ + DBUG_PRINT("info", ("- Calculating statistics")); if (write_loop || verbose) - printf("- Calculating statistics\n"); + VOID(printf("- Calculating statistics\n")); if (get_statistic(mrg,huff_counts)) goto err; NORMAL_SAFEMALLOC; @@ -536,29 +564,74 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) old_length+= (mrg->file[i]->s->state.state.data_file_length - mrg->file[i]->s->state.state.empty); + /* + Create a global priority queue in preparation for making + temporary Huffman trees. + */ if (init_queue(&queue,256,0,0,compare_huff_elements,0)) goto err; + + /* + Check each column if we should use pre-space-compress, end-space- + compress, empty-field-compress or zero-field-compress. + */ check_counts(huff_counts,fields,mrg->records); + + /* + Build a Huffman tree for each column. + */ huff_trees=make_huff_trees(huff_counts,trees); + + /* + If the packed lengths of combined columns is less then the sum of + the non-combined columns, then create common Huffman trees for them. + We do this only for byte compressed columns, not for distinct values + compressed columns. + */ if ((int) (used_trees=join_same_trees(huff_counts,trees)) < 0) goto err; + + /* + Assign codes to all byte or column values. + */ if (make_huff_decode_table(huff_trees,fields)) goto err; + /* Prepare a file buffer. */ init_file_buffer(new_file,0); + + /* + Reserve space in the target file for the fixed compressed file header. + */ file_buffer.pos_in_file=HEAD_LENGTH; if (! test_only) VOID(my_seek(new_file,file_buffer.pos_in_file,MY_SEEK_SET,MYF(0))); + /* + Write field infos: field type, pack type, length bits, tree number. + */ write_field_info(huff_counts,fields,used_trees); + + /* + Write decode trees. + */ if (!(tot_elements=write_huff_tree(huff_trees,trees))) goto err; + + /* + Calculate the total length of the compression info header. + This includes the fixed compressed file header, the column compression + type descriptions, and the decode trees. + */ header_length=(uint) file_buffer.pos_in_file+ (uint) (file_buffer.pos-file_buffer.buffer); - /* Compress file */ + /* + Compress the source file into the target file. + */ + DBUG_PRINT("info", ("- Compressing file")); if (write_loop || verbose) - printf("- Compressing file\n"); + VOID(printf("- Compressing file\n")); error=compress_isam_file(mrg,huff_counts); new_length=file_buffer.pos_in_file; if (!error && !test_only) @@ -568,16 +641,28 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) error=my_write(file_buffer.file,buff,sizeof(buff), MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; } + + /* + Write the fixed compressed file header. + */ if (!error) error=write_header(mrg,header_length,used_trees,tot_elements, new_length); + + /* Flush the file buffer. */ end_file_buffer(); + /* Display statistics. */ + DBUG_PRINT("info", ("Min record length: %6d Max length: %6d " + "Mean total length: %6ld\n", + mrg->min_pack_length, mrg->max_pack_length, + (ulong) (mrg->records ? (new_length/mrg->records) : 0))); if (verbose && mrg->records) - printf("Min record length: %6d Max length: %6d Mean total length: %6ld\n", - mrg->min_pack_length,mrg->max_pack_length, - (ulong) (new_length/mrg->records)); + VOID(printf("Min record length: %6d Max length: %6d " + "Mean total length: %6ld\n", mrg->min_pack_length, + mrg->max_pack_length, (ulong) (new_length/mrg->records))); + /* Close source and target file. */ if (!test_only) { error|=my_close(new_file,MYF(MY_WME)); @@ -588,6 +673,7 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) } } + /* Cleanup. */ free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields); if (! test_only && ! error) { @@ -629,15 +715,16 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) error|=my_close(join_isam_file,MYF(MY_WME)); if (error) { - VOID(fprintf(stderr,"Aborting: %s is not compressed\n",org_name)); + VOID(fprintf(stderr, "Aborting: %s is not compressed\n", org_name)); VOID(my_delete(new_name,MYF(MY_WME))); DBUG_RETURN(-1); } if (write_loop || verbose) { if (old_length) - printf("%.4g%% \n", (((longlong) (old_length -new_length))*100.0/ - (longlong) old_length)); + VOID(printf("%.4g%% \n", + (((longlong) (old_length - new_length)) * 100.0 / + (longlong) old_length))); else puts("Empty file saved in compressed format"); } @@ -650,7 +737,7 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table) if (join_isam_file >= 0) VOID(my_close(join_isam_file,MYF(0))); mrg_close(mrg); - VOID(fprintf(stderr,"Aborted: %s is not compressed\n",org_name)); + VOID(fprintf(stderr, "Aborted: %s is not compressed\n", org_name)); DBUG_RETURN(-1); } @@ -677,6 +764,12 @@ static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records) (type == FIELD_NORMAL || type == FIELD_SKIP_ZERO)) count[i].max_zero_fill= count[i].field_length; + /* + For every column initialize a tree, which is used to detect distinct + column values. 'int_tree' works together with 'tree_buff' and + 'tree_pos'. It's keys are implemented by pointers into 'tree_buff'. + This is accomplished by '-1' as the element size. + */ init_tree(&count[i].int_tree,0,0,-1,(qsort_cmp2) compare_tree,0, NULL, NULL); if (records && type != FIELD_BLOB && type != FIELD_VARCHAR) @@ -762,10 +855,13 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) ulong tot_blob_length=0; if (! error) { + /* glob_crc is a checksum over all bytes of all records. */ if (static_row_size) glob_crc+=mi_static_checksum(mrg->file[0],record); else glob_crc+=mi_checksum(mrg->file[0],record); + + /* Count the incidence of values separately for every column. */ for (pos=record,count=huff_counts ; count < end_count ; count++, @@ -773,15 +869,48 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) { next_pos=end_pos=(start_pos=pos)+count->field_length; - /* Put value in tree if there is room for it */ + /* + Put the whole column value in a tree if there is room for it. + 'int_tree' is used to quickly check for duplicate values. + 'tree_buff' collects as many distinct column values as + possible. If the field length is > 1, it is tree_buff_length, + else 2 bytes. Each value is 'field_length' bytes big. If there + are more distinct column values than fit into the buffer, we + give up with this tree. BLOBs and VARCHARs do not have a + tree_buff as it can only be used with fixed length columns. + For the special case of field length == 1, we handle only the + case that there is only one distinct value in the table(s). + Otherwise, we can have a maximum of 256 distinct values. This + is then handled by the normal Huffman tree build. + + Another limit for collecting distinct column values is the + number of values itself. Since we would need to build a + Huffman tree for the values, we are limited by the 'IS_OFFSET' + constant. This constant expresses a bit which is used to + determine if a tree element holds a final value or an offset + to a child element. Hence, all values and offsets need to be + smaller than 'IS_OFFSET'. A tree element is implemented with + two integer values, one for the left branch and one for the + right branch. For the extreme case that the first element + points to the last element, the number of integers in the tree + must be less or equal to IS_OFFSET. So the number of elements + must be less or equal to IS_OFFSET / 2. + + WARNING: At first, we insert a pointer into the record buffer + as the key for the tree. If we got a new distinct value, which + is really inserted into the tree, instead of being counted + only, we will copy the column value from the record buffer to + 'tree_buff' and adjust the key pointer of the tree accordingly. + */ if (count->tree_buff) { global_count=count; if (!(element=tree_insert(&count->int_tree,pos, 0, count->int_tree.custom_arg)) || (element->count == 1 && - count->tree_buff + tree_buff_length < - count->tree_pos + count->field_length) || + (count->tree_buff + tree_buff_length < + count->tree_pos + count->field_length)) || + (count->int_tree.elements_in_tree > IS_OFFSET / 2) || (count->field_length == 1 && count->int_tree.elements_in_tree > 1)) { @@ -791,10 +920,17 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) } else { + /* + If tree_insert() succeeds, it either creates a new element + or increments the counter of an existing element. + */ if (element->count == 1) - { /* New element */ + { + /* Copy the new column value into 'tree_buff'. */ memcpy(count->tree_pos,pos,(size_t) count->field_length); + /* Adjust the key pointer in the tree. */ tree_set_pointer(element,count->tree_pos); + /* Point behind the last column value so far. */ count->tree_pos+=count->field_length; } } @@ -804,15 +940,21 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) if (count->field_type == FIELD_NORMAL || count->field_type == FIELD_SKIP_ENDSPACE) { + /* Ignore trailing space. */ for ( ; end_pos > pos ; end_pos--) if (end_pos[-1] != ' ') break; + /* Empty fields are just counted. Go to the next record. */ if (end_pos == pos) { count->empty_fields++; count->max_zero_fill=0; continue; } + /* + Count the total of all trailing spaces and the number of + short trailing spaces. Remember the longest trailing space. + */ length= (uint) (next_pos-end_pos); count->tot_end_space+=length; if (length < 8) @@ -820,18 +962,25 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) if (count->max_end_space < length) count->max_end_space = length; } + if (count->field_type == FIELD_NORMAL || count->field_type == FIELD_SKIP_PRESPACE) { + /* Ignore leading space. */ for (pos=start_pos; pos < end_pos ; pos++) if (pos[0] != ' ') break; + /* Empty fields are just counted. Go to the next record. */ if (end_pos == pos) { count->empty_fields++; count->max_zero_fill=0; continue; } + /* + Count the total of all leading spaces and the number of + short leading spaces. Remember the longest leading space. + */ length= (uint) (pos-start_pos); count->tot_pre_space+=length; if (length < 8) @@ -839,6 +988,8 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) if (count->max_pre_space < length) count->max_pre_space = length; } + + /* Calculate pos, end_pos, and max_length for variable length fields. */ if (count->field_type == FIELD_BLOB) { uint field_length=count->field_length -mi_portable_sizeof_char_ptr; @@ -857,45 +1008,121 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) end_pos= pos+length; set_if_bigger(count->max_length,length); } + + /* Evaluate 'max_zero_fill' for short fields. */ if (count->field_length <= 8 && (count->field_type == FIELD_NORMAL || count->field_type == FIELD_SKIP_ZERO)) { uint i; + /* Zero fields are just counted. Go to the next record. */ if (!memcmp((byte*) start_pos,zero_string,count->field_length)) { count->zero_fields++; continue; } + /* + max_zero_fill starts with field_length. It is decreased every + time a shorter "zero trailer" is found. It is set to zero when + an empty field is found (see above). This suggests that the + variable should be called 'min_zero_fill'. + */ for (i =0 ; i < count->max_zero_fill && ! end_pos[-1 - (int) i] ; i++) ; if (i < count->max_zero_fill) count->max_zero_fill=i; } + + /* Ignore zero fields and check fields. */ if (count->field_type == FIELD_ZERO || count->field_type == FIELD_CHECK) continue; + + /* + Count the incidence of every byte value in the + significant field value. + */ for ( ; pos < end_pos ; pos++) count->counts[(uchar) *pos]++; + + /* Step to next field. */ } + if (tot_blob_length > max_blob_length) max_blob_length=tot_blob_length; record_count++; if (write_loop && record_count % WRITE_COUNT == 0) { - printf("%lu\r",(ulong) record_count); VOID(fflush(stdout)); + VOID(printf("%lu\r", (ulong) record_count)); + VOID(fflush(stdout)); } } else if (error != HA_ERR_RECORD_DELETED) { - fprintf(stderr,"Got error %d while reading rows",error); + VOID(fprintf(stderr, "Got error %d while reading rows", error)); break; } + + /* Step to next record. */ } if (write_loop) { - printf(" \r"); VOID(fflush(stdout)); + VOID(printf(" \r")); + VOID(fflush(stdout)); } + + /* + If --debug=d,fakebigcodes is set, fake the counts to get big Huffman + codes. + */ + DBUG_EXECUTE_IF("fakebigcodes", fakebigcodes(huff_counts, end_count);); + + DBUG_PRINT("info", ("Found the following number of incidents " + "of the byte codes:")); + if (verbose >= 2) + VOID(printf("Found the following number of incidents " + "of the byte codes:\n")); + for (count= huff_counts ; count < end_count; count++) + { + uint idx; + my_off_t total_count; + char llbuf[32]; + + DBUG_PRINT("info", ("column: %3u", count - huff_counts + 1)); + if (verbose >= 2) + VOID(printf("column: %3u\n", count - huff_counts + 1)); + if (count->tree_buff) + { + DBUG_PRINT("info", ("number of distinct values: %u", + (count->tree_pos - count->tree_buff) / + count->field_length)); + if (verbose >= 2) + VOID(printf("number of distinct values: %u\n", + (count->tree_pos - count->tree_buff) / + count->field_length)); + } + total_count= 0; + for (idx= 0; idx < 256; idx++) + { + if (count->counts[idx]) + { + total_count+= count->counts[idx]; + DBUG_PRINT("info", ("counts[0x%02x]: %12s", idx, + llstr((longlong) count->counts[idx], llbuf))); + if (verbose >= 2) + VOID(printf("counts[0x%02x]: %12s\n", idx, + llstr((longlong) count->counts[idx], llbuf))); + } + } + DBUG_PRINT("info", ("total: %12s", llstr((longlong) total_count, + llbuf))); + if ((verbose >= 2) && total_count) + { + VOID(printf("total: %12s\n", + llstr((longlong) total_count, llbuf))); + } + } + mrg->records=record_count; mrg->max_blob_length=max_blob_length; my_afree((gptr) record); @@ -944,9 +1171,14 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->field_type=FIELD_NORMAL; huff_counts->pack_type=0; + /* Check for zero-filled records (in this column), or zero records. */ if (huff_counts->zero_fields || ! records) { my_off_t old_space_count; + /* + If there are only zero filled records (in this column), + or no records at all, we are done. + */ if (huff_counts->zero_fields == records) { huff_counts->field_type= FIELD_ZERO; @@ -954,14 +1186,22 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->counts[0]=0; goto found_pack; } + /* Remeber the number of significant spaces. */ old_space_count=huff_counts->counts[' ']; - huff_counts->counts[' ']+=huff_counts->tot_end_space+ - huff_counts->tot_pre_space + - huff_counts->empty_fields * huff_counts->field_length; + /* Add all leading and trailing spaces. */ + huff_counts->counts[' ']+= (huff_counts->tot_end_space + + huff_counts->tot_pre_space + + huff_counts->empty_fields * + huff_counts->field_length); + /* Check, what the compressed length of this would be. */ old_length=calc_packed_length(huff_counts,0)+records/8; + /* Get the number of zero bytes. */ length=huff_counts->zero_fields*huff_counts->field_length; + /* Add it to the counts. */ huff_counts->counts[0]+=length; + /* Check, what the compressed length of this would be. */ new_length=calc_packed_length(huff_counts,0); + /* If the compression without the zeroes would be shorter, we are done. */ if (old_length < new_length && huff_counts->field_length > 1) { huff_counts->field_type=FIELD_SKIP_ZERO; @@ -969,9 +1209,16 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, huff_counts->bytes_packed=old_length- records/8; goto found_pack; } + /* Remove the insignificant spaces, but keep the zeroes. */ huff_counts->counts[' ']=old_space_count; } + /* Check, what the compressed length of this column would be. */ huff_counts->bytes_packed=calc_packed_length(huff_counts,0); + + /* + If there are enough empty records (in this column), + treating them specially may pay off. + */ if (huff_counts->empty_fields) { if (huff_counts->field_length > 2 && @@ -1003,6 +1250,11 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, } } } + + /* + If there are enough trailing spaces (in this column), + treating them specially may pay off. + */ if (huff_counts->tot_end_space) { huff_counts->counts[' ']+=huff_counts->tot_pre_space; @@ -1012,6 +1264,11 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, goto found_pack; huff_counts->counts[' ']-=huff_counts->tot_pre_space; } + + /* + If there are enough leading spaces (in this column), + treating them specially may pay off. + */ if (huff_counts->tot_pre_space) { if (test_space_compress(huff_counts,records,huff_counts->max_pre_space, @@ -1041,6 +1298,8 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, { HUFF_TREE tree; + DBUG_EXECUTE_IF("forceintervall", + huff_counts->bytes_packed= ~ (my_off_t) 0;); tree.element_buffer=0; if (!make_huff_tree(&tree,huff_counts) && tree.bytes_packed+tree.tree_pack_length < huff_counts->bytes_packed) @@ -1066,14 +1325,27 @@ static void check_counts(HUFF_COUNTS *huff_counts, uint trees, fill_zero_fields++; field_count[huff_counts->field_type]++; } + DBUG_PRINT("info", ("normal: %3d empty-space: %3d " + "empty-zero: %3d empty-fill: %3d", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIP_ZERO],fill_zero_fields)); + DBUG_PRINT("info", ("pre-space: %3d end-space: %3d " + "intervall-fields: %3d zero: %3d", + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO])); if (verbose) - printf("\nnormal: %3d empty-space: %3d empty-zero: %3d empty-fill: %3d\npre-space: %3d end-space: %3d intervall-fields: %3d zero: %3d\n", - field_count[FIELD_NORMAL],space_fields, - field_count[FIELD_SKIP_ZERO],fill_zero_fields, - field_count[FIELD_SKIP_PRESPACE], - field_count[FIELD_SKIP_ENDSPACE], - field_count[FIELD_INTERVALL], - field_count[FIELD_ZERO]); + VOID(printf("\nnormal: %3d empty-space: %3d " + "empty-zero: %3d empty-fill: %3d\n" + "pre-space: %3d end-space: %3d " + "intervall-fields: %3d zero: %3d\n", + field_count[FIELD_NORMAL],space_fields, + field_count[FIELD_SKIP_ZERO],fill_zero_fields, + field_count[FIELD_SKIP_PRESPACE], + field_count[FIELD_SKIP_ENDSPACE], + field_count[FIELD_INTERVALL], + field_count[FIELD_ZERO])); DBUG_VOID_RETURN; } @@ -1170,8 +1442,24 @@ static HUFF_TREE* make_huff_trees(HUFF_COUNTS *huff_counts, uint trees) DBUG_RETURN(huff_tree); } - /* Update huff_tree according to huff_counts->counts or - huff_counts->tree_buff */ +/* + Build a Huffman tree. + + SYNOPSIS + make_huff_tree() + huff_tree The Huffman tree. + huff_counts The counts. + + DESCRIPTION + Build a Huffman tree according to huff_counts->counts or + huff_counts->tree_buff. tree_buff, if non-NULL contains up to + tree_buff_length of distinct column values. In that case, whole + values can be Huffman encoded instead of single bytes. + + RETURN + 0 OK + != 0 Error +*/ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) { @@ -1182,12 +1470,14 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) first=last=0; if (huff_counts->tree_buff) { + /* Calculate the number of distinct values in tree_buff. */ found= (uint) (huff_counts->tree_pos - huff_counts->tree_buff) / huff_counts->field_length; first=0; last=found-1; } else { + /* Count the number of byte codes found in the column. */ for (i=found=0 ; i < 256 ; i++) { if (huff_counts->counts[i]) @@ -1201,6 +1491,7 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) found=2; } + /* When using 'tree_buff' we can have more that 256 values. */ if (queue.max_elements < found) { delete_queue(&queue); @@ -1208,6 +1499,7 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) return -1; } + /* Allocate or reallocate an element buffer for the Huffman tree. */ if (!huff_tree->element_buffer) { if (!(huff_tree->element_buffer= @@ -1235,15 +1527,25 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) if (huff_counts->tree_buff) { huff_tree->elements=0; - tree_walk(&huff_counts->int_tree, - (int (*)(void*, element_count,void*)) save_counts_in_queue, - (gptr) huff_tree, left_root_right); huff_tree->tree_pack_length=(1+15+16+5+5+ (huff_tree->char_bits+1)*found+ (huff_tree->offset_bits+1)* (found-2)+7)/8 + (uint) (huff_tree->counts->tree_pos- huff_tree->counts->tree_buff); + /* + Put a HUFF_ELEMENT into the queue for every distinct column value. + + tree_walk() calls save_counts_in_queue() for every element in + 'int_tree'. This takes elements from the target trees element + buffer and places references to them into the buffer of the + priority queue. We insert in column value order, but the order is + in fact irrelevant here. We will establish the correct order + later. + */ + tree_walk(&huff_counts->int_tree, + (int (*)(void*, element_count,void*)) save_counts_in_queue, + (gptr) huff_tree, left_root_right); } else { @@ -1252,7 +1554,15 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) (huff_tree->char_bits+1)*found+ (huff_tree->offset_bits+1)* (found-2)+7)/8; + /* + Put a HUFF_ELEMENT into the queue for every byte code found in the column. + The elements are taken from the target trees element buffer. + Instead of using queue_insert(), we just place references to the + elements into the buffer of the priority queue. We insert in byte + value order, but the order is in fact irrelevant here. We will + establish the correct order later. + */ for (i=first, found=0 ; i <= last ; i++) { if (huff_counts->counts[i]) @@ -1264,8 +1574,13 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) queue.root[found]=(byte*) new_huff_el; } } + /* + If there is only a single byte value in this field in all records, + add a second element with zero incidence. This is required to enter + the loop, which builds the Huffman tree. + */ while (found < 2) - { /* Our huff_trees request at least 2 elements */ + { new_huff_el=huff_tree->element_buffer+(found++); new_huff_el->count=0; new_huff_el->a.leaf.null=0; @@ -1276,21 +1591,53 @@ static int make_huff_tree(HUFF_TREE *huff_tree, HUFF_COUNTS *huff_counts) queue.root[found]=(byte*) new_huff_el; } } + + /* Make a queue from the queue buffer. */ queue.elements=found; + /* + Make a priority queue from the queue. Construct its index so that we + have a partially ordered tree. + */ for (i=found/2 ; i > 0 ; i--) _downheap(&queue,i); + + /* The Huffman algorithm. */ bytes_packed=0; bits_packed=0; for (i=1 ; i < found ; i++) { + /* + Pop the top element from the queue (the one with the least incidence). + Popping from a priority queue includes a re-ordering of the queue, + to get the next least incidence element to the top. + */ a=(HUFF_ELEMENT*) queue_remove(&queue,0); + /* + Copy the next least incidence element. The queue implementation + reserves root[0] for temporary purposes. root[1] is the top. + */ b=(HUFF_ELEMENT*) queue.root[1]; + /* Get a new element from the element buffer. */ new_huff_el=huff_tree->element_buffer+found+i; + /* The new element gets the sum of the two least incidence elements. */ new_huff_el->count=a->count+b->count; + /* + The Huffman algorithm assigns another bit to the code for a byte + every time that bytes incidence is combined (directly or indirectly) + to a new element as one of the two least incidence elements. + This means that one more bit per incidence of that byte is required + in the resulting file. So we add the new combined incidence as the + number of bits by which the result grows. + */ bits_packed+=(uint) (new_huff_el->count & 7); bytes_packed+=new_huff_el->count/8; - new_huff_el->a.nod.left=a; /* lesser in left */ + /* The new element points to its children, lesser in left. */ + new_huff_el->a.nod.left=a; new_huff_el->a.nod.right=b; + /* + Replace the copied top element by the new element and re-order the + queue. + */ queue.root[1]=(byte*) new_huff_el; queue_replaced(&queue); } @@ -1309,7 +1656,26 @@ static int compare_tree(void* cmp_arg __attribute__((unused)), return 0; } - /* Used by make_huff_tree to save intervall-counts in queue */ +/* + Organize distinct column values and their incidences into a priority queue. + + SYNOPSIS + save_counts_in_queue() + key The column value. + count The incidence of this value. + tree The Huffman tree to be built later. + + DESCRIPTION + We use the element buffer of the targeted tree. The distinct column + values are organized in a priority queue first. The Huffman + algorithm will later organize the elements into a Huffman tree. For + the time being, we just place references to the elements into the + queue buffer. The buffer will later be organized into a priority + queue. + + RETURN + 0 + */ static int save_counts_in_queue(byte *key, element_count count, HUFF_TREE *tree) @@ -1326,8 +1692,23 @@ static int save_counts_in_queue(byte *key, element_count count, } - /* Calculate length of file if given counts should be used */ - /* Its actually a faster version of make_huff_tree */ +/* + Calculate length of file if given counts should be used. + + SYNOPSIS + calc_packed_length() + huff_counts The counts for a column of the table(s). + add_tree_lenght If the decode tree length should be added. + + DESCRIPTION + We need to follow the Huffman algorithm until we know, how many bits + are required for each byte code. But we do not need the resulting + Huffman tree. Hence, we can leave out some steps which are essential + in make_huff_tree(). + + RETURN + Number of bytes required to compress this table column. +*/ static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, uint add_tree_lenght) @@ -1337,6 +1718,23 @@ static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, HUFF_ELEMENT element_buffer[256]; DBUG_ENTER("calc_packed_length"); + /* + WARNING: We use a small hack for efficiency: Instead of placing + references to HUFF_ELEMENTs into the queue, we just insert + references to the counts of the byte codes which appeared in this + table column. During the Huffman algorithm they are successively + replaced by references to HUFF_ELEMENTs. This works, because + HUFF_ELEMENTs have the incidence count at their beginning. + Regardless, wether the queue array contains references to counts of + type my_off_t or references to HUFF_ELEMENTs which have the count of + type my_off_t at their beginning, it always points to a count of the + same type. + + Instead of using queue_insert(), we just copy the references into + the buffer of the priority queue. We insert in byte value order, but + the order is in fact irrelevant here. We will establish the correct + order later. + */ first=last=0; for (i=found=0 ; i < 256 ; i++) { @@ -1345,31 +1743,73 @@ static my_off_t calc_packed_length(HUFF_COUNTS *huff_counts, if (! found++) first=i; last=i; + /* We start with root[1], which is the queues top element. */ queue.root[found]=(byte*) &huff_counts->counts[i]; } } if (!found) DBUG_RETURN(0); /* Empty tree */ + /* + If there is only a single byte value in this field in all records, + add a second element with zero incidence. This is required to enter + the loop, which follows the Huffman algorithm. + */ if (found < 2) queue.root[++found]=(byte*) &huff_counts->counts[last ? 0 : 1]; + /* Make a queue from the queue buffer. */ queue.elements=found; bytes_packed=0; bits_packed=0; + /* Add the length of the coding table, which would become part of the file. */ if (add_tree_lenght) bytes_packed=(8+9+5+5+(max_bit(last-first)+1)*found+ (max_bit(found-1)+1+1)*(found-2) +7)/8; + + /* + Make a priority queue from the queue. Construct its index so that we + have a partially ordered tree. + */ for (i=(found+1)/2 ; i > 0 ; i--) _downheap(&queue,i); + + /* The Huffman algorithm. */ for (i=0 ; i < found-1 ; i++) { - HUFF_ELEMENT *a,*b,*new_huff_el; - a=(HUFF_ELEMENT*) queue_remove(&queue,0); - b=(HUFF_ELEMENT*) queue.root[1]; - new_huff_el=element_buffer+i; - new_huff_el->count=a->count+b->count; + my_off_t *a; + my_off_t *b; + HUFF_ELEMENT *new_huff_el; + + /* + Pop the top element from the queue (the one with the least + incidence). Popping from a priority queue includes a re-ordering + of the queue, to get the next least incidence element to the top. + */ + a= (my_off_t*) queue_remove(&queue, 0); + /* + Copy the next least incidence element. The queue implementation + reserves root[0] for temporary purposes. root[1] is the top. + */ + b= (my_off_t*) queue.root[1]; + /* Create a new element in a local (automatic) buffer. */ + new_huff_el= element_buffer + i; + /* The new element gets the sum of the two least incidence elements. */ + new_huff_el->count= *a + *b; + /* + The Huffman algorithm assigns another bit to the code for a byte + every time that bytes incidence is combined (directly or indirectly) + to a new element as one of the two least incidence elements. + This means that one more bit per incidence of that byte is required + in the resulting file. So we add the new combined incidence as the + number of bits by which the result grows. + */ bits_packed+=(uint) (new_huff_el->count & 7); bytes_packed+=new_huff_el->count/8; + /* + Replace the copied top element by the new element and re-order the + queue. This successively replaces the references to counts by + references to HUFF_ELEMENTs. + */ queue.root[1]=(byte*) new_huff_el; queue_replaced(&queue); } @@ -1417,13 +1857,26 @@ static uint join_same_trees(HUFF_COUNTS *huff_counts, uint trees) } } } + DBUG_PRINT("info", ("Original trees: %d After join: %d", + trees, tree_number)); if (verbose) - printf("Original trees: %d After join: %d\n",trees,tree_number); + VOID(printf("Original trees: %d After join: %d\n", trees, tree_number)); return tree_number; /* Return trees left */ } - /* Fill in huff_tree decode tables */ +/* + Fill in huff_tree encode tables. + + SYNOPSIS + make_huff_decode_table() + huff_tree An array of HUFF_TREE which are to be encoded. + trees The number of HUFF_TREE in the array. + + RETURN + 0 success + != 0 error +*/ static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) { @@ -1434,12 +1887,13 @@ static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) { elements=huff_tree->counts->tree_buff ? huff_tree->elements : 256; if (!(huff_tree->code = - (ulong*) my_malloc(elements* - (sizeof(ulong)+sizeof(uchar)), - MYF(MY_WME | MY_ZEROFILL)))) + (ulonglong*) my_malloc(elements* + (sizeof(ulonglong) + sizeof(uchar)), + MYF(MY_WME | MY_ZEROFILL)))) return 1; huff_tree->code_len=(uchar*) (huff_tree->code+elements); - make_traverse_code_tree(huff_tree,huff_tree->root,32,0); + make_traverse_code_tree(huff_tree, huff_tree->root, + 8 * sizeof(ulonglong), LL(0)); } } return 0; @@ -1448,28 +1902,90 @@ static int make_huff_decode_table(HUFF_TREE *huff_tree, uint trees) static void make_traverse_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element, - uint size, ulong code) + uint size, ulonglong code) { uint chr; if (!element->a.leaf.null) { chr=element->a.leaf.element_nr; - huff_tree->code_len[chr]=(uchar) (32-size); - huff_tree->code[chr]= (code >> size); - if (huff_tree->height < 32-size) - huff_tree->height= 32-size; + huff_tree->code_len[chr]= (uchar) (8 * sizeof(ulonglong) - size); + huff_tree->code[chr]= (code >> size); + if (huff_tree->height < 8 * sizeof(ulonglong) - size) + huff_tree->height= 8 * sizeof(ulonglong) - size; } else { size--; make_traverse_code_tree(huff_tree,element->a.nod.left,size,code); - make_traverse_code_tree(huff_tree,element->a.nod.right,size, - code+((ulong) 1L << size)); + make_traverse_code_tree(huff_tree, element->a.nod.right, size, + code + (((ulonglong) 1) << size)); } return; } +/* + Convert a value into binary digits. + + SYNOPSIS + bindigits() + value The value. + length The number of low order bits to convert. + + NOTE + The result string is in static storage. It is reused on every call. + So you cannot use it twice in one expression. + + RETURN + A pointer to a static NUL-terminated string. + */ + +static char *bindigits(ulonglong value, uint bits) +{ + static char digits[72]; + char *ptr= digits; + uint idx= bits; + + DBUG_ASSERT(idx < sizeof(digits)); + while (idx) + *(ptr++)= '0' + ((value >> (--idx)) & 1); + *ptr= '\0'; + return digits; +} + + +/* + Convert a value into hexadecimal digits. + + SYNOPSIS + hexdigits() + value The value. + + NOTE + The result string is in static storage. It is reused on every call. + So you cannot use it twice in one expression. + + RETURN + A pointer to a static NUL-terminated string. + */ + +static char *hexdigits(ulonglong value) +{ + static char digits[20]; + char *ptr= digits; + uint idx= 2 * sizeof(value); /* Two hex digits per byte. */ + + DBUG_ASSERT(idx < sizeof(digits)); + while (idx) + { + if ((*(ptr++)= '0' + ((value >> (4 * (--idx))) & 0xf)) > '9') + *(ptr - 1)+= 'a' - '9' - 1; + } + *ptr= '\0'; + return digits; +} + + /* Write header to new packed data file */ static int write_header(PACK_MRG_INFO *mrg,uint head_length,uint trees, @@ -1503,15 +2019,64 @@ static void write_field_info(HUFF_COUNTS *counts, uint fields, uint trees) uint huff_tree_bits; huff_tree_bits=max_bit(trees ? trees-1 : 0); + DBUG_PRINT("info", ("")); + DBUG_PRINT("info", ("column types:")); + DBUG_PRINT("info", ("FIELD_NORMAL 0")); + DBUG_PRINT("info", ("FIELD_SKIP_ENDSPACE 1")); + DBUG_PRINT("info", ("FIELD_SKIP_PRESPACE 2")); + DBUG_PRINT("info", ("FIELD_SKIP_ZERO 3")); + DBUG_PRINT("info", ("FIELD_BLOB 4")); + DBUG_PRINT("info", ("FIELD_CONSTANT 5")); + DBUG_PRINT("info", ("FIELD_INTERVALL 6")); + DBUG_PRINT("info", ("FIELD_ZERO 7")); + DBUG_PRINT("info", ("FIELD_VARCHAR 8")); + DBUG_PRINT("info", ("FIELD_CHECK 9")); + DBUG_PRINT("info", ("")); + DBUG_PRINT("info", ("pack type as a set of flags:")); + DBUG_PRINT("info", ("PACK_TYPE_SELECTED 1")); + DBUG_PRINT("info", ("PACK_TYPE_SPACE_FIELDS 2")); + DBUG_PRINT("info", ("PACK_TYPE_ZERO_FILL 4")); + DBUG_PRINT("info", ("")); + if (verbose >= 2) + { + VOID(printf("\n")); + VOID(printf("column types:\n")); + VOID(printf("FIELD_NORMAL 0\n")); + VOID(printf("FIELD_SKIP_ENDSPACE 1\n")); + VOID(printf("FIELD_SKIP_PRESPACE 2\n")); + VOID(printf("FIELD_SKIP_ZERO 3\n")); + VOID(printf("FIELD_BLOB 4\n")); + VOID(printf("FIELD_CONSTANT 5\n")); + VOID(printf("FIELD_INTERVALL 6\n")); + VOID(printf("FIELD_ZERO 7\n")); + VOID(printf("FIELD_VARCHAR 8\n")); + VOID(printf("FIELD_CHECK 9\n")); + VOID(printf("\n")); + VOID(printf("pack type as a set of flags:\n")); + VOID(printf("PACK_TYPE_SELECTED 1\n")); + VOID(printf("PACK_TYPE_SPACE_FIELDS 2\n")); + VOID(printf("PACK_TYPE_ZERO_FILL 4\n")); + VOID(printf("\n")); + } for (i=0 ; i++ < fields ; counts++) { - write_bits((ulong) (int) counts->field_type,5); + write_bits((ulonglong) (int) counts->field_type, 5); write_bits(counts->pack_type,6); if (counts->pack_type & PACK_TYPE_ZERO_FILL) write_bits(counts->max_zero_fill,5); else write_bits(counts->length_bits,5); - write_bits((ulong) counts->tree->tree_number-1,huff_tree_bits); + write_bits((ulonglong) counts->tree->tree_number - 1, huff_tree_bits); + DBUG_PRINT("info", ("column: %3u type: %2u pack: %2u zero: %4u " + "lbits: %2u tree: %2u length: %4u", + i , counts->field_type, counts->pack_type, + counts->max_zero_fill, counts->length_bits, + counts->tree->tree_number, counts->field_length)); + if (verbose >= 2) + VOID(printf("column: %3u type: %2u pack: %2u zero: %4u lbits: %2u " + "tree: %2u length: %4u\n", i , counts->field_type, + counts->pack_type, counts->max_zero_fill, counts->length_bits, + counts->tree->tree_number, counts->field_length)); } flush_bits(); return; @@ -1524,45 +2089,72 @@ static void write_field_info(HUFF_COUNTS *counts, uint fields, uint trees) static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) { uint i,int_length; + uint tree_no; + uint codes; + uint errors= 0; uint *packed_tree,*offset,length; my_off_t elements; + /* Find the highest number of elements in the trees. */ for (i=length=0 ; i < trees ; i++) if (huff_tree[i].tree_number > 0 && huff_tree[i].elements > length) length=huff_tree[i].elements; + /* + Allocate a buffer for packing a decode tree. Two numbers per element + (left child and right child). + */ if (!(packed_tree=(uint*) my_alloca(sizeof(uint)*length*2))) { my_error(EE_OUTOFMEMORY,MYF(ME_BELL),sizeof(uint)*length*2); return 0; } + DBUG_PRINT("info", ("")); + if (verbose >= 2) + VOID(printf("\n")); + tree_no= 0; intervall_length=0; for (elements=0; trees-- ; huff_tree++) { + /* Skip columns that have been joined with other columns. */ if (huff_tree->tree_number == 0) continue; /* Deleted tree */ + tree_no++; + DBUG_PRINT("info", ("")); + if (verbose >= 3) + VOID(printf("\n")); + /* Count the total number of elements (byte codes or column values). */ elements+=huff_tree->elements; huff_tree->max_offset=2; + /* Build a tree of offsets and codes for decoding in 'packed_tree'. */ if (huff_tree->elements <= 1) offset=packed_tree; else offset=make_offset_code_tree(huff_tree,huff_tree->root,packed_tree); + + /* This should be the same as 'length' above. */ huff_tree->offset_bits=max_bit(huff_tree->max_offset); + + /* + Since we check this during collecting the distinct column values, + this should never happen. + */ if (huff_tree->max_offset >= IS_OFFSET) { /* This should be impossible */ - VOID(fprintf(stderr,"Tree offset got too big: %d, aborted\n", - huff_tree->max_offset)); + VOID(fprintf(stderr, "Tree offset got too big: %d, aborted\n", + huff_tree->max_offset)); my_afree((gptr) packed_tree); return 0; } -#ifdef EXTRA_DBUG - printf("pos: %d elements: %d tree-elements: %d char_bits: %d\n", - (uint) (file_buffer.pos-file_buffer.buffer), - huff_tree->elements, (offset-packed_tree),huff_tree->char_bits); -#endif + DBUG_PRINT("info", ("pos: %lu elements: %u tree-elements: %lu " + "char_bits: %u\n", + (ulong) (file_buffer.pos - file_buffer.buffer), + huff_tree->elements, (ulong) (offset - packed_tree), + huff_tree->char_bits)); if (!huff_tree->counts->tree_buff) { + /* We do a byte compression on this column. Mark with bit 0. */ write_bits(0,1); write_bits(huff_tree->min_chr,8); write_bits(huff_tree->elements,9); @@ -1574,6 +2166,7 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) { int_length=(uint) (huff_tree->counts->tree_pos - huff_tree->counts->tree_buff); + /* We have distinct column values for this column. Mark with bit 1. */ write_bits(1,1); write_bits(huff_tree->elements,15); write_bits(int_length,16); @@ -1581,10 +2174,29 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) write_bits(huff_tree->offset_bits,5); intervall_length+=int_length; } + DBUG_PRINT("info", ("tree: %2u elements: %4u char_bits: %2u " + "offset_bits: %2u %s: %5u codelen: %2u", + tree_no, huff_tree->elements, huff_tree->char_bits, + huff_tree->offset_bits, huff_tree->counts->tree_buff ? + "bufflen" : "min_chr", huff_tree->counts->tree_buff ? + int_length : huff_tree->min_chr, huff_tree->height)); + if (verbose >= 2) + VOID(printf("tree: %2u elements: %4u char_bits: %2u offset_bits: %2u " + "%s: %5u codelen: %2u\n", tree_no, huff_tree->elements, + huff_tree->char_bits, huff_tree->offset_bits, + huff_tree->counts->tree_buff ? "bufflen" : "min_chr", + huff_tree->counts->tree_buff ? int_length : + huff_tree->min_chr, huff_tree->height)); + + /* Check that the code tree length matches the element count. */ length=(uint) (offset-packed_tree); if (length != huff_tree->elements*2-2) - printf("error: Huff-tree-length: %d != calc_length: %d\n", - length,huff_tree->elements*2-2); + { + VOID(fprintf(stderr, "error: Huff-tree-length: %d != calc_length: %d\n", + length, huff_tree->elements * 2 - 2)); + errors++; + break; + } for (i=0 ; i < length ; i++) { @@ -1593,16 +2205,122 @@ static my_off_t write_huff_tree(HUFF_TREE *huff_tree, uint trees) huff_tree->offset_bits+1); else write_bits(packed_tree[i]-huff_tree->min_chr,huff_tree->char_bits+1); + DBUG_PRINT("info", ("tree[0x%04x]: %s0x%04x", + i, (packed_tree[i] & IS_OFFSET) ? + " -> " : "", (packed_tree[i] & IS_OFFSET) ? + packed_tree[i] - IS_OFFSET + i : packed_tree[i])); + if (verbose >= 3) + VOID(printf("tree[0x%04x]: %s0x%04x\n", + i, (packed_tree[i] & IS_OFFSET) ? " -> " : "", + (packed_tree[i] & IS_OFFSET) ? + packed_tree[i] - IS_OFFSET + i : packed_tree[i])); } flush_bits(); + + /* + Display coding tables and check their correctness. + */ + codes= huff_tree->counts->tree_buff ? huff_tree->elements : 256; + for (i= 0; i < codes; i++) + { + ulonglong code; + uint bits; + uint len; + uint idx; + + if (! (len= huff_tree->code_len[i])) + continue; + DBUG_PRINT("info", ("code[0x%04x]: 0x%s bits: %2u bin: %s", i, + hexdigits(huff_tree->code[i]), huff_tree->code_len[i], + bindigits(huff_tree->code[i], + huff_tree->code_len[i]))); + if (verbose >= 3) + VOID(printf("code[0x%04x]: 0x%s bits: %2u bin: %s\n", i, + hexdigits(huff_tree->code[i]), huff_tree->code_len[i], + bindigits(huff_tree->code[i], huff_tree->code_len[i]))); + + /* Check that the encode table decodes correctly. */ + code= 0; + bits= 0; + idx= 0; + DBUG_EXECUTE_IF("forcechkerr1", len--;); + DBUG_EXECUTE_IF("forcechkerr2", bits= 8 * sizeof(code);); + DBUG_EXECUTE_IF("forcechkerr3", idx= length;); + for (;;) + { + if (! len) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: code 0x%s with %u bits not found\n", + hexdigits(huff_tree->code[i]), huff_tree->code_len[i])); + errors++; + break; + } + code<<= 1; + code|= (huff_tree->code[i] >> (--len)) & 1; + bits++; + if (bits > 8 * sizeof(code)) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: Huffman code too long: %u/%u\n", + bits, 8 * sizeof(code))); + errors++; + break; + } + idx+= code & 1; + if (idx >= length) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: illegal tree offset: %u/%u\n", + idx, length)); + errors++; + break; + } + if (packed_tree[idx] & IS_OFFSET) + idx+= packed_tree[idx] & ~IS_OFFSET; + else + break; /* Hit a leaf. This contains the result value. */ + } + if (errors) + break; + + DBUG_EXECUTE_IF("forcechkerr4", packed_tree[idx]++;); + if (packed_tree[idx] != i) + { + VOID(fflush(stdout)); + VOID(fprintf(stderr, "error: decoded value 0x%04x should be: 0x%04x\n", + packed_tree[idx], i)); + errors++; + break; + } + } /*end for (codes)*/ + if (errors) + break; + + /* Write column values in case of distinct column value compression. */ if (huff_tree->counts->tree_buff) { for (i=0 ; i < int_length ; i++) - write_bits((uint) (uchar) huff_tree->counts->tree_buff[i],8); + { + write_bits((ulonglong) (uchar) huff_tree->counts->tree_buff[i], 8); + DBUG_PRINT("info", ("column_values[0x%04x]: 0x%02x", + i, (uchar) huff_tree->counts->tree_buff[i])); + if (verbose >= 3) + VOID(printf("column_values[0x%04x]: 0x%02x\n", + i, (uchar) huff_tree->counts->tree_buff[i])); + } } flush_bits(); } + DBUG_PRINT("info", ("")); + if (verbose >= 2) + VOID(printf("\n")); my_afree((gptr) packed_tree); + if (errors) + { + VOID(fprintf(stderr, "Error: Generated decode trees are corrupt. Stop.\n")); + return 0; + } return elements; } @@ -1613,23 +2331,43 @@ static uint *make_offset_code_tree(HUFF_TREE *huff_tree, HUFF_ELEMENT *element, uint *prev_offset; prev_offset= offset; + /* + 'a.leaf.null' takes the same place as 'a.nod.left'. If this is null, + then there is no left child and, hence no right child either. This + is a property of a binary tree. An element is either a node with two + childs, or a leaf without childs. + + The current element is always a node with two childs. Go left first. + */ if (!element->a.nod.left->a.leaf.null) { - offset[0] =(uint) element->a.nod.left->a.leaf.element_nr; + /* Store the byte code or the index of the column value. */ + prev_offset[0] =(uint) element->a.nod.left->a.leaf.element_nr; offset+=2; } else { + /* + Recursively traverse the tree to the left. Mark it as an offset to + another tree node (in contrast to a byte code or column value index). + */ prev_offset[0]= IS_OFFSET+2; offset=make_offset_code_tree(huff_tree,element->a.nod.left,offset+2); } + + /* Now, check the right child. */ if (!element->a.nod.right->a.leaf.null) { + /* Store the byte code or the index of the column value. */ prev_offset[1]=element->a.nod.right->a.leaf.element_nr; return offset; } else { + /* + Recursively traverse the tree to the right. Mark it as an offset to + another tree node (in contrast to a byte code or column value index). + */ uint temp=(uint) (offset-prev_offset-1); prev_offset[1]= IS_OFFSET+ temp; if (huff_tree->max_offset < temp) @@ -1656,6 +2394,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) uint i,max_calc_length,pack_ref_length,min_record_length,max_record_length, intervall,field_length,max_pack_length,pack_blob_length; my_off_t record_count; + char llbuf[32]; ulong length,pack_length; byte *record,*pos,*end_pos,*record_pos,*start_pos; HUFF_COUNTS *count,*end_count; @@ -1663,12 +2402,23 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) MI_INFO *isam_file=mrg->file[0]; DBUG_ENTER("compress_isam_file"); + /* Allocate a buffer for the records (excluding blobs). */ if (!(record=(byte*) my_alloca(isam_file->s->base.reclength))) return -1; + end_count=huff_counts+isam_file->s->base.fields; min_record_length= (uint) ~0; max_record_length=0; + /* + Calculate the maximum number of bits required to pack the records. + Remember to understand 'max_zero_fill' as 'min_zero_fill'. + The tree height determines the maximum number of bits per value. + Some fields skip leading or trailing spaces or zeroes. The skipped + number of bytes is encoded by 'length_bits' bits. + Empty blobs and varchar are encoded with a single 1 bit. Other blobs + and varchar get a leading 0 bit. + */ for (i=max_calc_length=0 ; i < isam_file->s->base.fields ; i++) { if (!(huff_counts[i].pack_type & PACK_TYPE_ZERO_FILL)) @@ -1687,14 +2437,16 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) (huff_counts[i].field_length - huff_counts[i].max_zero_fill)* huff_counts[i].tree->height+huff_counts[i].length_bits; } - max_calc_length/=8; + max_calc_length= (max_calc_length + 7) / 8; if (max_calc_length < 254) pack_ref_length=1; else if (max_calc_length <= 65535) pack_ref_length=3; else pack_ref_length=4; + record_count=0; + /* 'max_blob_length' is the max length of all blobs of a record. */ pack_blob_length=0; if (isam_file->s->base.blobs) { @@ -1707,6 +2459,7 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) } max_pack_length=pack_ref_length+pack_blob_length; + DBUG_PRINT("fields", ("===")); mrg_reset(mrg); while ((error=mrg_rrnd(mrg,record)) != HA_ERR_END_OF_FILE) { @@ -1722,15 +2475,29 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) end_pos=start_pos+(field_length=count->field_length); tree=count->tree; + DBUG_PRINT("fields", ("column: %3lu type: %2u pack: %2u zero: %4u " + "lbits: %2u tree: %2u length: %4u", + (ulong) (count - huff_counts + 1), + count->field_type, + count->pack_type, count->max_zero_fill, + count->length_bits, count->tree->tree_number, + count->field_length)); + + /* Check if the column contains spaces only. */ if (count->pack_type & PACK_TYPE_SPACE_FIELDS) { for (pos=start_pos ; *pos == ' ' && pos < end_pos; pos++) ; if (pos == end_pos) { + DBUG_PRINT("fields", + ("PACK_TYPE_SPACE_FIELDS spaces only, bits: 1")); + DBUG_PRINT("fields", ("---")); write_bits(1,1); start_pos=end_pos; continue; } + DBUG_PRINT("fields", + ("PACK_TYPE_SPACE_FIELDS not only spaces, bits: 1")); write_bits(0,1); } end_pos-=count->max_zero_fill; @@ -1740,65 +2507,129 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) case FIELD_SKIP_ZERO: if (!memcmp((byte*) start_pos,zero_string,field_length)) { + DBUG_PRINT("fields", ("FIELD_SKIP_ZERO zeroes only, bits: 1")); write_bits(1,1); start_pos=end_pos; break; } + DBUG_PRINT("fields", ("FIELD_SKIP_ZERO not only zeroes, bits: 1")); write_bits(0,1); /* Fall through */ case FIELD_NORMAL: + DBUG_PRINT("fields", ("FIELD_NORMAL %lu bytes", + (ulong) (end_pos - start_pos))); for ( ; start_pos < end_pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } break; case FIELD_SKIP_ENDSPACE: for (pos=end_pos ; pos > start_pos && pos[-1] == ' ' ; pos--) ; - length=(uint) (end_pos-pos); + length= (ulong) (end_pos - pos); if (count->pack_type & PACK_TYPE_SELECTED) { if (length > count->min_space) { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE more than min_space, bits: 1")); + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(1,1); write_bits(length,count->length_bits); } else { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE not more than min_space, " + "bits: 1")); write_bits(0,1); pos=end_pos; } } else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_ENDSPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(length,count->length_bits); + } + /* Encode all significant bytes. */ + DBUG_PRINT("fields", ("FIELD_SKIP_ENDSPACE %lu bytes", + (ulong) (pos - start_pos))); for ( ; start_pos < pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } start_pos=end_pos; break; case FIELD_SKIP_PRESPACE: for (pos=start_pos ; pos < end_pos && pos[0] == ' ' ; pos++) ; - length=(uint) (pos-start_pos); + length= (ulong) (pos - start_pos); if (count->pack_type & PACK_TYPE_SELECTED) { if (length > count->min_space) { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE more than min_space, bits: 1")); + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(1,1); write_bits(length,count->length_bits); } else { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE not more than min_space, " + "bits: 1")); pos=start_pos; write_bits(0,1); } } else + { + DBUG_PRINT("fields", + ("FIELD_SKIP_PRESPACE skip %lu/%u bytes, bits: %2u", + length, field_length, count->length_bits)); write_bits(length,count->length_bits); + } + /* Encode all significant bytes. */ + DBUG_PRINT("fields", ("FIELD_SKIP_PRESPACE %lu bytes", + (ulong) (end_pos - start_pos))); for (start_pos=pos ; start_pos < end_pos ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint) tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } break; case FIELD_CONSTANT: case FIELD_ZERO: case FIELD_CHECK: + DBUG_PRINT("fields", ("FIELD_CONSTANT/ZERO/CHECK")); start_pos=end_pos; break; case FIELD_INTERVALL: @@ -1806,6 +2637,10 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) pos=(byte*) tree_search(&count->int_tree, start_pos, count->int_tree.custom_arg); intervall=(uint) (pos - count->tree_buff)/field_length; + DBUG_PRINT("fields", ("FIELD_INTERVALL")); + DBUG_PRINT("fields", ("index: %4u code: 0x%s bits: %2u", + intervall, hexdigits(tree->code[intervall]), + (uint) tree->code_len[intervall])); write_bits(tree->code[intervall],(uint) tree->code_len[intervall]); start_pos=end_pos; break; @@ -1814,21 +2649,36 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) ulong blob_length=_mi_calc_blob_length(field_length- mi_portable_sizeof_char_ptr, start_pos); + /* Empty blobs are encoded with a single 1 bit. */ if (!blob_length) { - write_bits(1,1); /* Empty blob */ + DBUG_PRINT("fields", ("FIELD_BLOB empty, bits: 1")); + write_bits(1,1); } else { byte *blob,*blob_end; + DBUG_PRINT("fields", ("FIELD_BLOB not empty, bits: 1")); write_bits(0,1); + /* Write the blob length. */ + DBUG_PRINT("fields", ("FIELD_BLOB %lu bytes, bits: %2u", + blob_length, count->length_bits)); write_bits(blob_length,count->length_bits); memcpy_fixed(&blob,end_pos-mi_portable_sizeof_char_ptr, sizeof(char*)); blob_end=blob+blob_length; + /* Encode the blob bytes. */ for ( ; blob < blob_end ; blob++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *blob, hexdigits(tree->code[(uchar) *blob]), + (uint) tree->code_len[(uchar) *blob], + bindigits(tree->code[(uchar) *start_pos], + (uint)tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *blob], (uint) tree->code_len[(uchar) *blob]); + } tot_blob_length+=blob_length; } start_pos= end_pos; @@ -1839,18 +2689,34 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) uint pack_length= HA_VARCHAR_PACKLENGTH(count->field_length-1); ulong col_length= (pack_length == 1 ? (uint) *(uchar*) start_pos : uint2korr(start_pos)); + /* Empty varchar are encoded with a single 1 bit. */ if (!col_length) { + DBUG_PRINT("fields", ("FIELD_VARCHAR empty, bits: 1")); write_bits(1,1); /* Empty varchar */ } else { byte *end=start_pos+pack_length+col_length; + DBUG_PRINT("fields", ("FIELD_VARCHAR not empty, bits: 1")); write_bits(0,1); + /* Write the varchar length. */ + DBUG_PRINT("fields", ("FIELD_VARCHAR %lu bytes, bits: %2u", + col_length, count->length_bits)); write_bits(col_length,count->length_bits); + /* Encode the varchar bytes. */ for (start_pos+=pack_length ; start_pos < end ; start_pos++) + { + DBUG_PRINT("fields", + ("value: 0x%02x code: 0x%s bits: %2u bin: %s", + (uchar) *start_pos, + hexdigits(tree->code[(uchar) *start_pos]), + (uint) tree->code_len[(uchar) *start_pos], + bindigits(tree->code[(uchar) *start_pos], + (uint)tree->code_len[(uchar) *start_pos]))); write_bits(tree->code[(uchar) *start_pos], (uint) tree->code_len[(uchar) *start_pos]); + } } start_pos= end_pos; break; @@ -1859,12 +2725,17 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) abort(); /* Impossible */ } start_pos+=count->max_zero_fill; + DBUG_PRINT("fields", ("---")); } flush_bits(); - length=(ulong) (file_buffer.pos-record_pos)-max_pack_length; + length=(ulong) ((byte*) file_buffer.pos - record_pos) - max_pack_length; pack_length=save_pack_length(record_pos,length); if (pack_blob_length) pack_length+=save_pack_length(record_pos+pack_length,tot_blob_length); + DBUG_PRINT("fields", ("record: %lu length: %lu blob-length: %lu " + "length-bytes: %lu", (ulong) record_count, length, + tot_blob_length, pack_length)); + DBUG_PRINT("fields", ("===")); /* Correct file buffer if the header was smaller */ if (pack_length != max_pack_length) @@ -1876,9 +2747,11 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) min_record_length=(uint) length; if (length > (ulong) max_record_length) max_record_length=(uint) length; - if (write_loop && ++record_count % WRITE_COUNT == 0) + record_count++; + if (write_loop && record_count % WRITE_COUNT == 0) { - printf("%lu\r",(ulong) record_count); VOID(fflush(stdout)); + VOID(printf("%lu\r", (ulong) record_count)); + VOID(fflush(stdout)); } } else if (error != HA_ERR_RECORD_DELETED) @@ -1888,8 +2761,11 @@ static int compress_isam_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) error=0; else { - fprintf(stderr,"%s: Got error %d reading records\n",my_progname,error); + VOID(fprintf(stderr, "%s: Got error %d reading records\n", + my_progname, error)); } + if (verbose >= 2) + VOID(printf("wrote %s records.\n", llstr((longlong) record_count, llbuf))); my_afree((gptr) record); mrg->ref_length=max_pack_length; @@ -1929,7 +2805,7 @@ static void init_file_buffer(File file, pbool read_buffer) file_buffer.pos=file_buffer.buffer; file_buffer.bits=BITS_SAVED; } - file_buffer.current_byte=0; + file_buffer.bitbucket= 0; } @@ -1972,7 +2848,8 @@ static int flush_buffer(ulong neaded_length) tmp=my_realloc(file_buffer.buffer, neaded_length,MYF(MY_WME)); if (!tmp) return 1; - file_buffer.pos= tmp + (ulong) (file_buffer.pos - file_buffer.buffer); + file_buffer.pos= ((uchar*) tmp + + (ulong) (file_buffer.pos - file_buffer.buffer)); file_buffer.buffer=tmp; file_buffer.end=tmp+neaded_length-8; } @@ -1987,68 +2864,59 @@ static void end_file_buffer(void) /* output `bits` low bits of `value' */ -static void write_bits (register ulong value, register uint bits) +static void write_bits(register ulonglong value, register uint bits) { - if ((file_buffer.bits-=(int) bits) >= 0) + DBUG_ASSERT(((bits < 8 * sizeof(value)) && ! (value >> bits)) || + (bits == 8 * sizeof(value))); + + if ((file_buffer.bits-= (int) bits) >= 0) { - file_buffer.current_byte|=value << file_buffer.bits; + file_buffer.bitbucket|= value << file_buffer.bits; } else { - reg3 uint byte_buff; + reg3 ulonglong bit_buffer; bits= (uint) -file_buffer.bits; - DBUG_ASSERT(bits <= 8 * sizeof(value)); - byte_buff= (file_buffer.current_byte | - ((bits != 8 * sizeof(value)) ? (uint) (value >> bits) : 0)); -#if BITS_SAVED == 32 - *file_buffer.pos++= (byte) (byte_buff >> 24) ; - *file_buffer.pos++= (byte) (byte_buff >> 16) ; + bit_buffer= (file_buffer.bitbucket | + ((bits != 8 * sizeof(value)) ? (value >> bits) : 0)); +#if BITS_SAVED == 64 + *file_buffer.pos++= (uchar) (bit_buffer >> 56); + *file_buffer.pos++= (uchar) (bit_buffer >> 48); + *file_buffer.pos++= (uchar) (bit_buffer >> 40); + *file_buffer.pos++= (uchar) (bit_buffer >> 32); #endif - *file_buffer.pos++= (byte) (byte_buff >> 8) ; - *file_buffer.pos++= (byte) byte_buff; + *file_buffer.pos++= (uchar) (bit_buffer >> 24); + *file_buffer.pos++= (uchar) (bit_buffer >> 16); + *file_buffer.pos++= (uchar) (bit_buffer >> 8); + *file_buffer.pos++= (uchar) (bit_buffer); - DBUG_ASSERT(bits <= 8 * sizeof(ulong)); if (bits != 8 * sizeof(value)) - value&= (((ulong) 1) << bits) - 1; -#if BITS_SAVED == 16 - if (bits >= sizeof(uint)) - { - bits-=8; - *file_buffer.pos++= (uchar) (value >> bits); - value&= (1 << bits)-1; - if (bits >= sizeof(uint)) - { - bits-=8; - *file_buffer.pos++= (uchar) (value >> bits); - value&= (1 << bits)-1; - } - } -#endif + value&= (((ulonglong) 1) << bits) - 1; if (file_buffer.pos >= file_buffer.end) VOID(flush_buffer(~ (ulong) 0)); file_buffer.bits=(int) (BITS_SAVED - bits); - file_buffer.current_byte=(uint) (value << (BITS_SAVED - bits)); + file_buffer.bitbucket= value << (BITS_SAVED - bits); } return; } /* Flush bits in bit_buffer to buffer */ -static void flush_bits (void) +static void flush_bits(void) { - uint bits,byte_buff; + int bits; + ulonglong bit_buffer; - bits=(file_buffer.bits) & ~7; - byte_buff = file_buffer.current_byte >> bits; - bits=BITS_SAVED - bits; + bits= file_buffer.bits & ~7; + bit_buffer= file_buffer.bitbucket >> bits; + bits= BITS_SAVED - bits; while (bits > 0) { - bits-=8; - *file_buffer.pos++= (byte) (uchar) (byte_buff >> bits) ; + bits-= 8; + *file_buffer.pos++= (uchar) (bit_buffer >> bits); } - file_buffer.bits=BITS_SAVED; - file_buffer.current_byte=0; - return; + file_buffer.bits= BITS_SAVED; + file_buffer.bitbucket= 0; } @@ -2196,3 +3064,131 @@ static int mrg_close(PACK_MRG_INFO *mrg) my_free((gptr) mrg->file,MYF(0)); return error; } + + +#if !defined(DBUG_OFF) +/* + Fake the counts to get big Huffman codes. + + SYNOPSIS + fakebigcodes() + huff_counts A pointer to the counts array. + end_count A pointer past the counts array. + + DESCRIPTION + + Huffman coding works by removing the two least frequent values from + the list of values and add a new value with the sum of their + incidences in a loop until only one value is left. Every time a + value is reused for a new value, it gets one more bit for its + encoding. Hence, the least frequent values get the longest codes. + + To get a maximum code length for a value, two of the values must + have an incidence of 1. As their sum is 2, the next infrequent value + must have at least an incidence of 2, then 4, 8, 16 and so on. This + means that one needs 2**n bytes (values) for a code length of n + bits. However, using more distinct values forces the use of longer + codes, or reaching the code length with less total bytes (values). + + To get 64(32)-bit codes, I sort the counts by decreasing incidence. + I assign counts of 1 to the two most frequent values, a count of 2 + for the next one, then 4, 8, and so on until 2**64-1(2**30-1). All + the remaining values get 1. That way every possible byte has an + assigned code, though not all codes are used if not all byte values + are present in the column. + + This strategy would work with distinct column values too, but + requires that at least 64(32) values are present. To make things + easier here, I cancel all distinct column values and force byte + compression for all columns. + + RETURN + void +*/ + +static void fakebigcodes(HUFF_COUNTS *huff_counts, HUFF_COUNTS *end_count) +{ + HUFF_COUNTS *count; + my_off_t *cur_count_p; + my_off_t *end_count_p; + my_off_t **cur_sort_p; + my_off_t **end_sort_p; + my_off_t *sort_counts[256]; + my_off_t total; + DBUG_ENTER("fakebigcodes"); + + for (count= huff_counts; count < end_count; count++) + { + /* + Remove distinct column values. + */ + if (huff_counts->tree_buff) + { + my_free((gptr) huff_counts->tree_buff, MYF(0)); + delete_tree(&huff_counts->int_tree); + huff_counts->tree_buff= NULL; + DBUG_PRINT("fakebigcodes", ("freed distinct column values")); + } + + /* + Sort counts by decreasing incidence. + */ + cur_count_p= count->counts; + end_count_p= cur_count_p + 256; + cur_sort_p= sort_counts; + while (cur_count_p < end_count_p) + *(cur_sort_p++)= cur_count_p++; + (void) qsort(sort_counts, 256, sizeof(my_off_t*), (qsort_cmp) fakecmp); + + /* + Assign faked counts. + */ + cur_sort_p= sort_counts; +#if SIZEOF_LONG_LONG > 4 + end_sort_p= sort_counts + 8 * sizeof(ulonglong) - 1; +#else + end_sort_p= sort_counts + 8 * sizeof(ulonglong) - 2; +#endif + /* Most frequent value gets a faked count of 1. */ + **(cur_sort_p++)= 1; + total= 1; + while (cur_sort_p < end_sort_p) + { + **(cur_sort_p++)= total; + total<<= 1; + } + /* Set the last value. */ + **(cur_sort_p++)= --total; + /* + Set the remaining counts. + */ + end_sort_p= sort_counts + 256; + while (cur_sort_p < end_sort_p) + **(cur_sort_p++)= 1; + } + DBUG_VOID_RETURN; +} + + +/* + Compare two counts for reverse sorting. + + SYNOPSIS + fakecmp() + count1 One count. + count2 Another count. + + RETURN + 1 count1 < count2 + 0 count1 == count2 + -1 count1 > count2 +*/ + +static int fakecmp(my_off_t **count1, my_off_t **count2) +{ + return ((**count1 < **count2) ? 1 : + (**count1 > **count2) ? -1 : 0); +} +#endif + + diff --git a/mysql-test/README b/mysql-test/README index 65e6186613a..10d64784ed4 100644 --- a/mysql-test/README +++ b/mysql-test/README @@ -2,8 +2,17 @@ This directory contains a test suite for mysql daemon. To run the currently existing test cases, simply execute ./mysql-test-run in this directory. It will fire up the newly built mysqld and test it. -If you want to run the test with a running MySQL server use the --external -option to mysql-test-run. +If you want to run a test with a running MySQL server use the --extern +option to mysql-test-run. Please note that in this mode the test suite +expects user to specify test names to run. Otherwise it falls back to the +normal "non-extern" behaviour. The reason is that some tests +could not run with external server. Here is the sample command +to test "alias" and "analyze" tests on external server: + +mysql-test-run --extern alias analyze + +To match your setup you might also need to provide --socket, --user and +other relevant options. Note that you do not have to have to do make install, and you could actually have a co-existing MySQL installation - the tests will not diff --git a/mysql-test/lib/mtr_cases.pl b/mysql-test/lib/mtr_cases.pl index 72cbe72bc0a..2babaabc24c 100644 --- a/mysql-test/lib/mtr_cases.pl +++ b/mysql-test/lib/mtr_cases.pl @@ -154,6 +154,14 @@ sub collect_one_test_case($$$$$) { } } + if ( defined mtr_match_prefix($tname,"federated") ) + { + $tinfo->{'slave_num'}= 1; # Default, use one slave + + # FIXME currently we always restart slaves + $tinfo->{'slave_restart'}= 1; + } + # FIXME what about embedded_server + ndbcluster, skip ?! my $master_opt_file= "$testdir/$tname-master.opt"; diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 6e2fb50d988..74d1a8dc6f2 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -303,7 +303,7 @@ sub mysqld_arguments ($$$$$); sub stop_masters_slaves (); sub stop_masters (); sub stop_slaves (); -sub run_mysqltest ($$); +sub run_mysqltest ($); sub usage ($); ###################################################################### @@ -1342,10 +1342,11 @@ sub run_testcase ($) { if ( ! $glob_use_running_server and ! $glob_use_embedded_server ) { - if ( $tinfo->{'master_restart'} or $master->[0]->{'uses_special_flags'} ) + if ( $tinfo->{'master_restart'} or + $master->[0]->{'running_master_is_special'} ) { stop_masters(); - $master->[0]->{'uses_special_flags'}= 0; # Forget about why we stopped + $master->[0]->{'running_master_is_special'}= 0; # Forget why we stopped } # ---------------------------------------------------------------------- @@ -1423,9 +1424,9 @@ sub run_testcase ($) { } } - if ( @{$tinfo->{'master_opt'}} ) + if ( $tinfo->{'master_restart'} ) { - $master->[0]->{'uses_special_flags'}= 1; + $master->[0]->{'running_master_is_special'}= 1; } } @@ -1472,7 +1473,7 @@ sub run_testcase ($) { } unlink($path_timefile); - my $res= run_mysqltest($tinfo, $tinfo->{'master_opt'}); + my $res= run_mysqltest($tinfo); if ( $res == 0 ) { @@ -1553,11 +1554,12 @@ sub do_before_start_master ($$) { } } + # FIXME only remove the ones that are tied to this master # Remove old master.info and relay-log.info files - unlink("$opt_vardir/master-data/master.info"); - unlink("$opt_vardir/master-data/relay-log.info"); - unlink("$opt_vardir/master1-data/master.info"); - unlink("$opt_vardir/master1-data/relay-log.info"); + unlink("$master->[0]->{'path_myddir'}/master.info"); + unlink("$master->[0]->{'path_myddir'}/relay-log.info"); + unlink("$master->[1]->{'path_myddir'}/master.info"); + unlink("$master->[1]->{'path_myddir'}/relay-log.info"); # Run master initialization shell script if one exists if ( $init_script ) @@ -1651,12 +1653,15 @@ sub mysqld_arguments ($$$$$) { if ( $type eq 'master' ) { - mtr_add_arg($args, "%s--log-bin=%s/log/master-bin", $prefix, $opt_vardir); + my $id= $idx > 0 ? $idx + 101 : 1; + + mtr_add_arg($args, "%s--log-bin=%s/log/master-bin%s", $prefix, + $opt_vardir, $sidx); mtr_add_arg($args, "%s--pid-file=%s", $prefix, $master->[$idx]->{'path_mypid'}); mtr_add_arg($args, "%s--port=%d", $prefix, $master->[$idx]->{'path_myport'}); - mtr_add_arg($args, "%s--server-id=1", $prefix); + mtr_add_arg($args, "%s--server-id=%d", $prefix, $id); mtr_add_arg($args, "%s--socket=%s", $prefix, $master->[$idx]->{'path_mysock'}); mtr_add_arg($args, "%s--innodb_data_file_path=ibdata1:128M:autoextend", $prefix); @@ -1664,6 +1669,11 @@ sub mysqld_arguments ($$$$$) { mtr_add_arg($args, "%s--datadir=%s", $prefix, $master->[$idx]->{'path_myddir'}); + if ( $idx > 0 ) + { + mtr_add_arg($args, "%s--skip-innodb", $prefix); + } + if ( $opt_skip_ndbcluster ) { mtr_add_arg($args, "%s--skip-ndbcluster", $prefix); @@ -1673,7 +1683,7 @@ sub mysqld_arguments ($$$$$) { if ( $type eq 'slave' ) { my $slave_server_id= 2 + $idx; - my $slave_rpl_rank= $idx > 0 ? 2 : $slave_server_id; + my $slave_rpl_rank= $slave_server_id; mtr_add_arg($args, "%s--datadir=%s", $prefix, $slave->[$idx]->{'path_myddir'}); @@ -1973,9 +1983,8 @@ sub stop_slaves () { } -sub run_mysqltest ($$) { +sub run_mysqltest ($) { my $tinfo= shift; - my $master_opts= shift; my $cmdline_mysqldump= "$exe_mysqldump --no-defaults -uroot " . "--socket=$master->[0]->{'path_mysock'} --password="; diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh index e3fcfdf2b1e..376946eb8aa 100644 --- a/mysql-test/mysql-test-run.sh +++ b/mysql-test/mysql-test-run.sh @@ -307,7 +307,7 @@ while test $# -gt 0; do --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem \ --ssl-cert=$MYSQL_TEST_DIR/std_data/server-cert.pem \ --ssl-key=$MYSQL_TEST_DIR/std_data/server-key.pem" - MYSQL_TEST_SSL_OPTS="--ssl-ca=$BASEDIR/SSL/cacert.pem \ + MYSQL_TEST_SSL_OPTS="--ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem \ --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem \ --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem" ;; --no-manager | --skip-manager) USE_MANAGER=0 ;; diff --git a/mysql-test/r/case.result b/mysql-test/r/case.result index a854cf4c7b0..b02f85132aa 100644 --- a/mysql-test/r/case.result +++ b/mysql-test/r/case.result @@ -160,6 +160,21 @@ t1 CREATE TABLE `t1` ( `COALESCE('a' COLLATE latin1_bin,'b')` varchar(1) character set latin1 collate latin1_bin NOT NULL default '' ) ENGINE=MyISAM DEFAULT CHARSET=latin1 DROP TABLE t1; +SELECT 'case+union+test' +UNION +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; +case+union+test +case+union+test +nobug +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; +CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END +nobug +SELECT 'case+union+test' +UNION +SELECT CASE '1' WHEN '2' THEN 'BUG' ELSE 'nobug' END; +case+union+test +case+union+test +nobug CREATE TABLE t1 (EMPNUM INT); INSERT INTO t1 VALUES (0), (2); CREATE TABLE t2 (EMPNUM DECIMAL (4, 2)); diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result index 1c6a4393dfc..92ec76fb024 100644 --- a/mysql-test/r/func_str.result +++ b/mysql-test/r/func_str.result @@ -800,3 +800,14 @@ field(0,NULL,1,0) field("",NULL,"bar","") field(0.0,NULL,1.0,0.0) select field(NULL,1,2,NULL), field(NULL,1,2,0); field(NULL,1,2,NULL) field(NULL,1,2,0) 0 0 +CREATE TABLE t1 (str varchar(20) PRIMARY KEY); +CREATE TABLE t2 (num int primary key); +INSERT INTO t1 VALUES ('notnumber'); +INSERT INTO t2 VALUES (0), (1); +SELECT * FROM t1, t2 WHERE num=str; +str num +notnumber 0 +SELECT * FROM t1, t2 WHERE num=substring(str from 1 for 6); +str num +notnumber 0 +DROP TABLE t1,t2; diff --git a/mysql-test/r/heap.result b/mysql-test/r/heap.result index 22304c4a93d..b905dae3aba 100644 --- a/mysql-test/r/heap.result +++ b/mysql-test/r/heap.result @@ -367,13 +367,13 @@ count(*) 9 explain select count(*) from t1 where v='a '; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref v v 13 const 9 Using where +1 SIMPLE t1 ref v v 13 const 10 Using where explain select count(*) from t1 where c='a '; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref c c 11 const 9 Using where +1 SIMPLE t1 ref c c 11 const 10 Using where explain select count(*) from t1 where t='a '; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref t t 13 const 9 Using where +1 SIMPLE t1 ref t t 13 const 10 Using where explain select count(*) from t1 where v like 'a%'; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL v NULL NULL NULL 271 Using where @@ -399,7 +399,7 @@ qq *a *a*a * explain select * from t1 where v='a'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref v v 13 const 9 Using where +1 SIMPLE t1 ref v v 13 const 10 Using where select v,count(*) from t1 group by v limit 10; v count(*) a 1 diff --git a/mysql-test/r/heap_hash.result b/mysql-test/r/heap_hash.result index 9720fe4843a..d8d89b786b5 100644 --- a/mysql-test/r/heap_hash.result +++ b/mysql-test/r/heap_hash.result @@ -231,18 +231,19 @@ explain select * from t1 where a='aaad'; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref a a 8 const 1 Using where insert into t1 select * from t1; +flush tables; explain select * from t1 where a='aaaa'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaab'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaac'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaad'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where flush tables; explain select * from t1 where a='aaaa'; id select_type table type possible_keys key key_len ref rows Extra @@ -261,16 +262,16 @@ delete from t1; insert into t1 select * from t2; explain select * from t1 where a='aaaa'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaab'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaac'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where explain select * from t1 where a='aaad'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref a a 8 const 1 Using where +1 SIMPLE t1 ref a a 8 const 2 Using where drop table t1, t2; create table t1 ( id int unsigned not null primary key auto_increment, @@ -345,15 +346,15 @@ insert into t3 select name, name from t1; show index from t3; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t3 1 a 1 a NULL NULL NULL NULL HASH -t3 1 a 2 b NULL 15 NULL NULL HASH +t3 1 a 2 b NULL 13 NULL NULL HASH show index from t3; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t3 1 a 1 a NULL NULL NULL NULL HASH -t3 1 a 2 b NULL 15 NULL NULL HASH +t3 1 a 2 b NULL 13 NULL NULL HASH explain select * from t1 ignore key(btree_idx), t3 where t1.name='matt' and t3.a = concat('',t1.name) and t3.b=t1.name; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 ref a a 44 const,const 6 Using where 1 SIMPLE t1 ref heap_idx heap_idx 22 const 7 Using where +1 SIMPLE t3 ref a a 44 const,const 7 Using where drop table t1, t2, t3; create temporary table t1 ( a int, index (a) ) engine=memory; insert into t1 values (1),(2),(3),(4),(5); diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 52e2a4046cf..662fffe1ba1 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -123,3 +123,14 @@ key1a = 2 and key1b is null and key3a = 2 and key3b is null; count(*) 4 drop table t1,t2; +create table t1 ( +id1 int, +id2 date , +index idx2 (id1,id2), +index idx1 (id2) +) engine = innodb; +insert into t1 values(1,'20040101'), (2,'20040102'); +select * from t1 where id1 = 1 and id2= '20040101'; +id1 id2 +1 2004-01-01 +drop table t1; diff --git a/mysql-test/r/innodb.result b/mysql-test/r/innodb.result index b07a6b03c8a..838e02c3a89 100644 --- a/mysql-test/r/innodb.result +++ b/mysql-test/r/innodb.result @@ -1721,12 +1721,13 @@ count(*) 0 explain select count(*) from t1 where x > -16; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY PRIMARY 8 NULL 1 Using where; Using index +1 SIMPLE t1 index PRIMARY PRIMARY 8 NULL 2 Using where; Using index select count(*) from t1 where x > -16; count(*) -1 +2 select * from t1 where x > -16; x +18446744073709551600 18446744073709551601 select count(*) from t1 where x = 18446744073709551601; count(*) diff --git a/mysql-test/r/insert_select.result b/mysql-test/r/insert_select.result index 11384b0feff..026dae8381a 100644 --- a/mysql-test/r/insert_select.result +++ b/mysql-test/r/insert_select.result @@ -634,3 +634,18 @@ ff1 ff2 1 2 2 1 drop table t1, t2; +create table t1 (a int unique); +create table t2 (a int, b int); +insert into t1 values (1),(2); +insert into t2 values (1,2); +select * from t1; +a +1 +2 +insert into t1 select t2.a from t2 on duplicate key update a= a + t2.b; +select * from t1; +a +2 +3 +drop table t1; +drop table t2; diff --git a/mysql-test/r/insert_update.result b/mysql-test/r/insert_update.result index 6d3aa941c8c..739beea6286 100644 --- a/mysql-test/r/insert_update.result +++ b/mysql-test/r/insert_update.result @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t1, t2; CREATE TABLE t1 (a INT, b INT, c INT, UNIQUE (A), UNIQUE(B)); INSERT t1 VALUES (1,2,10), (3,4,20); INSERT t1 VALUES (5,6,30) ON DUPLICATE KEY UPDATE c=c+100; diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index c7d27a8e60d..ae7d0474e24 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -549,6 +549,66 @@ select count(*) from t2 where x = 18446744073709551601; count(*) 0 drop table t1,t2; +create table t1 (x bigint unsigned not null primary key) engine=innodb; +insert into t1(x) values (0xfffffffffffffff0); +insert into t1(x) values (0xfffffffffffffff1); +select * from t1; +x +18446744073709551600 +18446744073709551601 +select count(*) from t1 where x>0; +count(*) +2 +select count(*) from t1 where x=0; +count(*) +0 +select count(*) from t1 where x<0; +count(*) +0 +select count(*) from t1 where x < -16; +count(*) +0 +select count(*) from t1 where x = -16; +count(*) +0 +select count(*) from t1 where x > -16; +count(*) +2 +select count(*) from t1 where x = 18446744073709551601; +count(*) +1 +drop table t1; +create table t1 (a bigint unsigned); +create index t1i on t1(a); +insert into t1 select 18446744073709551615; +insert into t1 select 18446744073709551614; +explain select * from t1 where a <> -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a <> -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a > -1 or a < -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a > -1 or a < -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a > -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a > -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a < -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +select * from t1 where a < -1; +a +drop table t1; set names latin1; create table t1 (a char(10), b text, key (a)) character set latin1; INSERT INTO t1 (a) VALUES diff --git a/mysql-test/r/rpl_multi_update3.result b/mysql-test/r/rpl_multi_update3.result index 708b230b19f..1b757b1400c 100644 --- a/mysql-test/r/rpl_multi_update3.result +++ b/mysql-test/r/rpl_multi_update3.result @@ -4,6 +4,8 @@ reset master; reset slave; drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; start slave; + +-------- Test for BUG#9361 -------- CREATE TABLE t1 ( a int unsigned not null auto_increment primary key, b int unsigned @@ -41,3 +43,82 @@ a b 1 6 2 6 drop table t1,t2; + +-------- Test 1 for BUG#9361 -------- +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +CREATE TABLE t1 ( +a1 char(30), +a2 int, +a3 int, +a4 char(30), +a5 char(30) +); +CREATE TABLE t2 ( +b1 int, +b2 char(30) +); +INSERT INTO t1 VALUES ('Yes', 1, NULL, 'foo', 'bar'); +INSERT INTO t2 VALUES (1, 'baz'); +UPDATE t1 a, t2 +SET a.a1 = 'No' +WHERE a.a2 = +(SELECT b1 +FROM t2 +WHERE b2 = 'baz') +AND a.a3 IS NULL +AND a.a4 = 'foo' +AND a.a5 = 'bar'; +SELECT * FROM t1; +a1 a2 a3 a4 a5 +No 1 NULL foo bar +SELECT * FROM t2; +b1 b2 +1 baz +DROP TABLE t1, t2; + +-------- Test 2 for BUG#9361 -------- +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +CREATE TABLE t1 ( +i INT, +j INT, +x INT, +y INT, +z INT +); +CREATE TABLE t2 ( +i INT, +k INT, +x INT, +y INT, +z INT +); +CREATE TABLE t3 ( +j INT, +k INT, +x INT, +y INT, +z INT +); +INSERT INTO t1 VALUES ( 1, 2,13,14,15); +INSERT INTO t2 VALUES ( 1, 3,23,24,25); +INSERT INTO t3 VALUES ( 2, 3, 1,34,35), ( 2, 3, 1,34,36); +UPDATE t1 AS a +INNER JOIN t2 AS b +ON a.i = b.i +INNER JOIN t3 AS c +ON a.j = c.j AND b.k = c.k +SET a.x = b.x, +a.y = b.y, +a.z = ( +SELECT sum(z) +FROM t3 +WHERE y = 34 +) +WHERE b.x = 23; +SELECT * FROM t1; +i j x y z +1 2 23 24 71 +DROP TABLE t1, t2, t3; diff --git a/mysql-test/r/sp-security.result b/mysql-test/r/sp-security.result index ee72fde7324..4ace6f59411 100644 --- a/mysql-test/r/sp-security.result +++ b/mysql-test/r/sp-security.result @@ -194,3 +194,20 @@ use test; drop database sptest; delete from mysql.user where user='usera' or user='userb' or user='userc'; delete from mysql.procs_priv where user='usera' or user='userb' or user='userc'; +drop function if exists bug_9503; +create database mysqltest// +use mysqltest// +create table t1 (s1 int)// +grant select on t1 to user1@localhost// +create function bug_9503 () returns int sql security invoker begin declare v int; +select min(s1) into v from t1; return v; end// +use mysqltest; +select bug_9503(); +ERROR 42000: execute command denied to user 'user1'@'localhost' for routine 'mysqltest.bug_9503' +grant execute on function bug_9503 to user1@localhost; +do 1; +use test; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1@localhost; +drop function bug_9503; +use test; +drop database mysqltest; diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 6703147c635..400a2be01f1 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -1025,7 +1025,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 system NULL NULL NULL NULL 0 const row not found 2 UNCACHEABLE SUBQUERY t1 system NULL NULL NULL NULL 0 const row not found Warnings: -Note 1003 select sql_no_cache (select sql_no_cache ecrypt(_latin1'test') AS `ENCRYPT('test')` from `test`.`t1`) AS `(SELECT ENCRYPT('test') FROM t1)` from `test`.`t1` +Note 1003 select sql_no_cache (select sql_no_cache encrypt(_latin1'test') AS `ENCRYPT('test')` from `test`.`t1`) AS `(SELECT ENCRYPT('test') FROM t1)` from `test`.`t1` EXPLAIN EXTENDED SELECT (SELECT BENCHMARK(1,1) FROM t1) FROM t1; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 system NULL NULL NULL NULL 0 const row not found @@ -2816,3 +2816,24 @@ select * from t1; EMPNUM E1 DROP TABLE t1,t2; +CREATE TABLE t1(select_id BIGINT, values_id BIGINT); +INSERT INTO t1 VALUES (1, 1); +CREATE TABLE t2 (select_id BIGINT, values_id BIGINT, +PRIMARY KEY(select_id,values_id)); +INSERT INTO t2 VALUES (0, 1), (0, 2), (0, 3), (1, 5); +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 +WHERE select_id IN (1, 0)); +values_id +1 +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 +WHERE select_id BETWEEN 0 AND 1); +values_id +1 +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 +WHERE select_id = 0 OR select_id = 1); +values_id +1 +DROP TABLE t1, t2; diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 68cc0c4cb57..4c1db618b4b 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -1831,3 +1831,22 @@ select * from v1; t 01:00 drop view v1; +create table t1 (f1 date); +insert into t1 values ('2005-01-01'),('2005-02-02'); +create view v1 as select * from t1; +select * from v1 where f1='2005.02.02'; +f1 +2005-02-02 +select * from v1 where '2005.02.02'=f1; +f1 +2005-02-02 +drop view v1; +drop table t1; +CREATE VIEW v1 AS SELECT ENCRYPT("dhgdhgd"); +SELECT * FROM v1; +drop view v1; +CREATE VIEW v1 AS SELECT SUBSTRING_INDEX("dkjhgd:kjhdjh", ":", 1); +SELECT * FROM v1; +SUBSTRING_INDEX("dkjhgd:kjhdjh", ":", 1) +dkjhgd +drop view v1; diff --git a/mysql-test/t/case.test b/mysql-test/t/case.test index f2cfce9085d..e942333d5fe 100644 --- a/mysql-test/t/case.test +++ b/mysql-test/t/case.test @@ -111,6 +111,17 @@ explain extended SELECT SHOW CREATE TABLE t1; DROP TABLE t1; +# Test for BUG#10151 +SELECT 'case+union+test' +UNION +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; + +SELECT CASE LOWER('1') WHEN LOWER('2') THEN 'BUG' ELSE 'nobug' END; + +SELECT 'case+union+test' +UNION +SELECT CASE '1' WHEN '2' THEN 'BUG' ELSE 'nobug' END; + # # Tests for bug #9939: conversion of the arguments for COALESCE and IFNULL # diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test index 728f0e2c084..e6bf9f81dd0 100644 --- a/mysql-test/t/func_str.test +++ b/mysql-test/t/func_str.test @@ -529,3 +529,17 @@ DROP TABLE t1, t2; # select field(0,NULL,1,0), field("",NULL,"bar",""), field(0.0,NULL,1.0,0.0); select field(NULL,1,2,NULL), field(NULL,1,2,0); + +# +# Bug #10124: access by integer index with a string key that is not a number +# + +CREATE TABLE t1 (str varchar(20) PRIMARY KEY); +CREATE TABLE t2 (num int primary key); +INSERT INTO t1 VALUES ('notnumber'); +INSERT INTO t2 VALUES (0), (1); + +SELECT * FROM t1, t2 WHERE num=str; +SELECT * FROM t1, t2 WHERE num=substring(str from 1 for 6); + +DROP TABLE t1,t2; diff --git a/mysql-test/t/heap_hash.test b/mysql-test/t/heap_hash.test index 46669dd2b8f..21ac69ef3a1 100644 --- a/mysql-test/t/heap_hash.test +++ b/mysql-test/t/heap_hash.test @@ -169,6 +169,8 @@ explain select * from t1 where a='aaac'; explain select * from t1 where a='aaad'; insert into t1 select * from t1; +# avoid statistics differences between normal and ps-protocol tests +flush tables; explain select * from t1 where a='aaaa'; explain select * from t1 where a='aaab'; explain select * from t1 where a='aaac'; diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test index 5e270c161a2..c10ce3b9688 100644 --- a/mysql-test/t/index_merge_innodb.test +++ b/mysql-test/t/index_merge_innodb.test @@ -120,3 +120,14 @@ select count(*) from t1 where drop table t1,t2; +# Test for BUG#8441 +create table t1 ( + id1 int, + id2 date , + index idx2 (id1,id2), + index idx1 (id2) +) engine = innodb; +insert into t1 values(1,'20040101'), (2,'20040102'); +select * from t1 where id1 = 1 and id2= '20040101'; +drop table t1; + diff --git a/mysql-test/t/insert_select.test b/mysql-test/t/insert_select.test index 834561ed5f7..a6468c52645 100644 --- a/mysql-test/t/insert_select.test +++ b/mysql-test/t/insert_select.test @@ -173,3 +173,17 @@ insert into t1 values (1),(1),(2); insert into t2(ff1) select f1 from t1 on duplicate key update ff2=ff2+1; select * from t2; drop table t1, t2; +# +# BUGS #9728 - 'Decreased functionality in "on duplicate key update"' +# #8147 - 'a column proclaimed ambigous in INSERT ... SELECT .. ON +# DUPLICATE' +# +create table t1 (a int unique); +create table t2 (a int, b int); +insert into t1 values (1),(2); +insert into t2 values (1,2); +select * from t1; +insert into t1 select t2.a from t2 on duplicate key update a= a + t2.b; +select * from t1; +drop table t1; +drop table t2; diff --git a/mysql-test/t/insert_update.test b/mysql-test/t/insert_update.test index f5857840588..7653fd8dd42 100644 --- a/mysql-test/t/insert_update.test +++ b/mysql-test/t/insert_update.test @@ -1,5 +1,5 @@ --disable_warnings -DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t1, t2; --enable_warnings CREATE TABLE t1 (a INT, b INT, c INT, UNIQUE (A), UNIQUE(B)); diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index a5822602b82..a285a4b312c 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -418,6 +418,41 @@ select count(*) from t2 where x > -16; select count(*) from t2 where x = 18446744073709551601; drop table t1,t2; +--disable_warnings +create table t1 (x bigint unsigned not null primary key) engine=innodb; +--enable_warnings +insert into t1(x) values (0xfffffffffffffff0); +insert into t1(x) values (0xfffffffffffffff1); +select * from t1; +select count(*) from t1 where x>0; +select count(*) from t1 where x=0; +select count(*) from t1 where x<0; +select count(*) from t1 where x < -16; +select count(*) from t1 where x = -16; +select count(*) from t1 where x > -16; +select count(*) from t1 where x = 18446744073709551601; + +drop table t1; + +# +# Bug #11185 incorrect comparison of unsigned int to signed constant +# +create table t1 (a bigint unsigned); +create index t1i on t1(a); +insert into t1 select 18446744073709551615; +insert into t1 select 18446744073709551614; + +explain select * from t1 where a <> -1; +select * from t1 where a <> -1; +explain select * from t1 where a > -1 or a < -1; +select * from t1 where a > -1 or a < -1; +explain select * from t1 where a > -1; +select * from t1 where a > -1; +explain select * from t1 where a < -1; +select * from t1 where a < -1; + +drop table t1; + # # Bug #6045: Binary Comparison regression in MySQL 4.1 # Binary searches didn't use a case insensitive index. diff --git a/mysql-test/t/rpl_multi_update3.test b/mysql-test/t/rpl_multi_update3.test index b8c8ed79532..80b0603eb60 100644 --- a/mysql-test/t/rpl_multi_update3.test +++ b/mysql-test/t/rpl_multi_update3.test @@ -1,7 +1,13 @@ +source include/master-slave.inc; + +############################################################################## +# # Let's verify that multi-update with a subselect does not cause the slave to crash # (BUG#10442) - -source include/master-slave.inc; +# +--disable_query_log +SELECT '-------- Test for BUG#9361 --------' as ""; +--enable_query_log CREATE TABLE t1 ( a int unsigned not null auto_increment primary key, @@ -25,10 +31,129 @@ UPDATE t2, (SELECT a FROM t1) AS t SET t2.b = t.a+5 ; SELECT * FROM t1 ORDER BY a; SELECT * FROM t2 ORDER BY a; -save_master_pos; +sync_slave_with_master; connection slave; -sync_with_master; SELECT * FROM t1 ORDER BY a; SELECT * FROM t2 ORDER BY a; +connection master; drop table t1,t2; + +############################################################################## +# +# Test for BUG#9361: +# Subselects should work inside multi-updates +# +--disable_query_log +SELECT '-------- Test 1 for BUG#9361 --------' as ""; +--enable_query_log + +connection master; + +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_warnings + +CREATE TABLE t1 ( + a1 char(30), + a2 int, + a3 int, + a4 char(30), + a5 char(30) +); + +CREATE TABLE t2 ( + b1 int, + b2 char(30) +); + +# Insert one row per table +INSERT INTO t1 VALUES ('Yes', 1, NULL, 'foo', 'bar'); +INSERT INTO t2 VALUES (1, 'baz'); + +# This should update the row in t1 +UPDATE t1 a, t2 + SET a.a1 = 'No' + WHERE a.a2 = + (SELECT b1 + FROM t2 + WHERE b2 = 'baz') + AND a.a3 IS NULL + AND a.a4 = 'foo' + AND a.a5 = 'bar'; + +sync_slave_with_master; +connection slave; +SELECT * FROM t1; +SELECT * FROM t2; + +connection master; +DROP TABLE t1, t2; + +############################################################################## +# +# Second test for BUG#9361 +# + +--disable_query_log +SELECT '-------- Test 2 for BUG#9361 --------' as ""; +--enable_query_log + +connection master; + +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +--enable_warnings + +CREATE TABLE t1 ( + i INT, + j INT, + x INT, + y INT, + z INT +); + +CREATE TABLE t2 ( + i INT, + k INT, + x INT, + y INT, + z INT +); + +CREATE TABLE t3 ( + j INT, + k INT, + x INT, + y INT, + z INT +); + +INSERT INTO t1 VALUES ( 1, 2,13,14,15); +INSERT INTO t2 VALUES ( 1, 3,23,24,25); +INSERT INTO t3 VALUES ( 2, 3, 1,34,35), ( 2, 3, 1,34,36); + +UPDATE t1 AS a +INNER JOIN t2 AS b + ON a.i = b.i +INNER JOIN t3 AS c + ON a.j = c.j AND b.k = c.k +SET a.x = b.x, + a.y = b.y, + a.z = ( + SELECT sum(z) + FROM t3 + WHERE y = 34 + ) +WHERE b.x = 23; + +sync_slave_with_master; +connection slave; + +SELECT * FROM t1; + +connection master; +DROP TABLE t1, t2, t3; diff --git a/mysql-test/t/sp-security.test b/mysql-test/t/sp-security.test index e1d8043ccda..72fe6c332bf 100644 --- a/mysql-test/t/sp-security.test +++ b/mysql-test/t/sp-security.test @@ -304,3 +304,39 @@ drop database sptest; delete from mysql.user where user='usera' or user='userb' or user='userc'; delete from mysql.procs_priv where user='usera' or user='userb' or user='userc'; +# +# BUG#9503: reseting correct parameters of thread after error in SP function +# +connect (root,localhost,root,,test); +connection root; + +--disable_warnings +drop function if exists bug_9503; +--enable_warnings +delimiter //; +create database mysqltest// +use mysqltest// +create table t1 (s1 int)// +grant select on t1 to user1@localhost// +create function bug_9503 () returns int sql security invoker begin declare v int; +select min(s1) into v from t1; return v; end// +delimiter ;// + +connect (user1,localhost,user1,,test); +connection user1; +use mysqltest; +-- error 1370 +select bug_9503(); + +connection root; +grant execute on function bug_9503 to user1@localhost; + +connection user1; +do 1; +use test; + +connection root; +REVOKE ALL PRIVILEGES, GRANT OPTION FROM user1@localhost; +drop function bug_9503; +use test; +drop database mysqltest; diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index 2e6cea8468b..1e4930d385d 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -1837,3 +1837,25 @@ WHERE t1.EMPNUM NOT IN WHERE t1.EMPNUM = t2.EMPNUM); select * from t1; DROP TABLE t1,t2; + +# +# Test for bug #11487: range access in a subquery +# + +CREATE TABLE t1(select_id BIGINT, values_id BIGINT); +INSERT INTO t1 VALUES (1, 1); +CREATE TABLE t2 (select_id BIGINT, values_id BIGINT, + PRIMARY KEY(select_id,values_id)); +INSERT INTO t2 VALUES (0, 1), (0, 2), (0, 3), (1, 5); + +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 + WHERE select_id IN (1, 0)); +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 + WHERE select_id BETWEEN 0 AND 1); +SELECT values_id FROM t1 +WHERE values_id IN (SELECT values_id FROM t2 + WHERE select_id = 0 OR select_id = 1); + +DROP TABLE t1, t2; diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 13a5f8cef1f..06b523b3610 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -1673,3 +1673,26 @@ create view v1(k, K) as select 1,2; create view v1 as SELECT TIME_FORMAT(SEC_TO_TIME(3600),'%H:%i') as t; select * from v1; drop view v1; + +# +# bug #11325 Wrong date comparison in views +# +create table t1 (f1 date); +insert into t1 values ('2005-01-01'),('2005-02-02'); +create view v1 as select * from t1; +select * from v1 where f1='2005.02.02'; +select * from v1 where '2005.02.02'=f1; +drop view v1; +drop table t1; + +# +# using encrypt & substring_index in view (BUG#7024) +# +CREATE VIEW v1 AS SELECT ENCRYPT("dhgdhgd"); +disable_result_log; +SELECT * FROM v1; +enable_result_log; +drop view v1; +CREATE VIEW v1 AS SELECT SUBSTRING_INDEX("dkjhgd:kjhdjh", ":", 1); +SELECT * FROM v1; +drop view v1; diff --git a/mysys/my_access.c b/mysys/my_access.c index 28210bdfc7d..c01031827c0 100644 --- a/mysys/my_access.c +++ b/mysys/my_access.c @@ -93,18 +93,20 @@ int check_if_legal_filename(const char *path) path+= dirname_length(path); /* To start of filename */ if (!(end= strchr(path, FN_EXTCHAR))) end= strend(path); - if (path == end || (uint) (path - end) > MAX_RESERVED_NAME_LENGTH) + if (path == end || (uint) (end - path) > MAX_RESERVED_NAME_LENGTH) DBUG_RETURN(0); /* Simplify inner loop */ for (reserved_name= reserved_names; *reserved_name; reserved_name++) { const char *name= path; - while (name != end) + const char *current_reserved_name= *reserved_name; + + while (name != end && *current_reserved_name) { - if (my_toupper(&my_charset_latin1, *path) != - my_toupper(&my_charset_latin1, *name)) + if (*current_reserved_name != my_toupper(&my_charset_latin1, *name)) break; - if (name++ == end) + current_reserved_name++; + if (++name == end) DBUG_RETURN(1); /* Found wrong path */ } } diff --git a/mysys/tree.c b/mysys/tree.c index bec1ec680f1..1780913961e 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -263,6 +263,9 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, if (tree->flag & TREE_NO_DUPS) return(NULL); element->count++; + /* Avoid a wrap over of the count. */ + if (! element->count) + element->count--; } DBUG_EXECUTE("check_tree", test_rb_tree(tree->root);); return element; diff --git a/sql/field.cc b/sql/field.cc index aeefe64ace3..fb244de4275 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -3317,12 +3317,12 @@ int Field_long::store(const char *from,uint len,CHARSET_INFO *cs) } if (error) { - error= 1; + error= error != MY_ERRNO_EDOM ? 1 : 2; set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, 1); } else if (from+len != end && table->in_use->count_cuted_fields && check_int(from,len,end,cs)) - error= 1; + error= 2; store_tmp= (long) tmp; #ifdef WORDS_BIGENDIAN @@ -3584,7 +3584,7 @@ int Field_longlong::store(const char *from,uint len,CHARSET_INFO *cs) } else if (from+len != end && table->in_use->count_cuted_fields && check_int(from,len,end,cs)) - error= 1; + error= 2; #ifdef WORDS_BIGENDIAN if (table->s->db_low_byte_first) { @@ -3806,7 +3806,7 @@ int Field_float::store(const char *from,uint len,CHARSET_INFO *cs) { set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, (error ? ER_WARN_DATA_OUT_OF_RANGE : WARN_DATA_TRUNCATED), 1); - error= 1; + error= error ? 1 : 2; } Field_float::store(nr); return error; @@ -4093,7 +4093,7 @@ int Field_double::store(const char *from,uint len,CHARSET_INFO *cs) { set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, (error ? ER_WARN_DATA_OUT_OF_RANGE : WARN_DATA_TRUNCATED), 1); - error= 1; + error= error ? 1 : 2; } Field_double::store(nr); return error; @@ -4495,6 +4495,8 @@ int Field_timestamp::store(const char *from,uint len,CHARSET_INFO *cs) error= 1; } } + if (error > 1) + error= 2; #ifdef WORDS_BIGENDIAN if (table->s->db_low_byte_first) @@ -4791,7 +4793,7 @@ int Field_time::store(const char *from,uint len,CHARSET_INFO *cs) if (str_to_time(from, len, <ime, &error)) { tmp=0L; - error= 1; + error= 2; set_datetime_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, from, len, MYSQL_TIMESTAMP_TIME, 1); } @@ -4813,6 +4815,8 @@ int Field_time::store(const char *from,uint len,CHARSET_INFO *cs) from, len, MYSQL_TIMESTAMP_TIME, !error); error= 1; } + if (error > 1) + error= 2; } if (ltime.neg) @@ -5149,7 +5153,7 @@ int Field_date::store(const char *from, uint len,CHARSET_INFO *cs) &error) <= MYSQL_TIMESTAMP_ERROR) { tmp=0; - error= 1; + error= 2; } else tmp=(uint32) l_time.year*10000L + (uint32) (l_time.month*100+l_time.day); @@ -5353,7 +5357,7 @@ int Field_newdate::store(const char *from,uint len,CHARSET_INFO *cs) &error) <= MYSQL_TIMESTAMP_ERROR) { tmp=0L; - error= 1; + error= 2; } else tmp= l_time.day + l_time.month*32 + l_time.year*16*32; @@ -5836,7 +5840,7 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) from= tmpstr.ptr(); length= tmpstr.length(); if (conv_errors) - error= 1; + error= 2; } /* @@ -5860,7 +5864,7 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) from+= field_charset->cset->scan(field_charset, from, end, MY_SEQ_SPACES); if (from != end) - error= 1; + error= 2; } if (error) { @@ -6242,7 +6246,7 @@ int Field_varstring::store(const char *from,uint length,CHARSET_INFO *cs) table->in_use->abort_on_warning) error_code= ER_DATA_TOO_LONG; set_warning(level, error_code, 1); - return 1; + return 2; } return 0; } @@ -6822,7 +6826,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) from= tmpstr.ptr(); length= tmpstr.length(); if (conv_errors) - error= 1; + error= 2; } copy_length= max_data_length(); @@ -6837,7 +6841,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) copy_length, &well_formed_error); if (copy_length < length) - error= 1; + error= 2; Field_blob::store_length(copy_length); if (was_conversion || table->copy_blobs || copy_length <= MAX_FIELD_WIDTH) { // Must make a copy diff --git a/sql/ha_federated.cc b/sql/ha_federated.cc index 89210a2f3cd..77db17608bc 100644 --- a/sql/ha_federated.cc +++ b/sql/ha_federated.cc @@ -462,6 +462,12 @@ static int check_foreign_data_source(FEDERATED_SHARE *share) } else { + /* + Since we do not support transactions at this version, we can let the client + API silently reconnect. For future versions, we will need more logic to deal + with transactions + */ + mysql->reconnect= 1; /* Note: I am not using INORMATION_SCHEMA because this needs to work with < 5.0 if we can connect, then make sure the table exists @@ -988,6 +994,12 @@ int ha_federated::open(const char *name, int mode, uint test_if_locked) my_error(ER_CONNECT_TO_MASTER, MYF(0), mysql_error(mysql)); DBUG_RETURN(ER_CONNECT_TO_MASTER); } + /* + Since we do not support transactions at this version, we can let the client + API silently reconnect. For future versions, we will need more logic to deal + with transactions + */ + mysql->reconnect= 1; DBUG_RETURN(0); } diff --git a/sql/ha_heap.cc b/sql/ha_heap.cc index cd655eeb0a9..6e609a94be3 100644 --- a/sql/ha_heap.cc +++ b/sql/ha_heap.cc @@ -65,7 +65,15 @@ int ha_heap::open(const char *name, int mode, uint test_if_locked) { /* Initialize variables for the opened table */ set_keys_for_scanning(); - update_key_stats(); + /* + We cannot run update_key_stats() here because we do not have a + lock on the table. The 'records' count might just be changed + temporarily at this moment and we might get wrong statistics (Bug + #10178). Instead we request for update. This will be done in + ha_heap::info(), which is always called before key statistics are + used. + */ + key_stats_ok= FALSE; } return (file ? 0 : 1); } @@ -118,6 +126,8 @@ void ha_heap::update_key_stats() } } records_changed= 0; + /* At the end of update_key_stats() we can proudly claim they are OK. */ + key_stats_ok= TRUE; } @@ -132,7 +142,7 @@ int ha_heap::write_row(byte * buf) res= heap_write(file,buf); if (!res && (++records_changed*HEAP_STATS_UPDATE_THRESHOLD > file->s->records)) - update_key_stats(); + key_stats_ok= FALSE; return res; } @@ -145,7 +155,7 @@ int ha_heap::update_row(const byte * old_data, byte * new_data) res= heap_update(file,old_data,new_data); if (!res && ++records_changed*HEAP_STATS_UPDATE_THRESHOLD > file->s->records) - update_key_stats(); + key_stats_ok= FALSE; return res; } @@ -156,7 +166,7 @@ int ha_heap::delete_row(const byte * buf) res= heap_delete(file,buf); if (!res && table->s->tmp_table == NO_TMP_TABLE && ++records_changed*HEAP_STATS_UPDATE_THRESHOLD > file->s->records) - update_key_stats(); + key_stats_ok= FALSE; return res; } @@ -278,6 +288,13 @@ void ha_heap::info(uint flag) delete_length= info.deleted * info.reclength; if (flag & HA_STATUS_AUTO) auto_increment_value= info.auto_increment; + /* + If info() is called for the first time after open(), we will still + have to update the key statistics. Hoping that a table lock is now + in place. + */ + if (! key_stats_ok) + update_key_stats(); } int ha_heap::extra(enum ha_extra_function operation) @@ -289,7 +306,7 @@ int ha_heap::delete_all_rows() { heap_clear(file); if (table->s->tmp_table == NO_TMP_TABLE) - update_key_stats(); + key_stats_ok= FALSE; return 0; } @@ -448,6 +465,9 @@ ha_rows ha_heap::records_in_range(uint inx, key_range *min_key, min_key->flag != HA_READ_KEY_EXACT || max_key->flag != HA_READ_AFTER_KEY) return HA_POS_ERROR; // Can only use exact keys + + /* Assert that info() did run. We need current statistics here. */ + DBUG_ASSERT(key_stats_ok); return key->rec_per_key[key->key_parts-1]; } diff --git a/sql/ha_heap.h b/sql/ha_heap.h index 2aa065e0d96..7a97c727049 100644 --- a/sql/ha_heap.h +++ b/sql/ha_heap.h @@ -29,8 +29,10 @@ class ha_heap: public handler key_map btree_keys; /* number of records changed since last statistics update */ uint records_changed; + bool key_stats_ok; public: - ha_heap(TABLE *table): handler(table), file(0), records_changed(0) {} + ha_heap(TABLE *table): handler(table), file(0), records_changed(0), + key_stats_ok(0) {} ~ha_heap() {} const char *table_type() const { diff --git a/sql/item.cc b/sql/item.cc index ae000f60a58..d3888cef9d5 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -338,6 +338,7 @@ Item::Item(): place == IN_HAVING) thd->lex->current_select->select_n_having_items++; } + item_flags= 0; } /* @@ -358,7 +359,8 @@ Item::Item(THD *thd, Item *item): unsigned_flag(item->unsigned_flag), with_sum_func(item->with_sum_func), fixed(item->fixed), - collation(item->collation) + collation(item->collation), + item_flags(item->item_flags) { next= thd->free_list; // Put in free list thd->free_list= this; diff --git a/sql/item.h b/sql/item.h index c912ad3f0a7..c8180b4932a 100644 --- a/sql/item.h +++ b/sql/item.h @@ -225,6 +225,11 @@ typedef Item* (Item::*Item_transformer) (byte *arg); typedef void (*Cond_traverser) (const Item *item, void *arg); +/* + See comments for sql_yacc.yy: insert_update_elem rule + */ +#define MY_ITEM_PREFER_1ST_TABLE 1 + class Item { Item(const Item &); /* Prevent use of these */ void operator=(Item &); @@ -272,6 +277,7 @@ public: my_bool is_autogenerated_name; /* indicate was name of this Item autogenerated or set by user */ DTCollation collation; + uint8 item_flags; /* Flags on how item should be processed */ // alloc & destruct is done as start of select using sql_alloc Item(); @@ -584,6 +590,11 @@ public: cleanup(); delete this; } + virtual bool set_flags_processor(byte *args) + { + this->item_flags|= *((uint8*)args); + return false; + } virtual bool is_splocal() { return 0; } /* Needed for error checking */ }; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 5a2e14eef2e..58a7f3316d7 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -238,9 +238,10 @@ void Item_bool_func2::fix_length_and_dec() return; } - if (args[0]->type() == FIELD_ITEM) + Item *real_item= args[0]->real_item(); + if (real_item->type() == FIELD_ITEM) { - Field *field=((Item_field*) args[0])->field; + Field *field= ((Item_field*) real_item)->field; if (field->can_be_compared_as_longlong()) { if (convert_constant_item(thd, field,&args[1])) @@ -251,9 +252,10 @@ void Item_bool_func2::fix_length_and_dec() } } } - if (args[1]->type() == FIELD_ITEM /* && !args[1]->const_item() */) + real_item= args[1]->real_item(); + if (real_item->type() == FIELD_ITEM) { - Field *field=((Item_field*) args[1])->field; + Field *field= ((Item_field*) real_item)->field; if (field->can_be_compared_as_longlong()) { if (convert_constant_item(thd, field,&args[0])) @@ -1420,6 +1422,8 @@ Item *Item_func_case::find_item(String *str) my_decimal *first_expr_dec, first_expr_dec_val; longlong first_expr_int; double first_expr_real; + char buff[MAX_FIELD_WIDTH]; + String buff_str(buff,sizeof(buff),default_charset()); /* These will be initialized later */ LINT_INIT(first_expr_str); @@ -1433,7 +1437,7 @@ Item *Item_func_case::find_item(String *str) { case STRING_RESULT: // We can't use 'str' here as this may be overwritten - if (!(first_expr_str= args[first_expr_num]->val_str(&str_value))) + if (!(first_expr_str= args[first_expr_num]->val_str(&buff_str))) return else_expr_num != -1 ? args[else_expr_num] : 0; // Impossible break; case INT_RESULT: diff --git a/sql/item_func.cc b/sql/item_func.cc index 57f68bbc2a0..1dbf28b67cb 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4807,42 +4807,36 @@ Item_func_sp::execute(Item **itp) DBUG_ENTER("Item_func_sp::execute"); THD *thd= current_thd; ulong old_client_capabilites; - int res; + int res= -1; bool save_in_sub_stmt= thd->transaction.in_sub_stmt; + my_bool nsok; #ifndef NO_EMBEDDED_ACCESS_CHECKS st_sp_security_context save_ctx; #endif - if (! m_sp) + if (! m_sp && ! (m_sp= sp_find_function(thd, m_name, TRUE))) { - if (!(m_sp= sp_find_function(thd, m_name, TRUE))) - { - my_error(ER_SP_DOES_NOT_EXIST, MYF(0), "FUNCTION", m_name->m_qname.str); - DBUG_RETURN(-1); - } + my_error(ER_SP_DOES_NOT_EXIST, MYF(0), "FUNCTION", m_name->m_qname.str); + goto error; } old_client_capabilites= thd->client_capabilities; thd->client_capabilities &= ~CLIENT_MULTI_RESULTS; #ifndef EMBEDDED_LIBRARY - my_bool nsok= thd->net.no_send_ok; + nsok= thd->net.no_send_ok; thd->net.no_send_ok= TRUE; #endif #ifndef NO_EMBEDDED_ACCESS_CHECKS if (check_routine_access(thd, EXECUTE_ACL, m_sp->m_db.str, m_sp->m_name.str, 0, 0)) - DBUG_RETURN(-1); + goto error_check; sp_change_security_context(thd, m_sp, &save_ctx); if (save_ctx.changed && check_routine_access(thd, EXECUTE_ACL, m_sp->m_db.str, m_sp->m_name.str, 0, 0)) - { - sp_restore_security_context(thd, m_sp, &save_ctx); - thd->client_capabilities|= old_client_capabilites & CLIENT_MULTI_RESULTS; - DBUG_RETURN(-1); - } + goto error_check; #endif /* Like for SPs, we don't binlog the substatements. If the statement which @@ -4850,6 +4844,7 @@ Item_func_sp::execute(Item **itp) it's not (e.g. SELECT myfunc()) it won't be binlogged (documented known problem). */ + tmp_disable_binlog(thd); /* don't binlog the substatements */ thd->transaction.in_sub_stmt= TRUE; @@ -4864,16 +4859,21 @@ Item_func_sp::execute(Item **itp) ER_FAILED_ROUTINE_BREAK_BINLOG, ER(ER_FAILED_ROUTINE_BREAK_BINLOG)); +error_check_ctx: #ifndef NO_EMBEDDED_ACCESS_CHECKS sp_restore_security_context(thd, m_sp, &save_ctx); #endif + thd->client_capabilities|= old_client_capabilites & CLIENT_MULTI_RESULTS; + +error_check: #ifndef EMBEDDED_LIBRARY thd->net.no_send_ok= nsok; #endif thd->client_capabilities|= old_client_capabilites & CLIENT_MULTI_RESULTS; +error: DBUG_RETURN(res); } diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 8d2eb269915..c4beb3b08cb 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -322,7 +322,7 @@ public: Item_func_encrypt(Item *a, Item *b): Item_str_func(a,b) {} String *val_str(String *); void fix_length_and_dec() { maybe_null=1; max_length = 13; } - const char *func_name() const { return "ecrypt"; } + const char *func_name() const { return "encrypt"; } }; #include "sql_crypt.h" diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 393cb87c113..7ea72f3c858 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1487,7 +1487,7 @@ int subselect_uniquesubquery_engine::exec() TABLE *table= tab->table; for (store_key **copy=tab->ref.key_copy ; *copy ; copy++) { - if (tab->ref.key_err= (*copy)->copy()) + if ((tab->ref.key_err= (*copy)->copy()) & 1) { table->status= STATUS_NOT_FOUND; DBUG_RETURN(1); @@ -1540,7 +1540,7 @@ int subselect_indexsubquery_engine::exec() for (store_key **copy=tab->ref.key_copy ; *copy ; copy++) { - if (tab->ref.key_err= (*copy)->copy()) + if ((tab->ref.key_err= (*copy)->copy()) & 1) { table->status= STATUS_NOT_FOUND; DBUG_RETURN(1); diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 0a8627b1fa0..4c0c895f22a 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -2593,12 +2593,12 @@ static double ror_scan_selectivity(const ROR_INTERSECT_INFO *info, { tuple_arg= scan->sel_arg; /* Here we use the length of the first key part */ - tuple_arg->store_min(key_part->length, &key_ptr, 0); + tuple_arg->store_min(key_part->store_length, &key_ptr, 0); } while (tuple_arg->next_key_part != sel_arg) { tuple_arg= tuple_arg->next_key_part; - tuple_arg->store_min(key_part[tuple_arg->part].length, &key_ptr, 0); + tuple_arg->store_min(key_part[tuple_arg->part].store_length, &key_ptr, 0); } min_range.length= max_range.length= ((char*) key_ptr - (char*) key_val); records= (info->param->table->file-> @@ -3840,6 +3840,35 @@ get_mm_leaf(PARAM *param, COND *conf_func, Field *field, KEY_PART *key_part, if (!(tree=new SEL_ARG(field,str,str))) DBUG_RETURN(0); // out of memory + /* + Check if we are comparing an UNSIGNED integer with a negative constant. + In this case we know that: + (a) (unsigned_int [< | <=] negative_constant) == FALSE + (b) (unsigned_int [> | >=] negative_constant) == TRUE + In case (a) the condition is false for all values, and in case (b) it + is true for all values, so we can avoid unnecessary retrieval and condition + testing, and we also get correct comparison of unsinged integers with + negative integers (which otherwise fails because at query execution time + negative integers are cast to unsigned if compared with unsigned). + */ + Item_result field_result_type= field->result_type(); + Item_result value_result_type= value->result_type(); + if (field_result_type == INT_RESULT && value_result_type == INT_RESULT && + ((Field_num*)field)->unsigned_flag && !((Item_int*)value)->unsigned_flag) + { + longlong item_val= value->val_int(); + if (item_val < 0) + { + if (type == Item_func::LT_FUNC || type == Item_func::LE_FUNC) + { + tree->type= SEL_ARG::IMPOSSIBLE; + DBUG_RETURN(tree); + } + if (type == Item_func::GT_FUNC || type == Item_func::GE_FUNC) + DBUG_RETURN(0); + } + } + switch (type) { case Item_func::LT_FUNC: if (field_is_equal_to_item(field,value)) @@ -5992,7 +6021,10 @@ int QUICK_RANGE_SELECT::reset() next=0; range= NULL; cur_range= (QUICK_RANGE**) ranges.buffer; - + + if (file->inited == handler::NONE && (error= file->ha_index_init(index))) + DBUG_RETURN(error); + /* Do not allocate the buffers twice. */ if (multi_range_length) { diff --git a/sql/protocol.cc b/sql/protocol.cc index 57922cdc677..1c399a89a99 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -294,7 +294,12 @@ send_ok(THD *thd, ha_rows affected_rows, ulonglong id, const char *message) DBUG_ENTER("send_ok"); if (net->no_send_ok || !net->vio) // hack for re-parsing queries + { + DBUG_PRINT("info", ("no send ok: %s, vio present: %s", + (net->no_send_ok ? "YES" : "NO"), + (net->vio ? "YES" : "NO"))); DBUG_VOID_RETURN; + } buff[0]=0; // No fields pos=net_store_length(buff+1,(ulonglong) affected_rows); diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 29cee6da4d3..825ec34e410 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -509,7 +509,7 @@ sp_head::destroy() delete i; delete_dynamic(&m_instr); m_pcont->destroy(); - free_items(free_list); + free_items(); /* If we have non-empty LEX stack then we just came out of parser with @@ -596,7 +596,6 @@ sp_head::execute(THD *thd) ctx->clear_handler(); thd->query_error= 0; old_arena= thd->current_arena; - thd->current_arena= this; /* We have to save/restore this info when we are changing call level to @@ -636,23 +635,18 @@ sp_head::execute(THD *thd) break; DBUG_PRINT("execute", ("Instruction %u", ip)); thd->set_time(); // Make current_time() et al work - { - /* - We have to substitute free_list of executing statement to - current_arena to store there all new items created during execution - (for example '*' expanding, or items made during permanent subquery - transformation) - Note: Every statement have to have all its items listed in free_list - for correct cleaning them up - */ - Item *save_free_list= thd->current_arena->free_list; - thd->current_arena->free_list= i->free_list; - ret= i->execute(thd, &ip); - i->free_list= thd->current_arena->free_list; - thd->current_arena->free_list= save_free_list; - } + /* + We have to set thd->current_arena before executing the instruction + to store in the instruction free_list all new items, created + during the first execution (for example expanding of '*' or the + items made during other permanent subquery transformations). + */ + thd->current_arena= i; + ret= i->execute(thd, &ip); if (i->free_list) cleanup_items(i->free_list); + i->state= Query_arena::EXECUTED; + // Check if an exception has occurred and a handler has been found // Note: We havo to check even if ret==0, since warnings (and some // errors don't return a non-zero value. @@ -694,7 +688,6 @@ sp_head::execute(THD *thd) DBUG_ASSERT(!thd->derived_tables); thd->derived_tables= old_derived_tables; - cleanup_items(thd->current_arena->free_list); thd->current_arena= old_arena; state= EXECUTED; @@ -728,8 +721,8 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) sp_rcontext *nctx = NULL; uint i; int ret; - MEM_ROOT *old_mem_root, call_mem_root; - Item *old_free_list, *call_free_list; + MEM_ROOT call_mem_root; + Query_arena call_arena(&call_mem_root, INITIALIZED_FOR_SP), backup_arena; if (argcount != params) { @@ -741,14 +734,12 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) } init_alloc_root(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0); - old_mem_root= thd->mem_root; - thd->mem_root= &call_mem_root; - old_free_list= thd->free_list; // Keep the old list - thd->free_list= NULL; // Start a new one + + thd->set_n_backup_item_arena(&call_arena, &backup_arena); // QQ Should have some error checking here? (types, etc...) nctx= new sp_rcontext(csize, hmax, cmax); - nctx->callers_mem_root= old_mem_root; + nctx->callers_mem_root= backup_arena.mem_root; for (i= 0 ; i < argcount ; i++) { sp_pvar_t *pvar = m_pcont->find_pvar(i); @@ -780,9 +771,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) // Partially restore context now. // We still need the call mem root and free list for processing // of the result. - call_free_list= thd->free_list; - thd->free_list= old_free_list; - thd->mem_root= old_mem_root; + thd->restore_backup_item_arena(&call_arena, &backup_arena); if (m_type == TYPE_ENUM_FUNCTION && ret == 0) { @@ -802,8 +791,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, Item **resp) thd->spcont= octx; // Now get rid of the rest of the callee context - cleanup_items(call_free_list); - free_items(call_free_list); + call_arena.free_items(); free_root(&call_mem_root, MYF(0)); DBUG_RETURN(ret); @@ -835,8 +823,8 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) sp_rcontext *octx = thd->spcont; sp_rcontext *nctx = NULL; my_bool tmp_octx = FALSE; // True if we have allocated a temporary octx - MEM_ROOT *old_mem_root, call_mem_root; - Item *old_free_list, *call_free_list; + MEM_ROOT call_mem_root; + Query_arena call_arena(&call_mem_root, INITIALIZED_FOR_SP), backup_arena; if (args->elements != params) { @@ -846,10 +834,7 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) } init_alloc_root(&call_mem_root, MEM_ROOT_BLOCK_SIZE, 0); - old_mem_root= thd->mem_root; - thd->mem_root= &call_mem_root; - old_free_list= thd->free_list; // Keep the old list - thd->free_list= NULL; // Start a new one + thd->set_n_backup_item_arena(&call_arena, &backup_arena); if (csize > 0 || hmax > 0 || cmax > 0) { @@ -919,9 +904,7 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) // Partially restore context now. // We still need the call mem root and free list for processing // of out parameters. - call_free_list= thd->free_list; - thd->free_list= old_free_list; - thd->mem_root= old_mem_root; + thd->restore_backup_item_arena(&call_arena, &backup_arena); if (!ret && csize > 0) { @@ -996,8 +979,7 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) thd->spcont= octx; // Now get rid of the rest of the callee context - cleanup_items(call_free_list); - free_items(call_free_list); + call_arena.free_items(); thd->lex->unit.cleanup(); free_root(&call_mem_root, MYF(0)); @@ -1291,6 +1273,13 @@ void sp_head::add_instr(sp_instr *instr) { instr->free_list= m_thd->free_list; m_thd->free_list= 0; + /* + Memory root of every instruction is designated for permanent + transformations (optimizations) made on the parsed tree during + the first execution. It points to the memory root of the + entire stored procedure, as their life span is equal. + */ + instr->mem_root= &main_mem_root; insert_dynamic(&m_instr, (gptr)&instr); } diff --git a/sql/sp_head.h b/sql/sp_head.h index 2c75a320f30..aaef5a3d50e 100644 --- a/sql/sp_head.h +++ b/sql/sp_head.h @@ -274,7 +274,7 @@ private: // "Instructions"... // -class sp_instr : public Sql_alloc +class sp_instr :public Query_arena, public Sql_alloc { sp_instr(const sp_instr &); /* Prevent use of these */ void operator=(sp_instr &); @@ -282,17 +282,16 @@ class sp_instr : public Sql_alloc public: uint marked; - Item *free_list; // My Items uint m_ip; // My index sp_pcontext *m_ctx; // My parse context // Should give each a name or type code for debugging purposes? sp_instr(uint ip, sp_pcontext *ctx) - :Sql_alloc(), marked(0), free_list(0), m_ip(ip), m_ctx(ctx) + :Query_arena(0, INITIALIZED_FOR_SP), marked(0), m_ip(ip), m_ctx(ctx) {} virtual ~sp_instr() - { free_items(free_list); } + { free_items(); } // Execute this instrution. '*nextp' will be set to the index of the next // instruction to execute. (For most instruction this will be the diff --git a/sql/sql_base.cc b/sql/sql_base.cc index fee3bdfeb5f..e9af684767d 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -2626,7 +2626,6 @@ find_field_in_tables(THD *thd, Item_ident *item, TABLE_LIST *tables, uint length=(uint) strlen(name); char name_buff[NAME_LEN+1]; - if (item->cached_table) { /* @@ -2693,10 +2692,13 @@ find_field_in_tables(THD *thd, Item_ident *item, TABLE_LIST *tables, db= name_buff; } + bool search_global= item->item_flags & MY_ITEM_PREFER_1ST_TABLE; if (table_name && table_name[0]) { /* Qualified field */ - bool found_table=0; - for (; tables; tables= tables->next_local) + bool found_table=0; + uint table_idx= 0; + for (; tables; tables= search_global?tables->next_global:tables->next_local, + table_idx++) { /* TODO; Ensure that db and tables->db always points to something ! */ if (!my_strcasecmp(table_alias_charset, tables->alias, table_name) && @@ -2732,6 +2734,8 @@ find_field_in_tables(THD *thd, Item_ident *item, TABLE_LIST *tables, return (Field*) 0; } found=find; + if (table_idx == 0 && item->item_flags & MY_ITEM_PREFER_1ST_TABLE) + break; } } } @@ -2756,9 +2760,10 @@ find_field_in_tables(THD *thd, Item_ident *item, TABLE_LIST *tables, return (Field*) not_found_field; return (Field*) 0; } - bool allow_rowid= tables && !tables->next_local; // Only one table - for (; tables ; tables= tables->next_local) + uint table_idx= 0; + for (; tables ; tables= search_global?tables->next_global:tables->next_local, + table_idx++) { if (!tables->table && !tables->ancestor) { @@ -2793,7 +2798,9 @@ find_field_in_tables(THD *thd, Item_ident *item, TABLE_LIST *tables, my_error(ER_NON_UNIQ_ERROR, MYF(0), name, thd->where); return (Field*) 0; } - found=field; + found= field; + if (table_idx == 0 && item->item_flags & MY_ITEM_PREFER_1ST_TABLE) + break; } } if (found) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index f54135ea8cb..20f48da9283 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -156,18 +156,21 @@ bool foreign_key_prefix(Key *a, Key *b) /**************************************************************************** ** Thread specific functions ****************************************************************************/ +/* + Pass nominal parameters to Statement constructor only to ensure that + the destructor works OK in case of error. The main_mem_root will be + re-initialized in init(). +*/ THD::THD() - :user_time(0), global_read_lock(0), is_fatal_error(0), + :Statement(CONVENTIONAL_EXECUTION, 0, ALLOC_ROOT_MIN_BLOCK_SIZE, 0), + user_time(0), global_read_lock(0), is_fatal_error(0), rand_used(0), time_zone_used(0), last_insert_id_used(0), insert_id_used(0), clear_next_insert_id(0), in_lock_tables(0), bootstrap(0), derived_tables_processing(FALSE), spcont(NULL) { current_arena= this; -#ifndef DBUG_OFF - backup_arena= 0; -#endif host= user= priv_user= db= ip= 0; catalog= (char*)"std"; // the only catalog we have for now host_or_ip= "connecting host"; @@ -522,7 +525,7 @@ void THD::cleanup_after_query() next_insert_id= 0; } /* Free Items that were created during this execution */ - free_items(free_list); + free_items(); /* In the rest of code we assume that free_list never points to garbage: Keep this predicate true. @@ -1479,13 +1482,29 @@ Query_arena::Type Query_arena::type() const } +void Query_arena::free_items() +{ + Item *next; + DBUG_ENTER("Query_arena::free_items"); + /* This works because items are allocated with sql_alloc() */ + for (; free_list; free_list= next) + { + next= free_list->next; + free_list->delete_self(); + } + /* Postcondition: free_list is 0 */ + DBUG_VOID_RETURN; +} + + /* Statement functions */ -Statement::Statement(THD *thd) - :Query_arena(&main_mem_root, INITIALIZED), - id(++thd->statement_id_counter), +Statement::Statement(enum enum_state state_arg, ulong id_arg, + ulong alloc_block_size, ulong prealloc_size) + :Query_arena(&main_mem_root, state_arg), + id(id_arg), set_query_id(1), allow_sum_func(0), lex(&main_lex), @@ -1494,33 +1513,7 @@ Statement::Statement(THD *thd) cursor(0) { name.str= NULL; - init_sql_alloc(&main_mem_root, - thd->variables.query_alloc_block_size, - thd->variables.query_prealloc_size); -} - -/* - This constructor is called when Statement is a parent of THD and - for the backup statement. Some variables are initialized in - THD::init due to locking problems. -*/ - -Statement::Statement() - :Query_arena(&main_mem_root, CONVENTIONAL_EXECUTION), - id(0), - set_query_id(1), - allow_sum_func(0), /* initialized later */ - lex(&main_lex), - query(0), /* these two are set */ - query_length(0), /* in alloc_query() */ - cursor(0) -{ - /* - This is just to ensure that the destructor works correctly in - case of an error and the backup statement. The memory root will - be re-initialized in THD::init. - */ - init_sql_alloc(&main_mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0); + init_sql_alloc(&main_mem_root, alloc_block_size, prealloc_size); } @@ -1575,11 +1568,11 @@ void THD::end_statement() void Query_arena::set_n_backup_item_arena(Query_arena *set, Query_arena *backup) { DBUG_ENTER("Query_arena::set_n_backup_item_arena"); - DBUG_ASSERT(backup_arena == 0); + DBUG_ASSERT(backup->is_backup_arena == FALSE); backup->set_item_arena(this); set_item_arena(set); #ifndef DBUG_OFF - backup_arena= 1; + backup->is_backup_arena= TRUE; #endif DBUG_VOID_RETURN; } @@ -1588,10 +1581,11 @@ void Query_arena::set_n_backup_item_arena(Query_arena *set, Query_arena *backup) void Query_arena::restore_backup_item_arena(Query_arena *set, Query_arena *backup) { DBUG_ENTER("Query_arena::restore_backup_item_arena"); + DBUG_ASSERT(backup->is_backup_arena); set->set_item_arena(this); set_item_arena(backup); #ifndef DBUG_OFF - backup_arena= 0; + backup->is_backup_arena= FALSE; #endif DBUG_VOID_RETURN; } diff --git a/sql/sql_class.h b/sql/sql_class.h index fff789d3066..1f232b9ca21 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -663,7 +663,10 @@ public: Item *free_list; MEM_ROOT *mem_root; // Pointer to current memroot #ifndef DBUG_OFF - bool backup_arena; + bool is_backup_arena; /* True if this arena is used for backup. */ +#define INIT_ARENA_DBUG_INFO is_backup_arena= 0 +#else +#define INIT_ARENA_DBUG_INFO #endif enum enum_state { @@ -681,12 +684,14 @@ public: Query_arena(MEM_ROOT *mem_root_arg, enum enum_state state_arg) : free_list(0), mem_root(mem_root_arg), state(state_arg) - {} + { INIT_ARENA_DBUG_INFO; } /* This constructor is used only when Query_arena is created as backup storage for another instance of Query_arena. */ - Query_arena() {}; + Query_arena() { INIT_ARENA_DBUG_INFO; } + +#undef INIT_ARENA_DBUG_INFO virtual Type type() const; virtual ~Query_arena() {}; @@ -726,6 +731,8 @@ public: void set_n_backup_item_arena(Query_arena *set, Query_arena *backup); void restore_backup_item_arena(Query_arena *set, Query_arena *backup); void set_item_arena(Query_arena *set); + + void free_items(); }; @@ -809,13 +816,11 @@ public: public: - /* - This constructor is called when statement is a subobject of THD: - some variables are initialized in THD::init due to locking problems - */ - Statement(); + /* This constructor is called for backup statements */ + Statement() { clear_alloc_root(&main_mem_root); } - Statement(THD *thd); + Statement(enum enum_state state_arg, ulong id_arg, + ulong alloc_block_size, ulong prealloc_size); virtual ~Statement(); /* Assign execution context (note: not all members) of given stmt to self */ @@ -958,11 +963,6 @@ public: /* all prepared statements and cursors of this connection */ Statement_map stmt_map; /* - keeps THD state while it is used for active statement - Note: we perform special cleanup for it in THD destructor. - */ - Statement stmt_backup; - /* A pointer to the stack frame of handle_one_connection(), which is called first in the thread for handling a client */ @@ -1427,10 +1427,10 @@ public: }; #define tmp_disable_binlog(A) \ - ulong save_options= (A)->options; \ - (A)->options&= ~OPTION_BIN_LOG; + {ulong tmp_disable_binlog__save_options= (A)->options; \ + (A)->options&= ~OPTION_BIN_LOG -#define reenable_binlog(A) (A)->options= save_options; +#define reenable_binlog(A) (A)->options= tmp_disable_binlog__save_options;} /* Flags for the THD::system_thread (bitmap) variable */ #define SYSTEM_THREAD_DELAYED_INSERT 1 diff --git a/sql/sql_lex.h b/sql/sql_lex.h index a9bfb6da926..5cf0b66598f 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -642,6 +642,11 @@ public: static void print_order(String *str, ORDER *order); void print_limit(THD *thd, String *str); void fix_prepare_information(THD *thd, Item **conds); + /* + Destroy the used execution plan (JOIN) of this subtree (this + SELECT_LEX and all nested SELECT_LEXes and SELECT_LEX_UNITs). + */ + bool cleanup(); }; typedef class st_select_lex SELECT_LEX; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 33339cf0882..a60eb32564b 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3272,7 +3272,6 @@ end_with_restore_list: if (first_table->view && !first_table->contain_auto_increment) thd->last_insert_id= 0; // do not show last insert ID if VIEW have not it - break; } case SQLCOM_TRUNCATE: diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 73bbea79760..c97cb037f15 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1706,6 +1706,7 @@ bool mysql_stmt_prepare(THD *thd, char *packet, uint packet_length, LEX_STRING *name) { LEX *lex; + Statement stmt_backup; Prepared_statement *stmt= new Prepared_statement(thd); bool error; DBUG_ENTER("mysql_stmt_prepare"); @@ -1739,13 +1740,13 @@ bool mysql_stmt_prepare(THD *thd, char *packet, uint packet_length, DBUG_RETURN(TRUE); } - thd->set_n_backup_statement(stmt, &thd->stmt_backup); - thd->set_n_backup_item_arena(stmt, &thd->stmt_backup); + thd->set_n_backup_statement(stmt, &stmt_backup); + thd->set_n_backup_item_arena(stmt, &stmt_backup); if (alloc_query(thd, packet, packet_length)) { - thd->restore_backup_statement(stmt, &thd->stmt_backup); - thd->restore_backup_item_arena(stmt, &thd->stmt_backup); + thd->restore_backup_statement(stmt, &stmt_backup); + thd->restore_backup_item_arena(stmt, &stmt_backup); /* Statement map deletes statement on erase */ thd->stmt_map.erase(stmt); DBUG_RETURN(TRUE); @@ -1770,7 +1771,7 @@ bool mysql_stmt_prepare(THD *thd, char *packet, uint packet_length, transformation can be reused on execute, we set again thd->mem_root from stmt->mem_root (see setup_wild for one place where we do that). */ - thd->restore_backup_item_arena(stmt, &thd->stmt_backup); + thd->restore_backup_item_arena(stmt, &stmt_backup); if (!error) error= check_prepared_statement(stmt, test(name)); @@ -1786,7 +1787,7 @@ bool mysql_stmt_prepare(THD *thd, char *packet, uint packet_length, lex_end(lex); close_thread_tables(thd); cleanup_stmt_and_thd_after_use(stmt, thd); - thd->restore_backup_statement(stmt, &thd->stmt_backup); + thd->restore_backup_statement(stmt, &stmt_backup); thd->current_arena= thd; if (error) @@ -1949,6 +1950,7 @@ void mysql_stmt_execute(THD *thd, char *packet, uint packet_length) { ulong stmt_id= uint4korr(packet); ulong flags= (ulong) ((uchar) packet[4]); + Statement stmt_backup; Cursor *cursor; /* Query text for binary log, or empty string if the query is not put into @@ -2026,8 +2028,7 @@ void mysql_stmt_execute(THD *thd, char *packet, uint packet_length) if (stmt->param_count && stmt->set_params_data(stmt, &expanded_query)) goto set_params_data_err; #endif - thd->stmt_backup.set_statement(thd); - thd->set_statement(stmt); + thd->set_n_backup_statement(stmt, &stmt_backup); thd->current_arena= stmt; reinit_stmt_before_use(thd, stmt->lex); /* From now cursors assume that thd->mem_root is clean */ @@ -2064,7 +2065,7 @@ void mysql_stmt_execute(THD *thd, char *packet, uint packet_length) reset_stmt_params(stmt); } - thd->set_statement(&thd->stmt_backup); + thd->set_statement(&stmt_backup); thd->current_arena= thd; DBUG_VOID_RETURN; @@ -2089,6 +2090,7 @@ void mysql_sql_stmt_execute(THD *thd, LEX_STRING *stmt_name) binary log. */ String expanded_query; + Statement stmt_backup; DBUG_ENTER("mysql_sql_stmt_execute"); DBUG_ASSERT(thd->free_list == NULL); @@ -2110,16 +2112,16 @@ void mysql_sql_stmt_execute(THD *thd, LEX_STRING *stmt_name) /* Must go before setting variables, as it clears thd->user_var_events */ mysql_reset_thd_for_next_command(thd); - thd->set_n_backup_statement(stmt, &thd->stmt_backup); - thd->set_statement(stmt); + thd->set_n_backup_statement(stmt, &stmt_backup); if (stmt->set_params_from_vars(stmt, - thd->stmt_backup.lex->prepared_stmt_params, + stmt_backup.lex->prepared_stmt_params, &expanded_query)) { my_error(ER_WRONG_ARGUMENTS, MYF(0), "EXECUTE"); } thd->command= COM_STMT_EXECUTE; /* For nice messages in general log */ execute_stmt(thd, stmt, &expanded_query); + thd->set_statement(&stmt_backup); DBUG_VOID_RETURN; } @@ -2176,7 +2178,6 @@ static void execute_stmt(THD *thd, Prepared_statement *stmt, close_thread_tables(thd); // to close derived tables cleanup_stmt_and_thd_after_use(stmt, thd); reset_stmt_params(stmt); - thd->set_statement(&thd->stmt_backup); thd->current_arena= thd; if (stmt->state == Query_arena::PREPARED) @@ -2201,6 +2202,7 @@ void mysql_stmt_fetch(THD *thd, char *packet, uint packet_length) ulong stmt_id= uint4korr(packet); ulong num_rows= uint4korr(packet+4); Prepared_statement *stmt; + Statement stmt_backup; DBUG_ENTER("mysql_stmt_fetch"); statistic_increment(thd->status_var.com_stmt_fetch, &LOCK_status); @@ -2214,7 +2216,7 @@ void mysql_stmt_fetch(THD *thd, char *packet, uint packet_length) } thd->current_arena= stmt; - thd->set_n_backup_statement(stmt, &thd->stmt_backup); + thd->set_n_backup_statement(stmt, &stmt_backup); if (!(specialflag & SPECIAL_NO_PRIOR)) my_pthread_setprio(pthread_self(), QUERY_PRIOR); @@ -2226,7 +2228,7 @@ void mysql_stmt_fetch(THD *thd, char *packet, uint packet_length) if (!(specialflag & SPECIAL_NO_PRIOR)) my_pthread_setprio(pthread_self(), WAIT_PRIOR); - thd->restore_backup_statement(stmt, &thd->stmt_backup); + thd->restore_backup_statement(stmt, &stmt_backup); thd->current_arena= thd; if (!stmt->cursor->is_open()) @@ -2386,7 +2388,9 @@ void mysql_stmt_get_longdata(THD *thd, char *packet, ulong packet_length) Prepared_statement::Prepared_statement(THD *thd_arg) - :Statement(thd_arg), + :Statement(INITIALIZED, ++thd_arg->statement_id_counter, + thd_arg->variables.query_alloc_block_size, + thd_arg->variables.query_prealloc_size), thd(thd_arg), param_array(0), param_count(0), @@ -2425,7 +2429,7 @@ Prepared_statement::~Prepared_statement() { if (cursor) cursor->Cursor::~Cursor(); - free_items(free_list); + free_items(); delete lex->result; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b72cce30bbc..b487637ba9c 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -87,10 +87,9 @@ static void update_depend_map(JOIN *join, ORDER *order); static ORDER *remove_const(JOIN *join,ORDER *first_order,COND *cond, bool change_list, bool *simple_order); static int return_zero_rows(JOIN *join, select_result *res,TABLE_LIST *tables, - List<Item> &fields, bool send_row, - uint select_options, const char *info, - Item *having, Procedure *proc, - SELECT_LEX_UNIT *unit); + List<Item> &fields, bool send_row, + uint select_options, const char *info, + Item *having); static COND *build_equal_items(THD *thd, COND *cond, COND_EQUAL *inherited, List<TABLE_LIST> *join_list, @@ -1227,8 +1226,7 @@ JOIN::exec() send_row_on_empty_set(), select_options, zero_result_cause, - having, procedure, - unit); + having); DBUG_VOID_RETURN; } @@ -1437,7 +1435,7 @@ JOIN::exec() DBUG_VOID_RETURN; } end_read_record(&curr_join->join_tab->read_record); - curr_join->const_tables= curr_join->tables; // Mark free for join_free() + curr_join->const_tables= curr_join->tables; // Mark free for cleanup() curr_join->join_tab[0].table= 0; // Table is freed // No sum funcs anymore @@ -1667,9 +1665,9 @@ JOIN::exec() */ int -JOIN::cleanup() +JOIN::destroy() { - DBUG_ENTER("JOIN::cleanup"); + DBUG_ENTER("JOIN::destroy"); select_lex->join= 0; if (tmp_join) @@ -1684,12 +1682,11 @@ JOIN::cleanup() } tmp_join->tmp_join= 0; tmp_table_param.copy_field=0; - DBUG_RETURN(tmp_join->cleanup()); + DBUG_RETURN(tmp_join->destroy()); } cond_equal= 0; - lock=0; // It's faster to unlock later - join_free(1); + cleanup(1); if (exec_tmp_table1) free_tmp_table(thd, exec_tmp_table1); if (exec_tmp_table2) @@ -1697,12 +1694,6 @@ JOIN::cleanup() delete select; delete_dynamic(&keyuse); delete procedure; - for (SELECT_LEX_UNIT *lex_unit= select_lex->first_inner_unit(); - lex_unit != 0; - lex_unit= lex_unit->next_unit()) - { - error|= lex_unit->cleanup(); - } DBUG_RETURN(error); } @@ -1824,6 +1815,7 @@ Cursor::fetch(ulong num_rows) THD *thd= join->thd; JOIN_TAB *join_tab= join->join_tab + join->const_tables; enum_nested_loop_state error= NESTED_LOOP_OK; + Query_arena backup_arena; DBUG_ENTER("Cursor::fetch"); DBUG_PRINT("enter",("rows: %lu", num_rows)); @@ -1835,7 +1827,7 @@ Cursor::fetch(ulong num_rows) thd->lock= lock; thd->query_id= query_id; /* save references to memory, allocated during fetch */ - thd->set_n_backup_item_arena(this, &thd->stmt_backup); + thd->set_n_backup_item_arena(this, &backup_arena); join->fetch_limit+= num_rows; @@ -1851,7 +1843,7 @@ Cursor::fetch(ulong num_rows) ha_release_temporary_latches(thd); #endif - thd->restore_backup_item_arena(this, &thd->stmt_backup); + thd->restore_backup_item_arena(this, &backup_arena); DBUG_ASSERT(thd->free_list == 0); reset_thd(thd); @@ -1884,17 +1876,14 @@ Cursor::close() THD *thd= join->thd; DBUG_ENTER("Cursor::close"); - join->join_free(0); + /* + In case of UNIONs JOIN is freed inside of unit->cleanup(), + otherwise in select_lex->cleanup(). + */ if (unit) - { - /* In case of UNIONs JOIN is freed inside unit->cleanup() */ - unit->cleanup(); - } + (void) unit->cleanup(); else - { - join->cleanup(); - delete join; - } + (void) join->select_lex->cleanup(); { /* XXX: Another hack: closing tables used in the cursor */ DBUG_ASSERT(lock || open_tables || derived_tables); @@ -1913,8 +1902,7 @@ Cursor::close() } join= 0; unit= 0; - free_items(free_list); - free_list= 0; + free_items(); /* Must be last, as some memory might be allocated for free purposes, like in free_tmp_table() (TODO: fix this issue) @@ -2071,8 +2059,7 @@ err: if (free_join) { thd->proc_info="end"; - err= join->cleanup(); - delete join; + err= select_lex->cleanup(); DBUG_RETURN(err || thd->net.report_error); } DBUG_RETURN(join->error); @@ -5905,29 +5892,68 @@ void JOIN_TAB::cleanup() } +void JOIN::join_free(bool full) +{ + SELECT_LEX_UNIT *unit; + SELECT_LEX *sl; + DBUG_ENTER("JOIN::join_free"); + + /* + Optimization: if not EXPLAIN and we are done with the JOIN, + free all tables. + */ + full= full || (!select_lex->uncacheable && !thd->lex->describe); + + cleanup(full); + + for (unit= select_lex->first_inner_unit(); unit; unit= unit->next_unit()) + for (sl= unit->first_select_in_union(); sl; sl= sl->next_select()) + { + JOIN *join= sl->join; + if (join) + join->join_free(full); + } + + /* + We are not using tables anymore + Unlock all tables. We may be in an INSERT .... SELECT statement. + */ + if (full && lock && thd->lock && !(select_options & SELECT_NO_UNLOCK) && + !select_lex->subquery_in_having && + (select_lex == (thd->lex->unit.fake_select_lex ? + thd->lex->unit.fake_select_lex : &thd->lex->select_lex))) + { + /* + TODO: unlock tables even if the join isn't top level select in the + tree. + */ + mysql_unlock_read_tables(thd, lock); // Don't free join->lock + lock= 0; + } + + DBUG_VOID_RETURN; +} + + /* Free resources of given join SYNOPSIS - JOIN::join_free() + JOIN::cleanup() fill - true if we should free all resources, call with full==1 should be last, before it this function can be called with full==0 NOTE: with subquery this function definitely will be called several times, but even for simple query it can be called several times. */ -void -JOIN::join_free(bool full) -{ - JOIN_TAB *tab,*end; - DBUG_ENTER("JOIN::join_free"); - full= full || (!select_lex->uncacheable && - !thd->lex->subqueries && - !thd->lex->describe); // do not cleanup too early on EXPLAIN +void JOIN::cleanup(bool full) +{ + DBUG_ENTER("JOIN::cleanup"); if (table) { + JOIN_TAB *tab,*end; /* Only a sorted table may be cached. This sorted table is always the first non const table in join->table @@ -5938,16 +5964,6 @@ JOIN::join_free(bool full) filesort_free_buffers(table[const_tables]); } - for (SELECT_LEX_UNIT *unit= select_lex->first_inner_unit(); unit; - unit= unit->next_unit()) - { - JOIN *join; - for (SELECT_LEX *sl= unit->first_select_in_union(); sl; - sl= sl->next_select()) - if ((join= sl->join)) - join->join_free(full); - } - if (full) { for (tab= join_tab, end= tab+tables; tab != end; tab++) @@ -5964,23 +5980,10 @@ JOIN::join_free(bool full) } } } - /* We are not using tables anymore Unlock all tables. We may be in an INSERT .... SELECT statement. */ - if (full && lock && thd->lock && !(select_options & SELECT_NO_UNLOCK) && - !select_lex->subquery_in_having) - { - // TODO: unlock tables even if the join isn't top level select in the tree - if (select_lex == (thd->lex->unit.fake_select_lex ? - thd->lex->unit.fake_select_lex : &thd->lex->select_lex)) - { - mysql_unlock_read_tables(thd, lock); // Don't free join->lock - lock=0; - } - } - if (full) { group_fields.delete_elements(); @@ -6217,8 +6220,7 @@ remove_const(JOIN *join,ORDER *first_order, COND *cond, static int return_zero_rows(JOIN *join, select_result *result,TABLE_LIST *tables, List<Item> &fields, bool send_row, uint select_options, - const char *info, Item *having, Procedure *procedure, - SELECT_LEX_UNIT *unit) + const char *info, Item *having) { DBUG_ENTER("return_zero_rows"); @@ -11765,10 +11767,12 @@ cp_buffer_from_ref(THD *thd, TABLE_REF *ref) thd->count_cuted_fields= CHECK_FIELD_IGNORE; for (store_key **copy=ref->key_copy ; *copy ; copy++) { - if ((*copy)->copy()) + int res; + if ((res= (*copy)->copy())) { thd->count_cuted_fields= save_count_cuted_fields; - return 1; // Something went wrong + if ((res= res & 1)) + return res; // Something went wrong } } thd->count_cuted_fields= save_count_cuted_fields; diff --git a/sql/sql_select.h b/sql/sql_select.h index e5266944251..6d33ae2f978 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -325,7 +325,7 @@ class JOIN :public Sql_alloc int optimize(); int reinit(); void exec(); - int cleanup(); + int destroy(); void restore_tmp(); bool alloc_func_list(); bool make_sum_func_list(List<Item> &all_fields, List<Item> &send_fields, @@ -349,7 +349,15 @@ class JOIN :public Sql_alloc int rollup_send_data(uint idx); int rollup_write_data(uint idx, TABLE *table); bool test_in_subselect(Item **where); + /* + Release memory and, if possible, the open tables held by this execution + plan (and nested plans). It's used to release some tables before + the end of execution in order to increase concurrency and reduce + memory consumption. + */ void join_free(bool full); + /* Cleanup this JOIN, possibly for reuse */ + void cleanup(bool full); void clear(); bool save_join_tab(); bool send_row_on_empty_set() @@ -448,6 +456,7 @@ class store_key :public Sql_alloc char *null_ptr; char err; public: + enum store_key_result { STORE_KEY_OK, STORE_KEY_FATAL, STORE_KEY_CONV }; store_key(THD *thd, Field *field_arg, char *ptr, char *null, uint length) :null_ptr(null),err(0) { @@ -463,7 +472,7 @@ class store_key :public Sql_alloc ptr, (uchar*) null, 1); } virtual ~store_key() {} /* Not actually needed */ - virtual bool copy()=0; + virtual enum store_key_result copy()=0; virtual const char *name() const=0; }; @@ -484,10 +493,10 @@ class store_key_field: public store_key copy_field.set(to_field,from_field,0); } } - bool copy() + enum store_key_result copy() { copy_field.do_copy(©_field); - return err != 0; + return err != 0 ? STORE_KEY_FATAL : STORE_KEY_OK; } const char *name() const { return field_name; } }; @@ -504,9 +513,11 @@ public: null_ptr_arg ? null_ptr_arg : item_arg->maybe_null ? &err : NullS, length), item(item_arg) {} - bool copy() + enum store_key_result copy() { - return item->save_in_field_no_warnings(to_field, 1) || err != 0; + int res= item->save_in_field(to_field, 1); + return (err != 0 || res > 2 ? STORE_KEY_FATAL : (store_key_result) res); + } const char *name() const { return "func"; } }; @@ -524,15 +535,19 @@ public: &err : NullS, length, item_arg), inited(0) { } - bool copy() + enum store_key_result copy() { + int res; if (!inited) { inited=1; - if (item->save_in_field(to_field, 1)) - err= 1; + if ((res= item->save_in_field(to_field, 1))) + { + if (!err) + err= res; + } } - return err != 0; + return (err > 2 ? STORE_KEY_FATAL : (store_key_result) err); } const char *name() const { return "const"; } }; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index f59d7fffe85..87b67a5127a 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -553,7 +553,6 @@ bool st_select_lex_unit::exec() bool st_select_lex_unit::cleanup() { int error= 0; - JOIN *join; DBUG_ENTER("st_select_lex_unit::cleanup"); if (cleaned) @@ -572,29 +571,17 @@ bool st_select_lex_unit::cleanup() } for (SELECT_LEX *sl= first_select_in_union(); sl; sl= sl->next_select()) + error|= sl->cleanup(); + + if (fake_select_lex) { - if ((join= sl->join)) - { - error|= sl->join->cleanup(); - delete join; - } - else + JOIN *join; + if ((join= fake_select_lex->join)) { - // it can be DO/SET with subqueries - for (SELECT_LEX_UNIT *lex_unit= sl->first_inner_unit(); - lex_unit != 0; - lex_unit= lex_unit->next_unit()) - { - error|= lex_unit->cleanup(); - } + join->tables_list= 0; + join->tables= 0; } - } - if (fake_select_lex && (join= fake_select_lex->join)) - { - join->tables_list= 0; - join->tables= 0; - error|= join->cleanup(); - delete join; + error|= fake_select_lex->cleanup(); } DBUG_RETURN(error); @@ -650,3 +637,24 @@ bool st_select_lex_unit::change_result(select_subselect *result, res= fake_select_lex->join->change_result(result); return (res); } + + +bool st_select_lex::cleanup() +{ + bool error= FALSE; + DBUG_ENTER("st_select_lex::cleanup()"); + + if (join) + { + error|= join->destroy(); + delete join; + join= 0; + } + for (SELECT_LEX_UNIT *lex_unit= first_inner_unit(); lex_unit ; + lex_unit= lex_unit->next_unit()) + { + error|= lex_unit->cleanup(); + } + DBUG_RETURN(error); +} + diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 1b9176e2744..360bc421965 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -6120,9 +6120,24 @@ insert_update_elem: simple_ident_nospvar equal expr_or_default { LEX *lex= Lex; + uint8 tmp= MY_ITEM_PREFER_1ST_TABLE; if (lex->update_list.push_back($1) || lex->value_list.push_back($3)) YYABORT; + /* + INSERT INTO a1(a) SELECT b1.a FROM b1 ON DUPLICATE KEY + UPDATE a= a + b1.b + + Set MY_ITEM_PREFER_1ST_TABLE flag to $1 and $3 items + to prevent find_field_in_tables() doing further item searching + if it finds item occurence in first table in insert_table_list. + This allows to avoid ambiguity in resolving 'a' field in + example above. + */ + $1->walk(&Item::set_flags_processor, + (byte *) &tmp); + $3->walk(&Item::set_flags_processor, + (byte *) &tmp); }; opt_low_priority: |