From ac2b92c4760c7aa111d3f115f9af40fc657e18ed Mon Sep 17 00:00:00 2001 From: Vicentiu Ciorbaru Date: Tue, 28 Apr 2015 15:09:04 +0300 Subject: MDEV-7912 multitable delete with wrongly set sort_buffer_size crashes in merge_buffers Fixed overflow error that caused fewer bites to be allocated than necessary on Windows 64 bit. This is due to ulong being 32 bit on 64 bit Windows and 64 bit on 64 bit Linux. --- sql/uniques.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'sql/uniques.cc') diff --git a/sql/uniques.cc b/sql/uniques.cc index 72411be5cd6..455fe205717 100644 --- a/sql/uniques.cc +++ b/sql/uniques.cc @@ -97,7 +97,7 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg, max_elements= (ulong) (max_in_memory_size / ALIGN_SIZE(sizeof(TREE_ELEMENT)+size)); (void) open_cached_file(&file, mysql_tmpdir,TEMP_PREFIX, DISK_BUFFER_SIZE, - MYF(MY_WME)); + MYF(MY_WME)); } @@ -607,8 +607,10 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) return 1; if (flush_io_cache(&file) || reinit_io_cache(&file, READ_CACHE, 0L, 0, 0)) return 1; - ulong buff_sz= (max_in_memory_size / full_size + 1) * full_size; - if (!(merge_buffer= (uchar *) my_malloc((ulong) buff_sz, MYF(0)))) + size_t buff_sz= (max_in_memory_size / full_size + 1) * full_size; + DBUG_EXECUTE_IF("make_merge_buff_alloc_fail", + DBUG_SET("+d,simulate_out_of_memory");); + if (!(merge_buffer = (uchar *)my_malloc(buff_sz, MYF(MY_WME)))) return 1; if (buff_sz < (ulong) (full_size * (file_ptrs.elements + 1))) res= merge(table, merge_buffer, buff_sz >= full_size * MERGEBUFF2) ; @@ -737,9 +739,10 @@ bool Unique::get(TABLE *table) /* Not enough memory; Save the result to file && free memory used by tree */ if (flush()) return 1; - - ulong buff_sz= (max_in_memory_size / full_size + 1) * full_size; - if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(0)))) + DBUG_EXECUTE_IF("make_merge_buff_alloc_fail", + DBUG_SET("+d,simulate_out_of_memory");); + size_t buff_sz= (max_in_memory_size / full_size + 1) * full_size; + if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(MY_WME)))) return 1; if (merge(table, sort_buffer, FALSE)) -- cgit v1.2.1