diff options
Diffstat (limited to 'sql/uniques.cc')
| -rw-r--r-- | sql/uniques.cc | 106 |
1 files changed, 65 insertions, 41 deletions
diff --git a/sql/uniques.cc b/sql/uniques.cc index 1b8e91f2566..a0cebe3e4dd 100644 --- a/sql/uniques.cc +++ b/sql/uniques.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2001, 2010, Oracle and/or its affiliates. - Copyright (c) 2010, 2015, MariaDB + Copyright (c) 2010, 2020, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,7 +39,6 @@ #include "my_tree.h" // element_count #include "uniques.h" // Unique #include "sql_sort.h" -#include "myisamchk.h" // BUFFPEK int unique_write_to_file(uchar* key, element_count count, Unique *unique) { @@ -94,8 +93,8 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg, init_tree(&tree, (max_in_memory_size / 16), 0, size, comp_func, NULL, comp_func_fixed_arg, MYF(MY_THREAD_SPECIFIC)); /* If the following fail's the next add will also fail */ - my_init_dynamic_array(&file_ptrs, sizeof(BUFFPEK), 16, 16, - MYF(MY_THREAD_SPECIFIC)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &file_ptrs, sizeof(Merge_chunk), 16, + 16, MYF(MY_THREAD_SPECIFIC)); /* If you change the following, change it in get_max_elements function, too. */ @@ -162,7 +161,7 @@ inline double log2_n_fact(double x) static double get_merge_buffers_cost(uint *buff_elems, uint elem_size, uint *first, uint *last, - uint compare_factor) + double compare_factor) { uint total_buf_elems= 0; for (uint *pbuf= first; pbuf <= last; pbuf++) @@ -207,7 +206,7 @@ static double get_merge_buffers_cost(uint *buff_elems, uint elem_size, static double get_merge_many_buffs_cost(uint *buffer, uint maxbuffer, uint max_n_elems, uint last_n_elems, int elem_size, - uint compare_factor) + double compare_factor) { int i; double total_cost= 0.0; @@ -307,7 +306,7 @@ static double get_merge_many_buffs_cost(uint *buffer, double Unique::get_use_cost(uint *buffer, size_t nkeys, uint key_size, size_t max_in_memory_size, - uint compare_factor, + double compare_factor, bool intersect_fl, bool *in_memory) { size_t max_elements_in_tree; @@ -378,10 +377,10 @@ Unique::~Unique() /* Write tree to disk; clear tree */ bool Unique::flush() { - BUFFPEK file_ptr; + Merge_chunk file_ptr; elements+= tree.elements_in_tree; - file_ptr.count=tree.elements_in_tree; - file_ptr.file_pos=my_b_tell(&file); + file_ptr.set_rowcount(tree.elements_in_tree); + file_ptr.set_file_position(my_b_tell(&file)); tree_walk_action action= min_dupl_count ? (tree_walk_action) unique_write_to_file_with_count : @@ -493,7 +492,7 @@ void put_counter_into_merged_element(void *ptr, uint ofs, element_count cnt) */ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, - uint key_length, BUFFPEK *begin, BUFFPEK *end, + uint key_length, Merge_chunk *begin, Merge_chunk *end, tree_walk_action walk_action, void *walk_action_arg, qsort_cmp2 compare, void *compare_arg, IO_CACHE *file, bool with_counters) @@ -502,7 +501,8 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, QUEUE queue; if (end <= begin || merge_buffer_size < (size_t) (key_length * (end - begin + 1)) || - init_queue(&queue, (uint) (end - begin), offsetof(BUFFPEK, key), 0, + init_queue(&queue, (uint) (end - begin), + offsetof(Merge_chunk, m_current_key), 0, buffpek_compare, &compare_context, 0, 0)) return 1; /* we need space for one key when a piece of merge buffer is re-read */ @@ -513,10 +513,16 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, /* if piece_size is aligned reuse_freed_buffer will always hit */ uint piece_size= max_key_count_per_piece * key_length; ulong bytes_read; /* to hold return value of read_to_buffer */ - BUFFPEK *top; + Merge_chunk *top; int res= 1; uint cnt_ofs= key_length - (with_counters ? sizeof(element_count) : 0); element_count cnt; + + // read_to_buffer() needs only rec_length. + Sort_param sort_param; + sort_param.rec_length= key_length; + DBUG_ASSERT(!sort_param.using_addon_fields()); + /* Invariant: queue must contain top element from each tree, until a tree is not completely walked through. @@ -525,15 +531,16 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, */ for (top= begin; top != end; ++top) { - top->base= merge_buffer + (top - begin) * piece_size; - top->max_keys= max_key_count_per_piece; - bytes_read= read_to_buffer(file, top, key_length); + top->set_buffer(merge_buffer + (top - begin) * piece_size, + merge_buffer + (top - begin) * piece_size + piece_size); + top->set_max_keys(max_key_count_per_piece); + bytes_read= read_to_buffer(file, top, &sort_param, false); if (unlikely(bytes_read == (ulong) -1)) goto end; DBUG_ASSERT(bytes_read); queue_insert(&queue, (uchar *) top); } - top= (BUFFPEK *) queue_top(&queue); + top= (Merge_chunk *) queue_top(&queue); while (queue.elements > 1) { /* @@ -543,20 +550,21 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, elements in each tree are unique. Action is applied only to unique elements. */ - void *old_key= top->key; + void *old_key= top->current_key(); /* read next key from the cache or from the file and push it to the queue; this gives new top. */ - top->key+= key_length; - if (--top->mem_count) + top->advance_current_key(key_length); + top->decrement_mem_count(); + if (top->mem_count()) queue_replace_top(&queue); else /* next piece should be read */ { /* save old_key not to overwrite it in read_to_buffer */ memcpy(save_key_buff, old_key, key_length); old_key= save_key_buff; - bytes_read= read_to_buffer(file, top, key_length); + bytes_read= read_to_buffer(file, top, &sort_param, false); if (unlikely(bytes_read == (ulong) -1)) goto end; else if (bytes_read) /* top->key, top->mem_count are reset */ @@ -571,9 +579,9 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, reuse_freed_buff(&queue, top, key_length); } } - top= (BUFFPEK *) queue_top(&queue); + top= (Merge_chunk *) queue_top(&queue); /* new top has been obtained; if old top is unique, apply the action */ - if (compare(compare_arg, old_key, top->key)) + if (compare(compare_arg, old_key, top->current_key())) { cnt= with_counters ? get_counter_from_merged_element(old_key, cnt_ofs) : 1; @@ -582,9 +590,9 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, } else if (with_counters) { - cnt= get_counter_from_merged_element(top->key, cnt_ofs); + cnt= get_counter_from_merged_element(top->current_key(), cnt_ofs); cnt+= get_counter_from_merged_element(old_key, cnt_ofs); - put_counter_into_merged_element(top->key, cnt_ofs, cnt); + put_counter_into_merged_element(top->current_key(), cnt_ofs, cnt); } } /* @@ -598,13 +606,13 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, { cnt= with_counters ? - get_counter_from_merged_element(top->key, cnt_ofs) : 1; - if (walk_action(top->key, cnt, walk_action_arg)) + get_counter_from_merged_element(top->current_key(), cnt_ofs) : 1; + if (walk_action(top->current_key(), cnt, walk_action_arg)) goto end; - top->key+= key_length; + top->advance_current_key(key_length); } - while (--top->mem_count); - bytes_read= read_to_buffer(file, top, key_length); + while (top->decrement_mem_count()); + bytes_read= read_to_buffer(file, top, &sort_param, false); if (unlikely(bytes_read == (ulong) -1)) goto end; } @@ -657,16 +665,18 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) is needed when a piece of merge buffer is re-read, see merge_walk() */ size_t buff_sz= MY_MAX(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size; - if (!(merge_buffer = (uchar *)my_malloc(buff_sz, MYF(MY_WME)))) + if (!(merge_buffer = (uchar *)my_malloc(key_memory_Unique_merge_buffer, + buff_sz, MYF(MY_THREAD_SPECIFIC|MY_WME)))) return 1; if (buff_sz < full_size * (file_ptrs.elements + 1UL)) - res= merge(table, merge_buffer, buff_sz >= full_size * MERGEBUFF2) ; + res= merge(table, merge_buffer, buff_sz, + buff_sz >= full_size * MERGEBUFF2) ; if (!res) { res= merge_walk(merge_buffer, buff_sz, full_size, - (BUFFPEK *) file_ptrs.buffer, - (BUFFPEK *) file_ptrs.buffer + file_ptrs.elements, + (Merge_chunk *) file_ptrs.buffer, + (Merge_chunk *) file_ptrs.buffer + file_ptrs.elements, action, walk_action_arg, tree.compare, tree.custom_arg, &file, with_counters); } @@ -687,16 +697,18 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) All params are 'IN': table the parameter to access sort context buff merge buffer + buff_size size of merge buffer without_last_merge TRUE <=> do not perform the last merge RETURN VALUE 0 OK <> 0 error */ -bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) +bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size, + bool without_last_merge) { IO_CACHE *outfile= &sort.io_cache; - BUFFPEK *file_ptr= (BUFFPEK*) file_ptrs.buffer; + Merge_chunk *file_ptr= (Merge_chunk*) file_ptrs.buffer; uint maxbuffer= file_ptrs.elements - 1; my_off_t save_pos; bool error= 1; @@ -726,8 +738,17 @@ bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) sort_param.cmp_context.key_compare= tree.compare; sort_param.cmp_context.key_compare_arg= tree.custom_arg; + /* + We need to remove the size allocated for the unique buffer. + The sort_buffer_size is: + MY_MAX(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size; + */ + buff_size-= full_size; + /* Merge the buffers to one file, removing duplicates */ - if (merge_many_buff(&sort_param,buff,file_ptr,&maxbuffer,&file)) + if (merge_many_buff(&sort_param, + Bounds_checked_array<uchar>(buff, buff_size), + file_ptr,&maxbuffer,&file)) goto err; if (flush_io_cache(&file) || reinit_io_cache(&file,READ_CACHE,0L,0,0)) @@ -739,7 +760,8 @@ bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge) file_ptrs.elements= maxbuffer+1; return 0; } - if (merge_index(&sort_param, buff, file_ptr, maxbuffer, &file, outfile)) + if (merge_index(&sort_param, Bounds_checked_array<uchar>(buff, buff_size), + file_ptr, maxbuffer, &file, outfile)) goto err; error= 0; err: @@ -771,7 +793,8 @@ bool Unique::get(TABLE *table) { /* Whole tree is in memory; Don't use disk if you don't need to */ if ((sort.record_pointers= (uchar*) - my_malloc(size * tree.elements_in_tree, MYF(MY_THREAD_SPECIFIC)))) + my_malloc(key_memory_Filesort_info_record_pointers, + size * tree.elements_in_tree, MYF(MY_THREAD_SPECIFIC)))) { uchar *save_record_pointers= sort.record_pointers; tree_walk_action action= min_dupl_count ? @@ -795,11 +818,12 @@ bool Unique::get(TABLE *table) one key for Sort_param::unique_buff */ size_t buff_sz= MY_MAX(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size; - if (!(sort_buffer= (uchar*) my_malloc(buff_sz, + + if (!(sort_buffer= (uchar*) my_malloc(key_memory_Unique_sort_buffer, buff_sz, MYF(MY_THREAD_SPECIFIC|MY_WME)))) DBUG_RETURN(1); - if (merge(table, sort_buffer, FALSE)) + if (merge(table, sort_buffer, buff_sz, FALSE)) goto err; rc= 0; |