diff options
Diffstat (limited to 'sql')
47 files changed, 648 insertions, 288 deletions
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index f4101832214..4938f8da02b 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -237,6 +237,15 @@ FOREACH(se aria partition perfschema sql_sequence wsrep) ENDIF() ENDFOREACH() +IF(VISIBILITY_HIDDEN_FLAG AND TARGET partition AND WITH_UBSAN) + # the spider plugin needs some partition symbols from inside mysqld + # when built with ubsan, in which case we need to remove + # -fvisibility=hidden from partition + GET_TARGET_PROPERTY(f partition COMPILE_FLAGS) + STRING(REPLACE "${VISIBILITY_HIDDEN_FLAG}" "" f ${f}) + SET_TARGET_PROPERTIES(partition PROPERTIES COMPILE_FLAGS "${f}") +ENDIF() + IF(WIN32) SET(MYSQLD_SOURCE main.cc message.rc) ELSE() diff --git a/sql/discover.h b/sql/discover.h index 1775f5d6551..750c2944ede 100644 --- a/sql/discover.h +++ b/sql/discover.h @@ -28,7 +28,7 @@ int writefile(const char *path, const char *db, const char *table, inline void deletefrm(const char *path) { char frm_name[FN_REFLEN]; - strxmov(frm_name, path, reg_ext, NullS); + strxnmov(frm_name, sizeof(frm_name)-1, path, reg_ext, NullS); mysql_file_delete(key_file_frm, frm_name, MYF(0)); } diff --git a/sql/field.cc b/sql/field.cc index 6b782f79d3a..5a618a5a2a9 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1924,17 +1924,11 @@ Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg, } -void Field::hash(ulong *nr, ulong *nr2) +void Field::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - uint len= pack_length(); - sort_charset()->hash_sort(ptr, len, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + hasher->add(sort_charset(), ptr, pack_length()); } size_t @@ -8345,17 +8339,12 @@ bool Field_varstring::is_equal(const Column_definition &new_field) const } -void Field_varstring::hash(ulong *nr, ulong *nr2) +void Field_varstring::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); - charset()->hash_sort(ptr + length_bytes, len, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); + hasher->add(charset(), ptr + length_bytes, len); } @@ -8730,6 +8719,17 @@ oom_error: } +void Field_blob::hash_not_null(Hasher *hasher) +{ + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + char *blob; + memcpy(&blob, ptr + packlength, sizeof(char*)); + if (blob) + hasher->add(Field_blob::charset(), blob, get_length(ptr)); +} + + double Field_blob::val_real(void) { DBUG_ASSERT(marked_for_read()); @@ -9792,20 +9792,27 @@ const DTCollation & Field_bit::dtcollation() const } -void Field_bit::hash(ulong *nr, ulong *nr2) +/* + This method always calculates hash over 8 bytes. + This is different from how the HEAP engine calculate hash: + HEAP takes into account the actual octet size, so say for BIT(18) + it calculates hash over three bytes only: + - the incomplete byte with bits 16..17 + - the two full bytes with bits 0..15 + See hp_rec_hashnr(), hp_hashnr() for details. + + The HEAP way is more efficient, especially for short lengths. + Let's consider fixing Field_bit eventually to do it in the HEAP way, + with proper measures to upgrade partitioned tables easy. +*/ +void Field_bit::hash_not_null(Hasher *hasher) { - if (is_null()) - { - *nr^= (*nr << 1) | 1; - } - else - { - CHARSET_INFO *cs= &my_charset_bin; - longlong value= Field_bit::val_int(); - uchar tmp[8]; - mi_int8store(tmp,value); - cs->hash_sort(tmp, 8, nr, nr2); - } + DBUG_ASSERT(marked_for_read()); + DBUG_ASSERT(!is_null()); + longlong value= Field_bit::val_int(); + uchar tmp[8]; + mi_int8store(tmp,value); + hasher->add(&my_charset_bin, tmp, 8); } diff --git a/sql/field.h b/sql/field.h index 4036f032257..bbd6595098f 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1845,7 +1845,14 @@ public: key_map get_possible_keys(); /* Hash value */ - virtual void hash(ulong *nr, ulong *nr2); + void hash(Hasher *hasher) + { + if (is_null()) + hasher->add_null(); + else + hash_not_null(hasher); + } + virtual void hash_not_null(Hasher *hasher); /** Get the upper limit of the MySQL integral and floating-point type. @@ -4221,7 +4228,7 @@ public: uchar *new_ptr, uint32 length, uchar *new_null_ptr, uint new_null_bit) override; bool is_equal(const Column_definition &new_field) const override; - void hash(ulong *nr, ulong *nr2) override; + void hash_not_null(Hasher *hasher) override; uint length_size() const override { return length_bytes; } void print_key_value(String *out, uint32 length) override; Binlog_type_info binlog_type_info() const override; @@ -4481,6 +4488,7 @@ public: bool make_empty_rec_store_default_value(THD *thd, Item *item) override; int store(const char *to, size_t length, CHARSET_INFO *charset) override; using Field_str::store; + void hash_not_null(Hasher *hasher) override; double val_real() override; longlong val_int() override; String *val_str(String *, String *) override; @@ -5051,7 +5059,7 @@ public: if (bit_ptr) bit_ptr= ADD_TO_PTR(bit_ptr, ptr_diff, uchar*); } - void hash(ulong *nr, ulong *nr2) override; + void hash_not_null(Hasher *hasher) override; SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *param, KEY_PART *key_part, const Item_bool_func *cond, diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 08e3b45e02b..fdb4726a930 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -10034,8 +10034,7 @@ uint8 ha_partition::table_cache_type() uint32 ha_partition::calculate_key_hash_value(Field **field_array) { - ulong nr1= 1; - ulong nr2= 4; + Hasher hasher; bool use_51_hash; use_51_hash= MY_TEST((*field_array)->table->part_info->key_algorithm == partition_info::KEY_ALGORITHM_51); @@ -10062,12 +10061,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) { if (field->is_null()) { - nr1^= (nr1 << 1) | 1; + hasher.add_null(); continue; } /* Force this to my_hash_sort_bin, which was used in 5.1! */ uint len= field->pack_length(); - my_charset_bin.hash_sort(field->ptr, len, &nr1, &nr2); + hasher.add(&my_charset_bin, field->ptr, len); /* Done with this field, continue with next one. */ continue; } @@ -10085,12 +10084,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) { if (field->is_null()) { - nr1^= (nr1 << 1) | 1; + hasher.add_null(); continue; } /* Force this to my_hash_sort_bin, which was used in 5.1! */ uint len= field->pack_length(); - my_charset_latin1.hash_sort(field->ptr, len, &nr1, &nr2); + hasher.add(&my_charset_latin1, field->ptr, len); continue; } /* New types in mysql-5.6. */ @@ -10117,9 +10116,9 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array) } /* fall through, use collation based hashing. */ } - field->hash(&nr1, &nr2); + field->hash(&hasher); } while (*(++field_array)); - return (uint32) nr1; + return (uint32) hasher.finalize(); } diff --git a/sql/handler.cc b/sql/handler.cc index 5884339c809..1a4cc630a37 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4848,6 +4848,35 @@ int handler::check_collation_compatibility() } +int handler::check_long_hash_compatibility() const +{ + if (!table->s->old_long_hash_function()) + return 0; + KEY *key= table->key_info; + KEY *key_end= key + table->s->keys; + for ( ; key < key_end; key++) + { + if (key->algorithm == HA_KEY_ALG_LONG_HASH) + { + /* + The old (pre-MDEV-27653) hash function was wrong. + So the long hash unique constraint can have some + duplicate records. REPAIR TABLE can't fix this, + it will fail on a duplicate key error. + Only "ALTER IGNORE TABLE .. FORCE" can fix this. + So we need to return HA_ADMIN_NEEDS_ALTER here, + (not HA_ADMIN_NEEDS_UPGRADE which is used elsewhere), + to properly send the error message text corresponding + to ER_TABLE_NEEDS_REBUILD (rather than to ER_TABLE_NEEDS_UPGRADE) + to the user. + */ + return HA_ADMIN_NEEDS_ALTER; + } + } + return 0; +} + + int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) { int error; @@ -4885,6 +4914,9 @@ int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) if (unlikely((error= check_collation_compatibility()))) return error; + + if (unlikely((error= check_long_hash_compatibility()))) + return error; return check_for_upgrade(check_opt); } diff --git a/sql/handler.h b/sql/handler.h index 41bc50e52d4..4ab62302ac9 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -3497,6 +3497,7 @@ public: } int check_collation_compatibility(); + int check_long_hash_compatibility() const; int ha_check_for_upgrade(HA_CHECK_OPT *check_opt); /** to be actually called to get 'check()' functionality*/ int ha_check(THD *thd, HA_CHECK_OPT *check_opt); diff --git a/sql/item.cc b/sql/item.cc index ce55d77eb4d..92f3d55fcf9 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -10831,7 +10831,7 @@ table_map Item_direct_view_ref::used_tables() const table_map used= (*ref)->used_tables(); return (used ? used : - ((null_ref_table != NO_NULL_TABLE) ? + (null_ref_table != NO_NULL_TABLE && !null_ref_table->const_table ? null_ref_table->map : (table_map)0 )); } diff --git a/sql/item.h b/sql/item.h index e15d454c65f..344cb81c331 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1499,6 +1499,12 @@ public: */ inline ulonglong val_uint() { return (ulonglong) val_int(); } + virtual bool hash_not_null(Hasher *hasher) + { + DBUG_ASSERT(0); + return true; + } + /* Return string representation of this item object. @@ -3691,6 +3697,13 @@ public: { return Sql_mode_dependency(0, field->value_depends_on_sql_mode()); } + bool hash_not_null(Hasher *hasher) override + { + if (field->is_null()) + return true; + field->hash_not_null(hasher); + return false; + } longlong val_int_endpoint(bool left_endp, bool *incl_endp) override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; bool get_date_result(THD *thd, MYSQL_TIME *ltime,date_mode_t fuzzydate) diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 8a1e6d9ffc3..fcdb2aaf2d4 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4909,38 +4909,18 @@ Item_cond::fix_fields(THD *thd, Item **ref) if (check_stack_overrun(thd, STACK_MIN_SIZE, buff)) return TRUE; // Fatal error flag is set! - /* - The following optimization reduces the depth of an AND-OR tree. - E.g. a WHERE clause like - F1 AND (F2 AND (F2 AND F4)) - is parsed into a tree with the same nested structure as defined - by braces. This optimization will transform such tree into - AND (F1, F2, F3, F4). - Trees of OR items are flattened as well: - ((F1 OR F2) OR (F3 OR F4)) => OR (F1, F2, F3, F4) - Items for removed AND/OR levels will dangle until the death of the - entire statement. - The optimization is currently prepared statements and stored procedures - friendly as it doesn't allocate any memory and its effects are durable - (i.e. do not depend on PS/SP arguments). - */ - while ((item=li++)) + + while (li++) { - while (item->type() == Item::COND_ITEM && - ((Item_cond*) item)->functype() == functype() && - !((Item_cond*) item)->list.is_empty()) - { // Identical function - li.replace(((Item_cond*) item)->list); - ((Item_cond*) item)->list.empty(); - item= *li.ref(); // new current item - } + merge_sub_condition(li); + item= *li.ref(); if (is_top_level_item()) item->top_level_item(); /* replace degraded condition: was: <field> - become: <field> = 1 + become: <field> != 0 */ Item::Type type= item->type(); if (type == Item::FIELD_ITEM || type == Item::REF_ITEM) @@ -4956,7 +4936,9 @@ Item_cond::fix_fields(THD *thd, Item **ref) if (item->fix_fields_if_needed_for_bool(thd, li.ref())) return TRUE; /* purecov: inspected */ - item= *li.ref(); // item can be substituted in fix_fields + merge_sub_condition(li); + item= *li.ref(); // may be substituted in fix_fields/merge_item_if_possible + used_tables_cache|= item->used_tables(); if (item->can_eval_in_optimize() && !item->with_sp_var() && !cond_has_datetime_is_null(item)) @@ -5003,6 +4985,55 @@ Item_cond::fix_fields(THD *thd, Item **ref) return FALSE; } +/** + @brief + Merge a lower-level condition pointed by the iterator into this Item_cond + if possible + + @param li list iterator pointing to condition that must be + examined and merged if possible. + + @details + If an item pointed by the iterator is an instance of Item_cond with the + same functype() as this Item_cond (i.e. both are Item_cond_and or both are + Item_cond_or) then the arguments of that lower-level item can be merged + into the list of arguments of this upper-level Item_cond. + + This optimization reduces the depth of an AND-OR tree. + E.g. a WHERE clause like + F1 AND (F2 AND (F2 AND F4)) + is parsed into a tree with the same nested structure as defined + by braces. This optimization will transform such tree into + AND (F1, F2, F3, F4). + Trees of OR items are flattened as well: + ((F1 OR F2) OR (F3 OR F4)) => OR (F1, F2, F3, F4) + Items for removed AND/OR levels will dangle until the death of the + entire statement. + + The optimization is currently prepared statements and stored procedures + friendly as it doesn't allocate any memory and its effects are durable + (i.e. do not depend on PS/SP arguments). +*/ +void Item_cond::merge_sub_condition(List_iterator<Item>& li) +{ + Item *item= *li.ref(); + + /* + The check for list.is_empty() is to catch empty Item_cond_and() items. + We may encounter Item_cond_and with an empty list, because optimizer code + strips multiple equalities, combines items, then adds multiple equalities + back + */ + while (item->type() == Item::COND_ITEM && + ((Item_cond*) item)->functype() == functype() && + !((Item_cond*) item)->list.is_empty()) + { + li.replace(((Item_cond*) item)->list); + ((Item_cond*) item)->list.empty(); + item= *li.ref(); + } +} + bool Item_cond::eval_not_null_tables(void *opt_arg) diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 1422afb0fd0..4b7a1dc0596 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -3212,6 +3212,9 @@ public: Item *build_clone(THD *thd) override; bool excl_dep_on_table(table_map tab_map) override; bool excl_dep_on_grouping_fields(st_select_lex *sel) override; + +private: + void merge_sub_condition(List_iterator<Item>& li); }; template <template<class> class LI, class T> class Item_equal_iterator; diff --git a/sql/item_func.cc b/sql/item_func.cc index a07595cbbd8..5fce3a3354d 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -1767,7 +1767,7 @@ static void calc_hash_for_unique(ulong &nr1, ulong &nr2, String *str) cs->hash_sort((uchar *)str->ptr(), str->length(), &nr1, &nr2); } -longlong Item_func_hash::val_int() +longlong Item_func_hash_mariadb_100403::val_int() { DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); unsigned_flag= true; @@ -1788,6 +1788,24 @@ longlong Item_func_hash::val_int() } +longlong Item_func_hash::val_int() +{ + DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); + unsigned_flag= true; + Hasher hasher; + for(uint i= 0;i<arg_count;i++) + { + if (args[i]->hash_not_null(&hasher)) + { + null_value= 1; + return 0; + } + } + null_value= 0; + return (longlong) hasher.finalize(); +} + + bool Item_func_hash::fix_length_and_dec(THD *thd) { decimals= 0; diff --git a/sql/item_func.h b/sql/item_func.h index 520dbdc90c7..6d1cf7d312e 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1209,6 +1209,18 @@ public: } }; +class Item_func_hash_mariadb_100403: public Item_func_hash +{ +public: + Item_func_hash_mariadb_100403(THD *thd, List<Item> &item) + :Item_func_hash(thd, item) + {} + longlong val_int(); + Item *get_copy(THD *thd) + { return get_item_copy<Item_func_hash_mariadb_100403>(thd, this); } + const char *func_name() const { return "<hash_mariadb_100403>"; } +}; + class Item_longlong_func: public Item_int_func { public: diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 4d69280cb98..eb9e167ca74 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -1848,6 +1848,18 @@ bool Item_func_ucase::fix_length_and_dec(THD *thd) } +bool Item_func_left::hash_not_null(Hasher *hasher) +{ + StringBuffer<STRING_BUFFER_USUAL_SIZE> buf; + String *str= val_str(&buf); + DBUG_ASSERT((str == NULL) == null_value); + if (!str) + return true; + hasher->add(collation.collation, str->ptr(), str->length()); + return false; +} + + String *Item_func_left::val_str(String *str) { DBUG_ASSERT(fixed()); diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 30e363e9428..3a3c53385b0 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -582,6 +582,7 @@ class Item_func_left :public Item_str_func String tmp_value; public: Item_func_left(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} + bool hash_not_null(Hasher *hasher) override; String *val_str(String *) override; bool fix_length_and_dec(THD *thd) override; LEX_CSTRING func_name_cstring() const override diff --git a/sql/item_sum.h b/sql/item_sum.h index b9bd99acac2..fda35e45261 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -367,7 +367,14 @@ public: int8 aggr_level; /* nesting level of the aggregating subquery */ int8 max_arg_level; /* max level of unbound column references */ int8 max_sum_func_level;/* max level of aggregation for embedded functions */ - bool quick_group; /* If incremental update of fields */ + + /* + true (the default value) means this aggregate function can be computed + with TemporaryTableWithPartialSums algorithm (see end_update()). + false means this aggregate function needs OrderedGroupBy algorithm (see + end_write_group()). + */ + bool quick_group; /* This list is used by the check for mixing non aggregated fields and sum functions in the ONLY_FULL_GROUP_BY_MODE. We save all outer fields diff --git a/sql/log.cc b/sql/log.cc index 4572b9cca82..61e9ddfda6f 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -1801,6 +1801,7 @@ int binlog_init(void *p) binlog_hton->prepare= binlog_prepare; binlog_hton->start_consistent_snapshot= binlog_start_consistent_snapshot; } + binlog_hton->flags= HTON_NOT_USER_SELECTABLE | HTON_HIDDEN | HTON_NO_ROLLBACK; return 0; } @@ -2119,7 +2120,9 @@ int binlog_commit_by_xid(handlerton *hton, XID *xid) THD *thd= current_thd; if (thd->is_current_stmt_binlog_disabled()) - return 0; + { + return thd->wait_for_prior_commit(); + } /* the asserted state can't be reachable with xa commit */ DBUG_ASSERT(!thd->get_stmt_da()->is_error() || @@ -2151,7 +2154,9 @@ int binlog_rollback_by_xid(handlerton *hton, XID *xid) THD *thd= current_thd; if (thd->is_current_stmt_binlog_disabled()) - return 0; + { + return thd->wait_for_prior_commit(); + } if (thd->get_stmt_da()->is_error() && thd->get_stmt_da()->sql_errno() == ER_XA_RBROLLBACK) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index 100b4c25096..5294b91738b 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -313,7 +313,7 @@ static char *get_plugindir() { static char plugin_dir[2*MAX_PATH]; get_basedir(plugin_dir, sizeof(plugin_dir), mysqld_path, '/'); - strcat(plugin_dir, "/" STR(INSTALL_PLUGINDIR)); + safe_strcat(plugin_dir, sizeof(plugin_dir), "/" STR(INSTALL_PLUGINDIR)); if (access(plugin_dir, 0) == 0) return plugin_dir; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ff69af07f6e..cdb54b065e6 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5140,12 +5140,11 @@ static int init_server_components() else // full wsrep initialization { // add basedir/bin to PATH to resolve wsrep script names - char* const tmp_path= (char*)my_alloca(strlen(mysql_home) + - strlen("/bin") + 1); + size_t tmp_path_size= strlen(mysql_home) + 5; /* including "/bin" */ + char* const tmp_path= (char*)my_alloca(tmp_path_size); if (tmp_path) { - strcpy(tmp_path, mysql_home); - strcat(tmp_path, "/bin"); + snprintf(tmp_path, tmp_path_size, "%s/bin", mysql_home); wsrep_prepend_PATH(tmp_path); } else @@ -5988,8 +5987,9 @@ int mysqld_main(int argc, char **argv) char real_server_version[2 * SERVER_VERSION_LENGTH + 10]; set_server_version(real_server_version, sizeof(real_server_version)); - strcat(real_server_version, "' as '"); - strcat(real_server_version, server_version); + safe_strcat(real_server_version, sizeof(real_server_version), "' as '"); + safe_strcat(real_server_version, sizeof(real_server_version), + server_version); sql_print_information(ER_DEFAULT(ER_STARTUP), my_progname, real_server_version, @@ -7982,7 +7982,8 @@ static int mysql_init_variables(void) } else my_path(prg_dev, my_progname, "mysql/bin"); - strcat(prg_dev,"/../"); // Remove 'bin' to get base dir + // Remove 'bin' to get base dir + safe_strcat(prg_dev, sizeof(prg_dev), "/../"); cleanup_dirname(mysql_home,prg_dev); } #else diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index d8510d7becd..24367897c89 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -56,8 +56,7 @@ rpt_handle_event(rpl_parallel_thread::queued_event *qev, rgi->event_relay_log_pos= qev->event_relay_log_pos; rgi->future_event_relay_log_pos= qev->future_event_relay_log_pos; strcpy(rgi->future_event_master_log_name, qev->future_event_master_log_name); - if (!(ev->is_artificial_event() || ev->is_relay_log_event() || - (ev->when == 0))) + if (event_can_update_last_master_timestamp(ev)) rgi->last_master_timestamp= ev->when + (time_t)ev->exec_time; err= apply_event_and_update_pos_for_parallel(ev, thd, rgi); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index edbb630c781..a8af950fa08 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -2283,11 +2283,9 @@ void rpl_group_info::cleanup_context(THD *thd, bool error) if (unlikely(error)) { - /* - trans_rollback above does not rollback XA transactions - (todo/fixme consider to do so. - */ - if (thd->transaction->xid_state.is_explicit_XA()) + // leave alone any XA prepared transactions + if (thd->transaction->xid_state.is_explicit_XA() && + thd->transaction->xid_state.get_state_code() != XA_PREPARED) xa_trans_force_rollback(thd); thd->release_transactional_locks(); diff --git a/sql/slave.cc b/sql/slave.cc index 7db5a31d439..682561c9cb7 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -4269,10 +4269,10 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, the user might be surprised to see a claim that the slave is up to date long before those queued events are actually executed. */ - if (!rli->mi->using_parallel() && - !(ev->is_artificial_event() || ev->is_relay_log_event() || (ev->when == 0))) + if ((!rli->mi->using_parallel()) && event_can_update_last_master_timestamp(ev)) { rli->last_master_timestamp= ev->when + (time_t) ev->exec_time; + rli->sql_thread_caught_up= false; DBUG_ASSERT(rli->last_master_timestamp >= 0); } @@ -4324,6 +4324,17 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, if (rli->mi->using_parallel()) { + if (unlikely((rli->last_master_timestamp == 0 || + rli->sql_thread_caught_up) && + event_can_update_last_master_timestamp(ev))) + { + if (rli->last_master_timestamp < ev->when) + { + rli->last_master_timestamp= ev->when; + rli->sql_thread_caught_up= false; + } + } + int res= rli->parallel.do_event(serial_rgi, ev, event_size); /* In parallel replication, we need to update the relay log position @@ -4344,7 +4355,7 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, This is the case for pre-10.0 events without GTID, and for handling slave_skip_counter. */ - if (!(ev->is_artificial_event() || ev->is_relay_log_event() || (ev->when == 0))) + if (event_can_update_last_master_timestamp(ev)) { /* Ignore FD's timestamp as it does not reflect the slave execution @@ -4352,7 +4363,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, data modification event execution last long all this time Seconds_Behind_Master is zero. */ - if (ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) + if (ev->get_type_code() != FORMAT_DESCRIPTION_EVENT && + rli->last_master_timestamp < ev->when) rli->last_master_timestamp= ev->when + (time_t) ev->exec_time; DBUG_ASSERT(rli->last_master_timestamp >= 0); @@ -7406,7 +7418,6 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) if (hot_log) mysql_mutex_unlock(log_lock); - rli->sql_thread_caught_up= false; DBUG_RETURN(ev); } if (opt_reckless_slave) // For mysql-test @@ -7570,7 +7581,6 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) rli->relay_log.wait_for_update_relay_log(rli->sql_driver_thd); // re-acquire data lock since we released it earlier mysql_mutex_lock(&rli->data_lock); - rli->sql_thread_caught_up= false; continue; } /* @@ -7761,12 +7771,19 @@ event(errno: %d cur_log->error: %d)", { sql_print_information("Error reading relay log event: %s", "slave SQL thread was killed"); - DBUG_RETURN(0); + goto end; } err: if (errmsg) sql_print_error("Error reading relay log event: %s", errmsg); + +end: + /* + Set that we are not caught up so if there is a hang/problem on restart, + Seconds_Behind_Master will still grow. + */ + rli->sql_thread_caught_up= false; DBUG_RETURN(0); } #ifdef WITH_WSREP diff --git a/sql/slave.h b/sql/slave.h index 5ca6054a178..e2bd5cec1b9 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -49,6 +49,7 @@ #include "rpl_filter.h" #include "rpl_tblmap.h" #include "rpl_gtid.h" +#include "log_event.h" #define SLAVE_NET_TIMEOUT 60 @@ -293,6 +294,17 @@ extern char *report_host, *report_password; extern I_List<THD> threads; +/* + Check that a binlog event (read from the relay log) is valid to update + last_master_timestamp. That is, a valid event is one with a consistent + timestamp which originated from a primary server. +*/ +static inline bool event_can_update_last_master_timestamp(Log_event *ev) +{ + return ev && !(ev->is_artificial_event() || ev->is_relay_log_event() || + (ev->when == 0)); +} + #else #define close_active_mi() /* no-op */ #endif /* HAVE_REPLICATION */ diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 71039affd2b..465145cf25f 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -44,7 +44,8 @@ const LEX_CSTRING msg_optimize= { STRING_WITH_LEN("optimize") }; /* Prepare, run and cleanup for mysql_recreate_table() */ -static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) +static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list, + Recreate_info *recreate_info) { bool result_code; DBUG_ENTER("admin_recreate_table"); @@ -65,7 +66,7 @@ static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) DEBUG_SYNC(thd, "ha_admin_try_alter"); tmp_disable_binlog(thd); // binlogging is done by caller if wanted result_code= (thd->open_temporary_tables(table_list) || - mysql_recreate_table(thd, table_list, false)); + mysql_recreate_table(thd, table_list, recreate_info, false)); reenable_binlog(thd); /* mysql_recreate_table() can push OK or ERROR. @@ -560,6 +561,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, bool open_error= 0; bool collect_eis= FALSE; bool open_for_modify= org_open_for_modify; + Recreate_info recreate_info; storage_engine_name[0]= 0; // Marker that's not used @@ -829,7 +831,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, { /* We use extra_open_options to be able to open crashed tables */ thd->open_options|= extra_open_options; - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info) ? + HA_ADMIN_FAILED : HA_ADMIN_OK; thd->open_options&= ~extra_open_options; goto send_result; } @@ -1012,12 +1015,31 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, repair was not implemented and we need to upgrade the table to a new version so we recreate the table with ALTER TABLE */ - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info); } send_result: lex->cleanup_after_one_table_open(); thd->clear_error(); // these errors shouldn't get client + + if (recreate_info.records_duplicate()) + { + protocol->prepare_for_resend(); + protocol->store(&table_name, system_charset_info); + protocol->store(operator_name, system_charset_info); + protocol->store(warning_level_names[Sql_condition::WARN_LEVEL_WARN].str, + warning_level_names[Sql_condition::WARN_LEVEL_WARN].length, + system_charset_info); + char buf[80]; + size_t length= my_snprintf(buf, sizeof(buf), + "Number of rows changed from %u to %u", + (uint) recreate_info.records_processed(), + (uint) recreate_info.records_copied()); + protocol->store(buf, length, system_charset_info); + if (protocol->write()) + goto err; + } + { Diagnostics_area::Sql_condition_iterator it= thd->get_stmt_da()->sql_conditions(); @@ -1128,7 +1150,7 @@ send_result_message: *save_next_global= table->next_global; table->next_local= table->next_global= 0; - result_code= admin_recreate_table(thd, table); + result_code= admin_recreate_table(thd, table, &recreate_info); trans_commit_stmt(thd); trans_commit(thd); close_thread_tables(thd); @@ -1340,6 +1362,8 @@ send_result_message: goto err; DEBUG_SYNC(thd, "admin_command_kill_after_modify"); } + thd->resume_subsequent_commits(suspended_wfc); + DBUG_EXECUTE_IF("inject_analyze_table_sleep", my_sleep(500000);); if (is_table_modified && is_cmd_replicated && (!opt_readonly || thd->slave_thread) && !thd->lex->no_write_to_binlog) { @@ -1349,10 +1373,8 @@ send_result_message: if (res) goto err; } - my_eof(thd); - thd->resume_subsequent_commits(suspended_wfc); - DBUG_EXECUTE_IF("inject_analyze_table_sleep", my_sleep(500000);); + DBUG_RETURN(FALSE); err: @@ -1501,6 +1523,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) LEX *m_lex= thd->lex; TABLE_LIST *first_table= m_lex->first_select_lex()->table_list.first; bool res= TRUE; + Recreate_info recreate_info; DBUG_ENTER("Sql_cmd_optimize_table::execute"); if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table, @@ -1509,7 +1532,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd) WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); res= (specialflag & SPECIAL_NO_NEW_FUNC) ? - mysql_recreate_table(thd, first_table, true) : + mysql_recreate_table(thd, first_table, &recreate_info, true) : mysql_admin_table(thd, first_table, &m_lex->check_opt, &msg_optimize, TL_WRITE, 1, 0, 0, 0, &handler::ha_optimize, 0, true); diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index ffcc7679273..5959203b36d 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -554,9 +554,11 @@ bool Sql_cmd_alter_table::execute(THD *thd) thd->work_part_info= 0; #endif + Recreate_info recreate_info; result= mysql_alter_table(thd, &select_lex->db, &lex->name, &create_info, first_table, + &recreate_info, &alter_info, select_lex->order_list.elements, select_lex->order_list.first, diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 17ce7b3c048..7ef77a4d331 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -8335,6 +8335,20 @@ bool THD::timestamp_to_TIME(MYSQL_TIME *ltime, my_time_t ts, return 0; } + +void THD::my_ok_with_recreate_info(const Recreate_info &info, + ulong warn_count) +{ + char buf[80]; + my_snprintf(buf, sizeof(buf), + ER_THD(this, ER_INSERT_INFO), + (ulong) info.records_processed(), + (ulong) info.records_duplicate(), + warn_count); + my_ok(this, info.records_processed(), 0L, buf); +} + + THD_list_iterator *THD_list_iterator::iterator() { return &server_threads; diff --git a/sql/sql_class.h b/sql/sql_class.h index ef299e4eb12..00476ffd1b5 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -247,6 +247,29 @@ public: }; +class Recreate_info +{ + ha_rows m_records_copied; + ha_rows m_records_duplicate; +public: + Recreate_info() + :m_records_copied(0), + m_records_duplicate(0) + { } + Recreate_info(ha_rows records_copied, + ha_rows records_duplicate) + :m_records_copied(records_copied), + m_records_duplicate(records_duplicate) + { } + ha_rows records_copied() const { return m_records_copied; } + ha_rows records_duplicate() const { return m_records_duplicate; } + ha_rows records_processed() const + { + return m_records_copied + m_records_duplicate; + } +}; + + #define TC_HEURISTIC_RECOVER_COMMIT 1 #define TC_HEURISTIC_RECOVER_ROLLBACK 2 extern ulong tc_heuristic_recover; @@ -4364,6 +4387,8 @@ public: inline bool vio_ok() const { return TRUE; } inline bool is_connected() { return TRUE; } #endif + + void my_ok_with_recreate_info(const Recreate_info &info, ulong warn_count); /** Mark the current error as fatal. Warning: this does not set any error, it sets a property of the error, so must be @@ -6277,6 +6302,12 @@ public: uint sum_func_count; uint hidden_field_count; uint group_parts,group_length,group_null_parts; + + /* + If we're doing a GROUP BY operation, shows which one is used: + true TemporaryTableWithPartialSums algorithm (see end_update()). + false OrderedGroupBy algorithm (see end_write_group()). + */ uint quick_group; /** Enabled when we have atleast one outer_sum_func. Needed when used diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index a526bfee2d2..ad385128d8d 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -102,49 +102,6 @@ bool LEX::check_dependencies_in_with_clauses() /** @brief - Resolve references to CTE in specification of hanging CTE - - @details - A CTE to which there are no references in the query is called hanging CTE. - Although such CTE is not used for execution its specification must be - subject to context analysis. All errors concerning references to - non-existing tables or fields occurred in the specification must be - reported as well as all other errors caught at the prepare stage. - The specification of a hanging CTE might contain references to other - CTE outside of the specification and within it if the specification - contains a with clause. This function resolves all such references for - all hanging CTEs encountered in the processed query. - - @retval - false on success - true on failure -*/ - -bool -LEX::resolve_references_to_cte_in_hanging_cte() -{ - for (With_clause *with_clause= with_clauses_list; - with_clause; with_clause= with_clause->next_with_clause) - { - for (With_element *with_elem= with_clause->with_list.first; - with_elem; with_elem= with_elem->next) - { - if (!with_elem->is_referenced()) - { - TABLE_LIST *first_tbl= - with_elem->spec->first_select()->table_list.first; - TABLE_LIST **with_elem_end_pos= with_elem->head->tables_pos.end_pos; - if (first_tbl && resolve_references_to_cte(first_tbl, with_elem_end_pos)) - return true; - } - } - } - return false; -} - - -/** - @brief Resolve table references to CTE from a sub-chain of table references @param tables Points to the beginning of the sub-chain @@ -289,8 +246,6 @@ LEX::check_cte_dependencies_and_resolve_references() return false; if (resolve_references_to_cte(query_tables, query_tables_last)) return true; - if (resolve_references_to_cte_in_hanging_cte()) - return true; return false; } @@ -489,47 +444,33 @@ With_element *find_table_def_in_with_clauses(TABLE_LIST *tbl, st_unit_ctxt_elem *ctxt) { With_element *found= 0; + st_select_lex_unit *top_unit= 0; for (st_unit_ctxt_elem *unit_ctxt_elem= ctxt; unit_ctxt_elem; unit_ctxt_elem= unit_ctxt_elem->prev) { st_select_lex_unit *unit= unit_ctxt_elem->unit; With_clause *with_clause= unit->with_clause; - /* - First look for the table definition in the with clause attached to 'unit' - if there is any such clause. - */ if (with_clause) { - found= with_clause->find_table_def(tbl, NULL); + /* + If the reference to tbl that has to be resolved belongs to + the FROM clause of a descendant of top_unit->with_element + and this with element belongs to with_clause then this + element must be used as the barrier for the search in the + the list of CTEs from with_clause unless the clause contains + RECURSIVE. + */ + With_element *barrier= 0; + if (top_unit && !with_clause->with_recursive && + top_unit->with_element && + top_unit->with_element->get_owner() == with_clause) + barrier= top_unit->with_element; + found= with_clause->find_table_def(tbl, barrier); if (found) break; } - /* - If 'unit' is the unit that defines a with element then reset 'unit' - to the unit whose attached with clause contains this with element. - */ - With_element *with_elem= unit->with_element; - if (with_elem) - { - if (!(unit_ctxt_elem= unit_ctxt_elem->prev)) - break; - unit= unit_ctxt_elem->unit; - } - with_clause= unit->with_clause; - /* - Now look for the table definition in this with clause. If the with clause - contains RECURSIVE the search is performed through all CTE definitions in - clause, otherwise up to the definition of 'with_elem' unless it is NULL. - */ - if (with_clause) - { - found= with_clause->find_table_def(tbl, - with_clause->with_recursive ? - NULL : with_elem); - if (found) - break; - } + top_unit= unit; } return found; } diff --git a/sql/sql_cte.h b/sql/sql_cte.h index e0fbd79803b..6a1f67d3258 100644 --- a/sql/sql_cte.h +++ b/sql/sql_cte.h @@ -326,8 +326,6 @@ public: friend bool LEX::resolve_references_to_cte(TABLE_LIST *tables, TABLE_LIST **tables_last); - friend - bool LEX::resolve_references_to_cte_in_hanging_cte(); }; const uint max_number_of_elements_in_with_clause= sizeof(table_map)*8; @@ -441,9 +439,6 @@ public: friend bool LEX::check_dependencies_in_with_clauses(); - - friend - bool LEX::resolve_references_to_cte_in_hanging_cte(); }; inline diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 5d713d2176e..6e042d25805 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4038,7 +4038,8 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) lex->current_select->join->select_options|= OPTION_BUFFER_RESULT; } else if (!(lex->current_select->options & OPTION_BUFFER_RESULT) && - thd->locked_tables_mode <= LTM_LOCK_TABLES) + thd->locked_tables_mode <= LTM_LOCK_TABLES && + !table->s->long_unique_table) { /* We must not yet prepare the result table if it is the same as one of the diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 5e85d1d0902..ab16a9d0b8b 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -1291,8 +1291,6 @@ void LEX::start(THD *thd_arg) stmt_var_list.empty(); proc_list.elements=0; - save_group_list.empty(); - save_order_list.empty(); win_ref= NULL; win_frame= NULL; frame_top_bound= NULL; diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 95ed308103d..9d419ec9a6a 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -1204,12 +1204,14 @@ public: group_list_ptrs, and re-establish the original list before each execution. */ SQL_I_List<ORDER> group_list; + SQL_I_List<ORDER> save_group_list; Group_list_ptrs *group_list_ptrs; List<Item> item_list; /* list of fields & expressions */ List<Item> pre_fix; /* above list before fix_fields */ List<Item> fix_after_optimize; SQL_I_List<ORDER> order_list; /* ORDER clause */ + SQL_I_List<ORDER> save_order_list; SQL_I_List<ORDER> gorder_list; Lex_select_limit limit_params; /* LIMIT clause parameters */ @@ -3559,8 +3561,6 @@ public: } - SQL_I_List<ORDER> save_group_list; - SQL_I_List<ORDER> save_order_list; LEX_CSTRING *win_ref; Window_frame *win_frame; Window_frame_bound *frame_top_bound; @@ -4823,8 +4823,8 @@ public: const LEX_CSTRING *constraint_name, Table_ident *ref_table_name, DDL_options ddl_options); + bool check_dependencies_in_with_clauses(); - bool resolve_references_to_cte_in_hanging_cte(); bool check_cte_dependencies_and_resolve_references(); bool resolve_references_to_cte(TABLE_LIST *tables, TABLE_LIST **tables_last); diff --git a/sql/sql_list.h b/sql/sql_list.h index edecb0f0be4..5a57c86ef9d 100644 --- a/sql/sql_list.h +++ b/sql/sql_list.h @@ -54,7 +54,7 @@ public: { elements= tmp.elements; first= tmp.first; - next= tmp.next; + next= elements ? tmp.next : &first;; return *this; } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 579e4cd27f7..ee894fe55de 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4210,8 +4210,10 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) WSREP_TO_ISOLATION_BEGIN(first_table->db.str, first_table->table_name.str, NULL); + Recreate_info recreate_info; res= mysql_alter_table(thd, &first_table->db, &first_table->table_name, - &create_info, first_table, &alter_info, + &create_info, first_table, + &recreate_info, &alter_info, 0, (ORDER*) 0, 0, lex->if_exists()); break; } @@ -8806,8 +8808,8 @@ TABLE_LIST *st_select_lex::convert_right_join() void st_select_lex::prepare_add_window_spec(THD *thd) { LEX *lex= thd->lex; - lex->save_group_list= group_list; - lex->save_order_list= order_list; + save_group_list= group_list; + save_order_list= order_list; lex->win_ref= NULL; lex->win_frame= NULL; lex->frame_top_bound= NULL; @@ -8834,8 +8836,8 @@ bool st_select_lex::add_window_def(THD *thd, win_part_list_ptr, win_order_list_ptr, win_frame); - group_list= thd->lex->save_group_list; - order_list= thd->lex->save_order_list; + group_list= save_group_list; + order_list= save_order_list; if (parsing_place != SELECT_LIST) { fields_in_window_functions+= win_part_list_ptr->elements + @@ -8861,8 +8863,8 @@ bool st_select_lex::add_window_spec(THD *thd, win_part_list_ptr, win_order_list_ptr, win_frame); - group_list= thd->lex->save_group_list; - order_list= thd->lex->save_order_list; + group_list= save_group_list; + order_list= save_order_list; if (parsing_place != SELECT_LIST) { fields_in_window_functions+= win_part_list_ptr->elements + diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index f8b4948a3ac..094ff52a4ea 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -342,7 +342,7 @@ static bool register_builtin(struct st_maria_plugin *, struct st_plugin_int *, struct st_plugin_int **); static void unlock_variables(THD *thd, struct system_variables *vars); static void cleanup_variables(struct system_variables *vars); -static void plugin_vars_free_values(sys_var *vars); +static void plugin_vars_free_values(st_mysql_sys_var **vars); static void restore_ptr_backup(uint n, st_ptr_backup *backup); static void intern_plugin_unlock(LEX *lex, plugin_ref plugin); static void reap_plugins(void); @@ -1293,7 +1293,7 @@ static void plugin_del(struct st_plugin_int *plugin, uint del_mask) if (!(plugin->state & del_mask)) DBUG_VOID_RETURN; /* Free allocated strings before deleting the plugin. */ - plugin_vars_free_values(plugin->system_vars); + plugin_vars_free_values(plugin->plugin->system_vars); restore_ptr_backup(plugin->nbackups, plugin->ptr_backup); if (plugin->plugin_dl) { @@ -2948,6 +2948,7 @@ sys_var *find_sys_var(THD *thd, const char *str, size_t length, /* called by register_var, construct_options and test_plugin_options. Returns the 'bookmark' for the named variable. + returns null for non thd-local variables. LOCK_system_variables_hash should be at least read locked */ static st_bookmark *find_bookmark(const char *plugin, const char *name, @@ -3004,7 +3005,6 @@ static size_t var_storage_size(int flags) /* returns a bookmark for thd-local variables, creating if neccessary. - returns null for non thd-local variables. Requires that a write lock is obtained on LOCK_system_variables_hash */ static st_bookmark *register_var(const char *plugin, const char *name, @@ -3358,27 +3358,35 @@ void plugin_thdvar_cleanup(THD *thd) variables are no longer accessible and the value space is lost. Note that only string values with PLUGIN_VAR_MEMALLOC are allocated and must be freed. - - @param[in] vars Chain of system variables of a plugin */ -static void plugin_vars_free_values(sys_var *vars) +static void plugin_vars_free_values(st_mysql_sys_var **vars) { DBUG_ENTER("plugin_vars_free_values"); - for (sys_var *var= vars; var; var= var->next) + if (!vars) + DBUG_VOID_RETURN; + + while(st_mysql_sys_var *var= *vars++) { - sys_var_pluginvar *piv= var->cast_pluginvar(); - if (piv && - ((piv->plugin_var->flags & PLUGIN_VAR_TYPEMASK) == PLUGIN_VAR_STR) && - (piv->plugin_var->flags & PLUGIN_VAR_MEMALLOC)) + if ((var->flags & PLUGIN_VAR_TYPEMASK) == PLUGIN_VAR_STR && + var->flags & PLUGIN_VAR_MEMALLOC) { - /* Free the string from global_system_variables. */ - char **valptr= (char**) piv->real_value_ptr(NULL, OPT_GLOBAL); + char **val; + if (var->flags & PLUGIN_VAR_THDLOCAL) + { + st_bookmark *v= find_bookmark(0, var->name, var->flags); + if (!v) + continue; + val= (char**)(global_system_variables.dynamic_variables_ptr + v->offset); + } + else + val= *(char***) (var + 1); + DBUG_PRINT("plugin", ("freeing value for: '%s' addr: %p", - var->name.str, valptr)); - my_free(*valptr); - *valptr= NULL; + var->name, val)); + my_free(*val); + *val= NULL; } } DBUG_VOID_RETURN; @@ -4038,7 +4046,7 @@ static my_option *construct_help_options(MEM_ROOT *mem_root, bzero(opts, sizeof(my_option) * count); /** - some plugin variables (those that don't have PLUGIN_VAR_NOSYSVAR flag) + some plugin variables have their names prefixed with the plugin name. Restore the names here to get the correct (not double-prefixed) help text. We won't need @@sysvars anymore and don't care about their proper names. @@ -4150,9 +4158,6 @@ static int test_plugin_options(MEM_ROOT *tmp_root, struct st_plugin_int *tmp, char *varname; sys_var *v; - if (o->flags & PLUGIN_VAR_NOSYSVAR) - continue; - tmp_backup[tmp->nbackups++].save(&o->name); if ((var= find_bookmark(tmp->name.str, o->name, o->flags))) { @@ -4168,6 +4173,12 @@ static int test_plugin_options(MEM_ROOT *tmp_root, struct st_plugin_int *tmp, my_casedn_str(&my_charset_latin1, varname); convert_dash_to_underscore(varname, len-1); } + if (o->flags & PLUGIN_VAR_NOSYSVAR) + { + o->name= varname; + continue; + } + const char *s= o->flags & PLUGIN_VAR_DEPRECATED ? "" : NULL; v= new (mem_root) sys_var_pluginvar(&chain, varname, tmp, o, s); v->test_load= (var ? &var->loaded : &static_unload); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index a88b8a54a08..f9064e61c2c 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3801,15 +3801,26 @@ bool JOIN::make_aggr_tables_info() /* If we have different sort & group then we must sort the data by group - and copy it to another tmp table + and copy it to another tmp table. + This code is also used if we are using distinct something we haven't been able to store in the temporary table yet like SEC_TO_TIME(SUM(...)). + + 3. Also, this is used when + - the query has Window functions, + - the GROUP BY operation is done with OrderedGroupBy algorithm. + In this case, the first temptable will contain pre-GROUP-BY data. Force + the creation of the second temporary table. Post-GROUP-BY dataset will be + written there, and then Window Function processing code will be able to + process it. */ if ((group_list && (!test_if_subpart(group_list, order) || select_distinct)) || - (select_distinct && tmp_table_param.using_outer_summary_function)) - { /* Must copy to another table */ + (select_distinct && tmp_table_param.using_outer_summary_function) || + (group_list && !tmp_table_param.quick_group && // (3) + select_lex->have_window_funcs())) // (3) + { /* Must copy to another table */ DBUG_PRINT("info",("Creating group table")); calc_group_buffer(this, group_list); @@ -24138,11 +24149,17 @@ end_send(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) /* @brief - Perform a GROUP BY operation over a stream of rows ordered by their group. - The result is sent into join->result. + Perform OrderedGroupBy operation and write the output into join->result. @detail - Also applies HAVING, etc. + The input stream is ordered by the GROUP BY expression, so groups come + one after another. We only need to accumulate the aggregate value, when + a GROUP BY group ends, check the HAVING and send the group. + + Note that the output comes in the GROUP BY order, which is required by + the MySQL's GROUP BY semantics. No further sorting is needed. + + @seealso end_write_group() also implements SortAndGroup */ enum_nested_loop_state @@ -24339,13 +24356,26 @@ end: /* @brief - Perform a GROUP BY operation over rows coming in arbitrary order. - - This is done by looking up the group in a temp.table and updating group - values. + Perform GROUP BY operation over rows coming in arbitrary order: use + TemporaryTableWithPartialSums algorithm. + + @detail + The TemporaryTableWithPartialSums algorithm is: + + CREATE TEMPORARY TABLE tmp ( + group_by_columns PRIMARY KEY, + partial_sum + ); + + for each row R in join output { + INSERT INTO tmp (R.group_by_columns, R.sum_value) + ON DUPLICATE KEY UPDATE partial_sum=partial_sum + R.sum_value; + } @detail Also applies HAVING, etc. + + @seealso end_unique_update() */ static enum_nested_loop_state @@ -24498,13 +24528,15 @@ end_unique_update(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), /* @brief - Perform a GROUP BY operation over a stream of rows ordered by their group. - Write the result into a temporary table. + Perform OrderedGroupBy operation and write the output into the temporary + table (join_tab->table). @detail - Also applies HAVING, etc. + The input stream is ordered by the GROUP BY expression, so groups come + one after another. We only need to accumulate the aggregate value, when + a GROUP BY group ends, check the HAVING and write the group. - The rows are written into temptable so e.g. filesort can read them. + @seealso end_send_group() also implements OrderedGroupBy */ enum_nested_loop_state diff --git a/sql/sql_select.h b/sql/sql_select.h index 4b64243e1c9..16c532f3634 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1559,12 +1559,30 @@ public: (set in make_join_statistics()) */ bool impossible_where; - List<Item> all_fields; ///< to store all fields that used in query + + /* + All fields used in the query processing. + + Initially this is a list of fields from the query's SQL text. + + Then, ORDER/GROUP BY and Window Function code add columns that need to + be saved to be available in the post-group-by context. These extra columns + are added to the front, because this->all_fields points to the suffix of + this list. + */ + List<Item> all_fields; ///Above list changed to use temporary table List<Item> tmp_all_fields1, tmp_all_fields2, tmp_all_fields3; ///Part, shared with list above, emulate following list List<Item> tmp_fields_list1, tmp_fields_list2, tmp_fields_list3; - List<Item> &fields_list; ///< hold field list passed to mysql_select + + /* + The original field list as it was passed to mysql_select(). This refers + to select_lex->item_list. + CAUTION: this list is a suffix of this->all_fields list, that is, it shares + elements with that list! + */ + List<Item> &fields_list; List<Item> procedure_fields_list; int error; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index db9a67da094..0cfecddccad 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -9823,6 +9823,7 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, Table_specification_st *create_info, TABLE_LIST *table_list, + Recreate_info *recreate_info, Alter_info *alter_info, uint order_num, ORDER *order, bool ignore, bool if_exists) @@ -11333,11 +11334,10 @@ end_temporary: thd->variables.option_bits&= ~OPTION_BIN_COMMIT_OFF; - my_snprintf(alter_ctx.tmp_buff, sizeof(alter_ctx.tmp_buff), - ER_THD(thd, ER_INSERT_INFO), - (ulong) (copied + deleted), (ulong) deleted, - (ulong) thd->get_stmt_da()->current_statement_warn_count()); - my_ok(thd, copied + deleted, 0L, alter_ctx.tmp_buff); + *recreate_info= Recreate_info(copied, deleted); + thd->my_ok_with_recreate_info(*recreate_info, + (ulong) thd->get_stmt_da()-> + current_statement_warn_count()); DEBUG_SYNC(thd, "alter_table_inplace_trans_commit"); DBUG_RETURN(false); @@ -11852,7 +11852,8 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, Like mysql_alter_table(). */ -bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) +bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, + Recreate_info *recreate_info, bool table_copy) { Table_specification_st create_info; Alter_info alter_info; @@ -11877,8 +11878,11 @@ bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) Alter_info::ALTER_TABLE_ALGORITHM_COPY); bool res= mysql_alter_table(thd, &null_clex_str, &null_clex_str, &create_info, - table_list, &alter_info, 0, - (ORDER *) 0, 0, 0); + table_list, recreate_info, &alter_info, 0, + (ORDER *) 0, + // Ignore duplicate records on REPAIR + thd->lex->sql_command == SQLCOM_REPAIR, + 0); table_list->next_global= next_table; DBUG_RETURN(res); } diff --git a/sql/sql_table.h b/sql/sql_table.h index c9e4d969482..ccde7d87120 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -163,6 +163,7 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, Table_specification_st *create_info, TABLE_LIST *table_list, + class Recreate_info *recreate_info, Alter_info *alter_info, uint order_num, ORDER *order, bool ignore, bool if_exists); @@ -170,7 +171,8 @@ bool mysql_compare_tables(TABLE *table, Alter_info *alter_info, HA_CREATE_INFO *create_info, bool *metadata_equal); -bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy); +bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, + class Recreate_info *recreate_info, bool table_copy); bool mysql_rename_table(handlerton *base, const LEX_CSTRING *old_db, const LEX_CSTRING *old_name, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, LEX_CUSTRING *id, diff --git a/sql/sql_type.h b/sql/sql_type.h index dcdf6438fd9..07dabb9ecb4 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -124,6 +124,32 @@ enum scalar_comparison_op }; +class Hasher +{ + ulong m_nr1; + ulong m_nr2; +public: + Hasher(): m_nr1(1), m_nr2(4) + { } + void add_null() + { + m_nr1^= (m_nr1 << 1) | 1; + } + void add(CHARSET_INFO *cs, const uchar *str, size_t length) + { + cs->coll->hash_sort(cs, str, length, &m_nr1, &m_nr2); + } + void add(CHARSET_INFO *cs, const char *str, size_t length) + { + add(cs, (const uchar *) str, length); + } + uint32 finalize() const + { + return (uint32) m_nr1; + } +}; + + enum partition_value_print_mode_t { PARTITION_VALUE_PRINT_MODE_SHOW= 0, diff --git a/sql/sql_type_fixedbin.h b/sql/sql_type_fixedbin.h index 077e4039643..223bf2cf398 100644 --- a/sql/sql_type_fixedbin.h +++ b/sql/sql_type_fixedbin.h @@ -1428,12 +1428,9 @@ public: item->type_handler() == type_handler()); return true; } - void hash(ulong *nr, ulong *nr2) override + void hash_not_null(Hasher *hasher) override { - if (is_null()) - *nr^= (*nr << 1) | 1; - else - FbtImpl::hash_record(ptr, nr, nr2); + FbtImpl::hash_record(ptr, hasher); } SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *prm, KEY_PART *key_part, const Item_bool_func *cond, diff --git a/sql/sql_type_fixedbin_storage.h b/sql/sql_type_fixedbin_storage.h index 6e18335bd4c..e3a46698463 100644 --- a/sql/sql_type_fixedbin_storage.h +++ b/sql/sql_type_fixedbin_storage.h @@ -122,9 +122,9 @@ public: Hash the in-record representation Used in Field::hash(). */ - static void hash_record(const uchar *ptr, ulong *nr, ulong *nr2) + static void hash_record(uchar *ptr, Hasher *hasher) { - my_charset_bin.hash_sort(ptr, binary_length(), nr, nr2); + hasher->add(&my_charset_bin, ptr, binary_length()); } static bool only_zero_bytes(const char *ptr, size_t length) diff --git a/sql/table.cc b/sql/table.cc index 3fffb974f0c..e0189401906 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1101,6 +1101,18 @@ static void mysql57_calculate_null_position(TABLE_SHARE *share, } } + +Item_func_hash *TABLE_SHARE::make_long_hash_func(THD *thd, + MEM_ROOT *mem_root, + List<Item> *field_list) + const +{ + if (old_long_hash_function()) + return new (mem_root) Item_func_hash_mariadb_100403(thd, *field_list); + return new (mem_root) Item_func_hash(thd, *field_list); +} + + /** Parse TABLE_SHARE::vcol_defs unpack_vcol_info_from_frm @@ -1306,7 +1318,10 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table, list_item= new (mem_root) Item_field(thd, keypart->field); field_list->push_back(list_item, mem_root); } - Item_func_hash *hash_item= new(mem_root)Item_func_hash(thd, *field_list); + + Item_func_hash *hash_item= table->s->make_long_hash_func(thd, mem_root, + field_list); + Virtual_column_info *v= new (mem_root) Virtual_column_info(); field->vcol_info= v; field->vcol_info->expr= hash_item; diff --git a/sql/table.h b/sql/table.h index add1871e899..b74c6f2ab8d 100644 --- a/sql/table.h +++ b/sql/table.h @@ -55,6 +55,7 @@ class Item; /* Needed by ORDER */ typedef Item (*Item_ptr); class Item_subselect; class Item_field; +class Item_func_hash; class GRANT_TABLE; class st_select_lex_unit; class st_select_lex; @@ -1197,6 +1198,22 @@ struct TABLE_SHARE void set_overlapped_keys(); void set_ignored_indexes(); key_map usable_indexes(THD *thd); + + bool old_long_hash_function() const + { + return mysql_version < 100428 || + (mysql_version >= 100500 && mysql_version < 100519) || + (mysql_version >= 100600 && mysql_version < 100612) || + (mysql_version >= 100700 && mysql_version < 100708) || + (mysql_version >= 100800 && mysql_version < 100807) || + (mysql_version >= 100900 && mysql_version < 100905) || + (mysql_version >= 101000 && mysql_version < 101003) || + (mysql_version >= 101100 && mysql_version < 101102); + } + Item_func_hash *make_long_hash_func(THD *thd, + MEM_ROOT *mem_root, + List<Item> *field_list) const; + void update_optimizer_costs(handlerton *hton); }; diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 2746cf732ce..2af57259be8 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1019,13 +1019,19 @@ void wsrep_init_startup (bool sst_first) With mysqldump SST (!sst_first) wait until the server reaches joiner state and procedd to accepting connections. */ + int err= 0; if (sst_first) { - server_state.wait_until_state(Wsrep_server_state::s_initializing); + err= server_state.wait_until_state(Wsrep_server_state::s_initializing); } else { - server_state.wait_until_state(Wsrep_server_state::s_joiner); + err= server_state.wait_until_state(Wsrep_server_state::s_joiner); + } + if (err) + { + WSREP_ERROR("Wsrep startup was interrupted"); + unireg_abort(1); } } @@ -1132,7 +1138,11 @@ void wsrep_stop_replication(THD *thd) { WSREP_DEBUG("Disconnect provider"); Wsrep_server_state::instance().disconnect(); - Wsrep_server_state::instance().wait_until_state(Wsrep_server_state::s_disconnected); + if (Wsrep_server_state::instance().wait_until_state( + Wsrep_server_state::s_disconnected)) + { + WSREP_WARN("Wsrep interrupted while waiting for disconnected state"); + } } /* my connection, should not terminate with wsrep_close_client_connection(), @@ -1154,7 +1164,11 @@ void wsrep_shutdown_replication() { WSREP_DEBUG("Disconnect provider"); Wsrep_server_state::instance().disconnect(); - Wsrep_server_state::instance().wait_until_state(Wsrep_server_state::s_disconnected); + if (Wsrep_server_state::instance().wait_until_state( + Wsrep_server_state::s_disconnected)) + { + WSREP_WARN("Wsrep interrupted while waiting for disconnected state"); + } } wsrep_close_client_connections(TRUE); @@ -3231,7 +3245,9 @@ static my_bool have_client_connections(THD *thd, void*) { DBUG_PRINT("quit",("Informing thread %lld that it's time to die", (longlong) thd->thread_id)); - if (is_client_connection(thd) && thd->killed == KILL_CONNECTION) + if (is_client_connection(thd) && + (thd->killed == KILL_CONNECTION || + thd->killed == KILL_CONNECTION_HARD)) { (void)abort_replicated(thd); return 1; @@ -3241,7 +3257,7 @@ static my_bool have_client_connections(THD *thd, void*) static void wsrep_close_thread(THD *thd) { - thd->set_killed(KILL_CONNECTION); + thd->set_killed(KILL_CONNECTION_HARD); MYSQL_CALLBACK(thread_scheduler, post_kill_notification, (thd)); mysql_mutex_lock(&thd->LOCK_thd_kill); thd->abort_current_cond_wait(true); @@ -3275,13 +3291,13 @@ static my_bool kill_all_threads(THD *thd, THD *caller_thd) if (is_client_connection(thd) && thd != caller_thd) { if (is_replaying_connection(thd)) - thd->set_killed(KILL_CONNECTION); + thd->set_killed(KILL_CONNECTION_HARD); else if (!abort_replicated(thd)) { /* replicated transactions must be skipped */ WSREP_DEBUG("closing connection %lld", (longlong) thd->thread_id); /* instead of wsrep_close_thread() we do now soft kill by THD::awake */ - thd->awake(KILL_CONNECTION); + thd->awake(KILL_CONNECTION_HARD); } } return 0; diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index c09a9392cb6..118bedecff3 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -336,9 +336,14 @@ static bool wsrep_sst_complete (THD* thd, if ((state == Wsrep_server_state::s_joiner || state == Wsrep_server_state::s_initialized)) { - Wsrep_server_state::instance().sst_received(client_service, - rcode); - WSREP_INFO("SST succeeded for position %s", start_pos_buf); + if (Wsrep_server_state::instance().sst_received(client_service, rcode)) + { + failed= true; + } + else + { + WSREP_INFO("SST succeeded for position %s", start_pos_buf); + } } else { diff --git a/sql/xa.cc b/sql/xa.cc index 457aacfeb30..0e421ac62f5 100644 --- a/sql/xa.cc +++ b/sql/xa.cc @@ -600,6 +600,7 @@ bool trans_xa_commit(THD *thd) if (auto xs= xid_cache_search(thd, thd->lex->xid)) { + bool xid_deleted= false; res= xa_trans_rolled_back(xs); /* Acquire metadata lock which will ensure that COMMIT is blocked @@ -610,7 +611,7 @@ bool trans_xa_commit(THD *thd) */ MDL_request mdl_request; MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT, - MDL_STATEMENT); + MDL_EXPLICIT); if (thd->mdl_context.acquire_lock(&mdl_request, thd->variables.lock_wait_timeout)) { @@ -621,25 +622,37 @@ bool trans_xa_commit(THD *thd) */ DBUG_ASSERT(thd->is_error()); - xs->acquired_to_recovered(); - DBUG_RETURN(true); + res= true; + goto _end_external_xid; } - DBUG_ASSERT(!xid_state.xid_cache_element); - - if (thd->wait_for_prior_commit()) + else { - DBUG_ASSERT(thd->is_error()); - - xs->acquired_to_recovered(); - DBUG_RETURN(true); + thd->backup_commit_lock= &mdl_request; } + DBUG_ASSERT(!xid_state.xid_cache_element); xid_state.xid_cache_element= xs; ha_commit_or_rollback_by_xid(thd->lex->xid, !res); - xid_state.xid_cache_element= 0; + if (!res && thd->is_error()) + { + // hton completion error retains xs/xid in the cache, + // unless there had been already one as reflected by `res`. + res= true; + goto _end_external_xid; + } + xid_cache_delete(thd, xs); + xid_deleted= true; + _end_external_xid: + xid_state.xid_cache_element= 0; res= res || thd->is_error(); - xid_cache_delete(thd, xs); + if (!xid_deleted) + xs->acquired_to_recovered(); + if (mdl_request.ticket) + { + thd->mdl_context.release_lock(mdl_request.ticket); + thd->backup_commit_lock= 0; + } } else my_error(ER_XAER_NOTA, MYF(0)); @@ -761,9 +774,11 @@ bool trans_xa_rollback(THD *thd) if (auto xs= xid_cache_search(thd, thd->lex->xid)) { + bool res; + bool xid_deleted= false; MDL_request mdl_request; MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT, - MDL_STATEMENT); + MDL_EXPLICIT); if (thd->mdl_context.acquire_lock(&mdl_request, thd->variables.lock_wait_timeout)) { @@ -774,23 +789,33 @@ bool trans_xa_rollback(THD *thd) */ DBUG_ASSERT(thd->is_error()); - xs->acquired_to_recovered(); - DBUG_RETURN(true); + goto _end_external_xid; } - xa_trans_rolled_back(xs); - DBUG_ASSERT(!xid_state.xid_cache_element); - - if (thd->wait_for_prior_commit()) + else { - DBUG_ASSERT(thd->is_error()); - xs->acquired_to_recovered(); - DBUG_RETURN(true); + thd->backup_commit_lock= &mdl_request; } + res= xa_trans_rolled_back(xs); + DBUG_ASSERT(!xid_state.xid_cache_element); xid_state.xid_cache_element= xs; ha_commit_or_rollback_by_xid(thd->lex->xid, 0); - xid_state.xid_cache_element= 0; + if (!res && thd->is_error()) + { + goto _end_external_xid; + } xid_cache_delete(thd, xs); + xid_deleted= true; + + _end_external_xid: + xid_state.xid_cache_element= 0; + if (!xid_deleted) + xs->acquired_to_recovered(); + if (mdl_request.ticket) + { + thd->mdl_context.release_lock(mdl_request.ticket); + thd->backup_commit_lock= 0; + } } else my_error(ER_XAER_NOTA, MYF(0)); |