diff options
Diffstat (limited to 'sql')
143 files changed, 9671 insertions, 2848 deletions
diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 711b7d9101a..dfbe568ac0b 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -52,6 +52,7 @@ ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql +${LIBFMT_INCLUDE_DIR} ${PCRE_INCLUDES} ${ZLIB_INCLUDE_DIR} ${SSL_INCLUDE_DIRS} @@ -97,9 +98,9 @@ SET (SQL_SOURCE filesort.cc gstream.cc signal_handler.cc handler.cc - hostname.cc init.cc item.cc item_buff.cc item_cmpfunc.cc - item_create.cc item_func.cc item_geofunc.cc item_row.cc - item_strfunc.cc item_subselect.cc item_sum.cc item_timefunc.cc + hostname.cc init.cc item.cc item_buff.cc item_cmpfunc.cc + item_create.cc item_func.cc item_geofunc.cc item_row.cc + item_strfunc.cc item_subselect.cc item_sum.cc item_timefunc.cc key.cc log.cc lock.cc log_event.cc log_event_server.cc rpl_record.cc rpl_reporting.cc @@ -108,33 +109,33 @@ SET (SQL_SOURCE mysqld.cc net_serv.cc keycaches.cc ../sql-common/client_plugin.c opt_range.cc opt_sum.cc - ../sql-common/pack.c parse_file.cc password.c procedure.cc + ../sql-common/pack.c parse_file.cc password.c procedure.cc protocol.cc records.cc repl_failsafe.cc rpl_filter.cc session_tracker.cc - set_var.cc - slave.cc sp.cc sp_cache.cc sp_head.cc sp_pcontext.cc - sp_rcontext.cc spatial.cc sql_acl.cc sql_analyse.cc sql_base.cc + set_var.cc + slave.cc sp.cc sp_cache.cc sp_head.cc sp_pcontext.cc + sp_rcontext.cc spatial.cc sql_acl.cc sql_analyse.cc sql_base.cc sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_cursor.cc sql_db.cc sql_delete.cc sql_derived.cc - sql_digest.cc sql_do.cc + sql_digest.cc sql_do.cc sql_error.cc sql_handler.cc sql_get_diagnostics.cc - sql_help.cc sql_insert.cc sql_lex.cc + sql_help.cc sql_insert.cc sql_lex.cc sql_list.cc sql_load.cc sql_manager.cc sql_parse.cc sql_bootstrap.cc - sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc + sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc debug_sync.cc debug.cc sql_repl.cc sql_select.cc sql_show.cc sql_state.c group_by_handler.cc derived_handler.cc select_handler.cc sql_statistics.cc sql_string.cc lex_string.h sql_table.cc sql_test.cc sql_trigger.cc sql_udf.cc sql_union.cc ddl_log.cc ddl_log.h - sql_update.cc sql_view.cc strfunc.cc table.cc thr_malloc.cc - sql_time.cc tztime.cc unireg.cc item_xmlfunc.cc + sql_update.cc sql_view.cc strfunc.cc table.cc thr_malloc.cc + sql_time.cc tztime.cc unireg.cc item_xmlfunc.cc uniques.cc rpl_tblmap.cc sql_binlog.cc event_scheduler.cc event_data_objects.cc - event_queue.cc event_db_repository.cc - sql_tablespace.cc events.cc ../sql-common/my_user.c + event_queue.cc event_db_repository.cc + events.cc ../sql-common/my_user.c partition_info.cc rpl_utility.cc rpl_utility_server.cc rpl_injector.cc sql_locale.cc rpl_rli.cc rpl_mi.cc sql_servers.cc sql_audit.cc @@ -150,6 +151,7 @@ SET (SQL_SOURCE sql_analyze_stmt.cc sql_join_cache.cc create_options.cc multi_range_read.cc + opt_histogram_json.cc opt_index_cond_pushdown.cc opt_subselect.cc opt_table_elimination.cc sql_expression_cache.cc gcalc_slicescan.cc gcalc_tools.cc @@ -328,24 +330,24 @@ ENDIF() # On Solaris, some extra effort is required in order to get dtrace probes # from static libraries -DTRACE_INSTRUMENT_STATIC_LIBS(mariadbd +DTRACE_INSTRUMENT_STATIC_LIBS(mariadbd "sql;mysys;mysys_ssl;${MYSQLD_STATIC_PLUGIN_LIBS}") - + SET(WITH_MYSQLD_LDFLAGS "" CACHE STRING "Additional linker flags for mysqld") MARK_AS_ADVANCED(WITH_MYSQLD_LDFLAGS) IF(WITH_MYSQLD_LDFLAGS) GET_TARGET_PROPERTY(MYSQLD_LINK_FLAGS mariadbd LINK_FLAGS) IF(NOT MYSQLD_LINK_FLAGS) - SET(MYSQLD_LINK_FLAGS) - ENDIF() - SET_TARGET_PROPERTIES(mariadbd PROPERTIES LINK_FLAGS + SET(MYSQLD_LINK_FLAGS) + ENDIF() + SET_TARGET_PROPERTIES(mariadbd PROPERTIES LINK_FLAGS "${MYSQLD_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}") ENDIF() -# Handle out-of-source build from source package with possibly broken -# bison. Copy bison output to from source to build directory, if not already +# Handle out-of-source build from source package with possibly broken +# bison. Copy bison output to from source to build directory, if not already # there IF (NOT BISON_FOUND) IF (NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) @@ -407,6 +409,10 @@ ADD_CUSTOM_TARGET( ) ADD_DEPENDENCIES(sql GenServerSource) +IF(TARGET libfmt) + ADD_DEPENDENCIES(sql libfmt) +ENDIF() + IF(WIN32 OR HAVE_DLOPEN AND NOT DISABLE_SHARED) ADD_LIBRARY(udf_example MODULE udf_example.c udf_example.def) SET_TARGET_PROPERTIES(udf_example PROPERTIES PREFIX "") @@ -456,9 +462,9 @@ IF(TARGET mariadbd AND (NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING ELSE() SET(ALL_ON_WINDOWS) ENDIF() - ADD_CUSTOM_TARGET(initial_database + ADD_CUSTOM_TARGET(initial_database ${ALL_ON_WINDOWS} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep ) ENDIF() @@ -467,7 +473,7 @@ IF(WIN32) FILE(TO_NATIVE_PATH ${my_bootstrap_sql} native_outfile) # Create bootstrapper SQL script - ADD_CUSTOM_COMMAND(OUTPUT + ADD_CUSTOM_COMMAND(OUTPUT ${my_bootstrap_sql} COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_BINARY_DIR}/scripts cmd /c copy mysql_system_tables.sql+mysql_system_tables_data.sql+fill_help_tables.sql+mysql_performance_tables.sql+mysql_test_db.sql+mysql_sys_schema.sql ${native_outfile} @@ -490,16 +496,15 @@ IF(WIN32) DEPENDS comp_sql ${my_bootstrap_sql} ) - MYSQL_ADD_EXECUTABLE(mariadb-install-db + MYSQL_ADD_EXECUTABLE(mariadb-install-db mysql_install_db.cc ${CMAKE_CURRENT_BINARY_DIR}/mysql_bootstrap_sql.c + password.c COMPONENT Server ) - SET_TARGET_PROPERTIES(mariadb-install-db PROPERTIES COMPILE_DEFINITIONS - "INSTALL_PLUGINDIR=${INSTALL_PLUGINDIR};INSTALL_SHAREDIR=${INSTALL_SHAREDIR}" - ) - TARGET_LINK_LIBRARIES(mariadb-install-db mysys shlwapi) + "INSTALL_PLUGINDIR=${INSTALL_PLUGINDIR};INSTALL_SHAREDIR=${INSTALL_SHAREDIR}") + TARGET_LINK_LIBRARIES(mariadb-install-db mysys mysys_ssl shlwapi) ADD_LIBRARY(winservice STATIC winservice.c) TARGET_LINK_LIBRARIES(winservice shell32) diff --git a/sql/backup.cc b/sql/backup.cc index 5f74c67add7..3e0986e944f 100644 --- a/sql/backup.cc +++ b/sql/backup.cc @@ -427,6 +427,9 @@ bool backup_end(THD *thd) !wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP)) { Wsrep_server_state &server_state= Wsrep_server_state::instance(); + THD_STAGE_INFO(thd, stage_waiting_flow); + WSREP_DEBUG("backup_end: waiting for flow control for %s", + wsrep_thd_query(thd)); server_state.resume_and_resync(); thd->wsrep_desynced_backup_stage= false; } @@ -578,7 +581,7 @@ static char *add_id_to_buffer(char *ptr, const LEX_CUSTRING *from) tmp.str= buff; tmp.length= MY_UUID_STRING_LENGTH; - my_uuid2str(from->str, buff); + my_uuid2str(from->str, buff, 1); return add_str_to_buffer(ptr, &tmp); } diff --git a/sql/create_options.cc b/sql/create_options.cc index 5437de0f0c3..cea5d7af950 100644 --- a/sql/create_options.cc +++ b/sql/create_options.cc @@ -1,4 +1,4 @@ -/* Copyright (C) 2010, 2020, MariaDB Corporation. +/* Copyright (C) 2010, 2020, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,6 +21,7 @@ #include "mariadb.h" #include "create_options.h" +#include "partition_info.h" #include <my_getopt.h> #include "set_var.h" @@ -339,8 +340,11 @@ bool parse_option_list(THD* thd, handlerton *hton, void *option_struct_arg, LEX_CSTRING name= { opt->name, opt->name_length }; default_val.str= strmake_root(root, str->ptr(), str->length()); default_val.length= str->length(); - val= new (root) engine_option_value(name, default_val, - opt->type != HA_OPTION_TYPE_ULL, option_list, &last); + val= new (root) engine_option_value( + name, default_val, opt->type != HA_OPTION_TYPE_ULL); + if (!val) + DBUG_RETURN(TRUE); + val->link(option_list, &last); val->parsed= true; } } @@ -497,6 +501,61 @@ bool parse_engine_table_options(THD *thd, handlerton *ht, TABLE_SHARE *share) DBUG_RETURN(FALSE); } +#ifdef WITH_PARTITION_STORAGE_ENGINE +/** + Parses engine-defined partition options + + @param [in] thd thread handler + @parem [in] table table with part_info + + @retval TRUE Error + @retval FALSE OK + + In the case of ALTER TABLE statements, table->part_info is set up + by mysql_unpack_partition(). So, one should not call the present + function before the call of mysql_unpack_partition(). +*/ +bool parse_engine_part_options(THD *thd, TABLE *table) +{ + MEM_ROOT *root= &table->mem_root; + TABLE_SHARE *share= table->s; + partition_info *part_info= table->part_info; + engine_option_value *tmp_option_list; + handlerton *ht; + DBUG_ENTER("parse_engine_part_options"); + + if (!part_info) + DBUG_RETURN(FALSE); + + List_iterator<partition_element> it(part_info->partitions); + while (partition_element *part_elem= it++) + { + if (merge_engine_options(share->option_list, part_elem->option_list, + &tmp_option_list, root)) + DBUG_RETURN(TRUE); + + if (!part_info->is_sub_partitioned()) + { + ht= part_elem->engine_type; + if (parse_option_list(thd, ht, &part_elem->option_struct, + &tmp_option_list, ht->table_options, TRUE, root)) + DBUG_RETURN(TRUE); + } + else + { + List_iterator<partition_element> sub_it(part_elem->subpartitions); + while (partition_element *sub_part_elem= sub_it++) + { + ht= sub_part_elem->engine_type; + if (parse_option_list(thd, ht, &sub_part_elem->option_struct, + &tmp_option_list, ht->table_options, TRUE, root)) + DBUG_RETURN(TRUE); + } + } + } + DBUG_RETURN(FALSE); +} +#endif bool engine_options_differ(void *old_struct, void *new_struct, ha_create_table_option *rules) @@ -694,10 +753,11 @@ uchar *engine_option_value::frm_read(const uchar *buff, const uchar *buff_end, return NULL; buff+= value.length; - engine_option_value *ptr=new (root) - engine_option_value(name, value, len & FRM_QUOTED_VALUE, start, end); + engine_option_value *ptr= + new (root) engine_option_value(name, value, len & FRM_QUOTED_VALUE); if (!ptr) return NULL; + ptr->link(start, end); return (uchar *)buff; } @@ -766,23 +826,40 @@ bool engine_table_options_frm_read(const uchar *buff, size_t length, /** Merges two lists of engine_option_value's with duplicate removal. -*/ -engine_option_value *merge_engine_table_options(engine_option_value *first, - engine_option_value *second, - MEM_ROOT *root) + @param [in] source option list + @param [in] changes option list whose options overwrite source's + @param [out] out new option list created by merging given two + @param [in] root MEM_ROOT for allocating memory + + @retval TRUE Error + @retval FALSE OK +*/ +bool merge_engine_options(engine_option_value *source, + engine_option_value *changes, + engine_option_value **out, MEM_ROOT *root) { - engine_option_value *UNINIT_VAR(end), *opt; - DBUG_ENTER("merge_engine_table_options"); + engine_option_value *UNINIT_VAR(end), *opt, *opt_copy; + *out= 0; + DBUG_ENTER("merge_engine_options"); - /* Create copy of first list */ - for (opt= first, first= 0; opt; opt= opt->next) - new (root) engine_option_value(opt, &first, &end); + /* Create copy of source list */ + for (opt= source; opt; opt= opt->next) + { + opt_copy= new (root) engine_option_value(opt); + if (!opt_copy) + DBUG_RETURN(TRUE); + opt_copy->link(out, &end); + } - for (opt= second; opt; opt= opt->next) - new (root) engine_option_value(opt->name, opt->value, opt->quoted_value, - &first, &end); - DBUG_RETURN(first); + for (opt= changes; opt; opt= opt->next) + { + opt_copy= new (root) engine_option_value(opt); + if (!opt_copy) + DBUG_RETURN(TRUE); + opt_copy->link(out, &end); + } + DBUG_RETURN(FALSE); } bool is_engine_option_known(engine_option_value *opt, diff --git a/sql/create_options.h b/sql/create_options.h index ce64516794b..4961231820f 100644 --- a/sql/create_options.h +++ b/sql/create_options.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2010 Monty Program Ab +/* Copyright (C) 2010, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -35,30 +35,23 @@ class engine_option_value: public Sql_alloc bool parsed; ///< to detect unrecognized options bool quoted_value; ///< option=VAL vs. option='VAL' - engine_option_value(engine_option_value *src, - engine_option_value **start, engine_option_value **end) : + engine_option_value(engine_option_value *src) : name(src->name), value(src->value), next(NULL), parsed(src->parsed), quoted_value(src->quoted_value) { - link(start, end); } engine_option_value(LEX_CSTRING &name_arg, LEX_CSTRING &value_arg, - bool quoted, - engine_option_value **start, engine_option_value **end) : + bool quoted) : name(name_arg), value(value_arg), next(NULL), parsed(false), quoted_value(quoted) { - link(start, end); } - engine_option_value(LEX_CSTRING &name_arg, - engine_option_value **start, engine_option_value **end) : + engine_option_value(LEX_CSTRING &name_arg): name(name_arg), value(null_clex_str), next(NULL), parsed(false), quoted_value(false) { - link(start, end); } engine_option_value(LEX_CSTRING &name_arg, ulonglong value_arg, - engine_option_value **start, engine_option_value **end, MEM_ROOT *root) : name(name_arg), next(NULL), parsed(false), quoted_value(false) { @@ -66,7 +59,6 @@ class engine_option_value: public Sql_alloc if (likely((value.str= str= (char *)alloc_root(root, 22)))) { value.length= longlong10_to_str(value_arg, str, 10) - str; - link(start, end); } } static uchar *frm_read(const uchar *buff, const uchar *buff_end, @@ -83,15 +75,18 @@ class Create_field; bool resolve_sysvar_table_options(handlerton *hton); void free_sysvar_table_options(handlerton *hton); bool parse_engine_table_options(THD *thd, handlerton *ht, TABLE_SHARE *share); +#ifdef WITH_PARTITION_STORAGE_ENGINE +bool parse_engine_part_options(THD *thd, TABLE *table); +#endif bool parse_option_list(THD* thd, handlerton *hton, void *option_struct, engine_option_value **option_list, ha_create_table_option *rules, bool suppress_warning, MEM_ROOT *root); bool engine_table_options_frm_read(const uchar *buff, size_t length, TABLE_SHARE *share); -engine_option_value *merge_engine_table_options(engine_option_value *source, - engine_option_value *changes, - MEM_ROOT *root); +bool merge_engine_options(engine_option_value *source, + engine_option_value *changes, + engine_option_value **out, MEM_ROOT *root); uint engine_table_options_frm_length(engine_option_value *table_option_list, List<Create_field> &create_fields, diff --git a/sql/ddl_log.cc b/sql/ddl_log.cc index 8722d88ba95..2ac4e256112 100644 --- a/sql/ddl_log.cc +++ b/sql/ddl_log.cc @@ -77,6 +77,8 @@ #define DDL_LOG_MAGIC_LENGTH 4 /* How many times to try to execute a ddl log entry that causes crashes */ #define DDL_LOG_MAX_RETRY 3 +#define DDL_LOG_RETRY_MASK 0xFF +#define DDL_LOG_RETRY_BITS 8 uchar ddl_log_file_magic[]= { (uchar) 254, (uchar) 254, (uchar) 11, (uchar) 2 }; @@ -155,7 +157,7 @@ mysql_mutex_t LOCK_gdl; #define DDL_LOG_XID_POS 10 /* Used to store unique uuid from the .frm file */ #define DDL_LOG_UUID_POS 18 -/* ID_POS can be used to store something unique, like file size (4 bytes) */ +/* ID_POS can be used to store something unique, like file size (8 bytes) */ #define DDL_LOG_ID_POS DDL_LOG_UUID_POS + MY_UUID_SIZE #define DDL_LOG_END_POS DDL_LOG_ID_POS + 8 @@ -335,7 +337,7 @@ static bool write_ddl_log_file_entry(uint entry_pos) static bool update_phase(uint entry_pos, uchar phase) { DBUG_ENTER("update_phase"); - DBUG_PRINT("enter", ("phase: %d", (int) phase)); + DBUG_PRINT("ddl_log", ("pos: %u phase: %u", entry_pos, (uint) phase)); DBUG_RETURN(mysql_file_pwrite(global_ddl_log.file_id, &phase, 1, global_ddl_log.io_size * entry_pos + @@ -369,6 +371,8 @@ static bool update_next_entry_pos(uint entry_pos, uint next_entry) uchar buff[4]; DBUG_ENTER("update_next_entry_pos"); + DBUG_PRINT("ddl_log", ("pos: %u->%u", entry_pos, next_entry)); + int4store(buff, next_entry); DBUG_RETURN(mysql_file_pwrite(global_ddl_log.file_id, buff, sizeof(buff), global_ddl_log.io_size * entry_pos + @@ -420,6 +424,7 @@ static bool disable_execute_entry(uint entry_pos) { uchar buff[1]; DBUG_ENTER("disable_execute_entry"); + DBUG_PRINT("ddl_log", ("pos: {%u}", entry_pos)); buff[0]= DDL_LOG_IGNORE_ENTRY_CODE; DBUG_RETURN(mysql_file_pwrite(global_ddl_log.file_id, buff, sizeof(buff), @@ -1296,13 +1301,15 @@ static int ddl_log_execute_action(THD *thd, MEM_ROOT *mem_root, mysql_mutex_assert_owner(&LOCK_gdl); DBUG_PRINT("ddl_log", - ("entry type: %u action type: %u (%s) phase: %u next: %u " + ("pos: %u=>%u->%u type: %u action: %u (%s) phase: %u " "handler: '%s' name: '%s' from_name: '%s' tmp_name: '%s'", + recovery_state.execute_entry_pos, + ddl_log_entry->entry_pos, + ddl_log_entry->next_entry, (uint) ddl_log_entry->entry_type, (uint) ddl_log_entry->action_type, ddl_log_action_name[ddl_log_entry->action_type], (uint) ddl_log_entry->phase, - ddl_log_entry->next_entry, ddl_log_entry->handler_name.str, ddl_log_entry->name.str, ddl_log_entry->from_name.str, @@ -2470,13 +2477,13 @@ bool ddl_log_write_entry(DDL_LOG_ENTRY *ddl_log_entry, error= FALSE; DBUG_PRINT("ddl_log", - ("entry type: %u action type: %u (%s) phase: %u next: %u " + ("pos: %u->%u action: %u (%s) phase: %u " "handler: '%s' name: '%s' from_name: '%s' tmp_name: '%s'", - (uint) ddl_log_entry->entry_type, + (*active_entry)->entry_pos, + (uint) ddl_log_entry->next_entry, (uint) ddl_log_entry->action_type, ddl_log_action_name[ddl_log_entry->action_type], (uint) ddl_log_entry->phase, - ddl_log_entry->next_entry, ddl_log_entry->handler_name.str, ddl_log_entry->name.str, ddl_log_entry->from_name.str, @@ -2510,6 +2517,7 @@ bool ddl_log_write_entry(DDL_LOG_ENTRY *ddl_log_entry, @param first_entry First entry in linked list of entries to execute. + @param cond_entry Check and don't execute if cond_entry is active @param[in,out] active_entry Entry to execute, 0 = NULL if the entry is written first time and needs to be returned. In this case the entry written @@ -2520,6 +2528,7 @@ bool ddl_log_write_entry(DDL_LOG_ENTRY *ddl_log_entry, */ bool ddl_log_write_execute_entry(uint first_entry, + uint cond_entry, DDL_LOG_MEMORY_ENTRY **active_entry) { uchar *file_entry_buf= global_ddl_log.file_entry_buf; @@ -2536,13 +2545,17 @@ bool ddl_log_write_execute_entry(uint first_entry, file_entry_buf[DDL_LOG_ENTRY_TYPE_POS]= (uchar)DDL_LOG_EXECUTE_CODE; int4store(file_entry_buf + DDL_LOG_NEXT_ENTRY_POS, first_entry); + int8store(file_entry_buf + DDL_LOG_ID_POS, ((ulonglong)cond_entry << DDL_LOG_RETRY_BITS)); if (!(*active_entry)) { if (ddl_log_get_free_entry(active_entry)) DBUG_RETURN(TRUE); got_free_entry= TRUE; - } + } + DBUG_PRINT("ddl_log", + ("pos: %u=>%u", + (*active_entry)->entry_pos, first_entry)); if (write_ddl_log_file_entry((*active_entry)->entry_pos)) { sql_print_error("DDL_LOG: Error writing execute entry %u", @@ -2575,6 +2588,7 @@ bool ddl_log_increment_phase(uint entry_pos) { bool error; DBUG_ENTER("ddl_log_increment_phase"); + DBUG_PRINT("ddl_log", ("pos: %u", entry_pos)); mysql_mutex_lock(&LOCK_gdl); error= ddl_log_increment_phase_no_lock(entry_pos); @@ -2754,13 +2768,13 @@ int ddl_log_execute_recovery() recovery_state.xid= ddl_log_entry.xid; /* purecov: begin tested */ - if (ddl_log_entry.unique_id > DDL_LOG_MAX_RETRY) + if ((ddl_log_entry.unique_id & DDL_LOG_RETRY_MASK) > DDL_LOG_MAX_RETRY) { error= -1; continue; } update_unique_id(i, ++ddl_log_entry.unique_id); - if (ddl_log_entry.unique_id > DDL_LOG_MAX_RETRY) + if ((ddl_log_entry.unique_id & DDL_LOG_RETRY_MASK) > DDL_LOG_MAX_RETRY) { sql_print_error("DDL_LOG: Aborting executing entry %u after %llu " "retries", i, ddl_log_entry.unique_id); @@ -2769,6 +2783,15 @@ int ddl_log_execute_recovery() } /* purecov: end tested */ + uint cond_entry= (uint)(ddl_log_entry.unique_id >> DDL_LOG_RETRY_BITS); + + if (cond_entry && is_execute_entry_active(cond_entry)) + { + if (disable_execute_entry(i)) + error= -1; + continue; + } + if (ddl_log_execute_entry_no_lock(thd, ddl_log_entry.next_entry)) { /* Real unpleasant scenario but we have to continue anyway */ @@ -2848,8 +2871,7 @@ void ddl_log_release() Methods for DDL_LOG_STATE */ -static void add_log_entry(DDL_LOG_STATE *state, - DDL_LOG_MEMORY_ENTRY *log_entry) +void ddl_log_add_entry(DDL_LOG_STATE *state, DDL_LOG_MEMORY_ENTRY *log_entry) { log_entry->next_active_log_entry= state->list; state->main_entry= state->list= log_entry; @@ -3028,7 +3050,7 @@ static bool ddl_log_write(DDL_LOG_STATE *ddl_state, ddl_log_release_memory_entry(log_entry); DBUG_RETURN(1); } - add_log_entry(ddl_state, log_entry); + ddl_log_add_entry(ddl_state, log_entry); ddl_state->flags|= ddl_log_entry->flags; // Update cache DBUG_RETURN(0); } @@ -3178,7 +3200,7 @@ static bool ddl_log_drop(THD *thd, DDL_LOG_STATE *ddl_state, } mysql_mutex_unlock(&LOCK_gdl); - add_log_entry(ddl_state, log_entry); + ddl_log_add_entry(ddl_state, log_entry); DBUG_RETURN(0); error: @@ -3478,7 +3500,7 @@ bool ddl_log_store_query(THD *thd, DDL_LOG_STATE *ddl_state, goto err; parent_entry_pos= ddl_state->list->entry_pos; entry_pos= first_entry->entry_pos; - add_log_entry(ddl_state, first_entry); + ddl_log_add_entry(ddl_state, first_entry); while (length) { @@ -3494,7 +3516,7 @@ bool ddl_log_store_query(THD *thd, DDL_LOG_STATE *ddl_state, if (ddl_log_get_free_entry(&next_entry)) goto err; ddl_log_entry.next_entry= next_entry_pos= next_entry->entry_pos; - add_log_entry(ddl_state, next_entry); + ddl_log_add_entry(ddl_state, next_entry); } else { @@ -3526,3 +3548,32 @@ err: mysql_mutex_unlock(&LOCK_gdl); DBUG_RETURN(1); } + + +/* + Log an delete frm file +*/ + +/* + TODO: Partitioning atomic DDL refactoring: this should be replaced with + ddl_log_create_table(). +*/ +bool ddl_log_delete_frm(DDL_LOG_STATE *ddl_state, const char *to_path) +{ + DDL_LOG_ENTRY ddl_log_entry; + DDL_LOG_MEMORY_ENTRY *log_entry; + DBUG_ENTER("ddl_log_delete_frm"); + bzero(&ddl_log_entry, sizeof(ddl_log_entry)); + ddl_log_entry.action_type= DDL_LOG_DELETE_ACTION; + ddl_log_entry.next_entry= ddl_state->list ? ddl_state->list->entry_pos : 0; + + lex_string_set(&ddl_log_entry.handler_name, reg_ext); + lex_string_set(&ddl_log_entry.name, to_path); + + mysql_mutex_assert_owner(&LOCK_gdl); + if (ddl_log_write_entry(&ddl_log_entry, &log_entry)) + DBUG_RETURN(1); + + ddl_log_add_entry(ddl_state, log_entry); + DBUG_RETURN(0); +} diff --git a/sql/ddl_log.h b/sql/ddl_log.h index a2a6af76a77..9960855a813 100644 --- a/sql/ddl_log.h +++ b/sql/ddl_log.h @@ -262,8 +262,14 @@ int ddl_log_execute_recovery(); bool ddl_log_write_entry(DDL_LOG_ENTRY *ddl_log_entry, DDL_LOG_MEMORY_ENTRY **active_entry); +bool ddl_log_write_execute_entry(uint first_entry, uint cond_entry, + DDL_LOG_MEMORY_ENTRY** active_entry); +inline bool ddl_log_write_execute_entry(uint first_entry, - DDL_LOG_MEMORY_ENTRY **active_entry); + DDL_LOG_MEMORY_ENTRY **active_entry) +{ + return ddl_log_write_execute_entry(first_entry, 0, active_entry); +} bool ddl_log_disable_execute_entry(DDL_LOG_MEMORY_ENTRY **active_entry); void ddl_log_complete(DDL_LOG_STATE *ddl_log_state); @@ -279,6 +285,7 @@ void ddl_log_release_memory_entry(DDL_LOG_MEMORY_ENTRY *log_entry); bool ddl_log_sync(); bool ddl_log_execute_entry(THD *thd, uint first_entry); +void ddl_log_add_entry(DDL_LOG_STATE *state, DDL_LOG_MEMORY_ENTRY *log_entry); void ddl_log_release_entries(DDL_LOG_STATE *ddl_log_state); bool ddl_log_rename_table(THD *thd, DDL_LOG_STATE *ddl_state, handlerton *hton, @@ -348,5 +355,6 @@ bool ddl_log_alter_table(THD *thd, DDL_LOG_STATE *ddl_state, bool is_renamed); bool ddl_log_store_query(THD *thd, DDL_LOG_STATE *ddl_log_state, const char *query, size_t length); +bool ddl_log_delete_frm(DDL_LOG_STATE *ddl_state, const char *to_path); extern mysql_mutex_t LOCK_gdl; #endif /* DDL_LOG_INCLUDED */ diff --git a/sql/discover.cc b/sql/discover.cc index 22d7008630a..201169357a2 100644 --- a/sql/discover.cc +++ b/sql/discover.cc @@ -233,7 +233,7 @@ int extension_based_table_discovery(MY_DIR *dirp, const char *ext_meta, cur++; } advance(from, to, cur, skip); - dirp->number_of_files= (uint)(to - dirp->dir_entry); + dirp->number_of_files= to - dirp->dir_entry; return 0; } diff --git a/sql/encryption.cc b/sql/encryption.cc index 13239b91910..3c7ba2e997b 100644 --- a/sql/encryption.cc +++ b/sql/encryption.cc @@ -109,6 +109,7 @@ int initialize_encryption_plugin(st_plugin_int *plugin) int finalize_encryption_plugin(st_plugin_int *plugin) { + int deinit_status= 0; bool used= plugin_ref_to_int(encryption_manager) == plugin; if (used) @@ -118,18 +119,15 @@ int finalize_encryption_plugin(st_plugin_int *plugin) encryption_handler.encryption_ctx_size_func= zero_size; } - if (plugin && plugin->plugin->deinit && plugin->plugin->deinit(NULL)) - { - DBUG_PRINT("warning", ("Plugin '%s' deinit function returned error.", - plugin->name.str)); - } + if (plugin && plugin->plugin->deinit) + deinit_status= plugin->plugin->deinit(NULL); if (used) { plugin_unlock(NULL, encryption_manager); encryption_manager= 0; } - return 0; + return deinit_status; } /****************************************************************** diff --git a/sql/field.cc b/sql/field.cc index f413d77be87..c49d4d6f4a7 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -964,6 +964,51 @@ Type_handler::aggregate_for_result_traditional(const Type_handler *a, } +bool Field::check_assignability_from(const Type_handler *from, + bool ignore) const +{ + /* + Using type_handler_for_item_field() here to get the data type handler + on both sides. This is needed to make sure aggregation for Field + works the same way with how Item_field aggregates for UNION or CASE, + so these statements: + SELECT a FROM t1 UNION SELECT b FROM t1; // Item_field vs Item_field + UPDATE t1 SET a=b; // Field vs Item_field + either both return "Illegal parameter data types" or both pass + the data type compatibility test. + For MariaDB standard data types, using type_handler_for_item_field() + turns ENUM/SET into just CHAR. + */ + Type_handler_hybrid_field_type th(type_handler()-> + type_handler_for_item_field()); + if (th.aggregate_for_result(from->type_handler_for_item_field())) + { + bool error= (!ignore && get_thd()->is_strict_mode()) || + (type_handler()->is_scalar_type() != from->is_scalar_type()); + /* + Display fully qualified column name for table columns. + Display non-qualified names for other things, + e.g. SP variables, SP return values, SP and CURSOR parameters. + */ + if (table->s->db.str && table->s->table_name.str) + my_printf_error(ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION, + "Cannot cast '%s' as '%s' in assignment of %`s.%`s.%`s", + MYF(error ? 0 : ME_WARNING), + from->name().ptr(), type_handler()->name().ptr(), + table->s->db.str, table->s->table_name.str, + field_name.str); + else + my_printf_error(ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION, + "Cannot cast '%s' as '%s' in assignment of %`s", + MYF(error ? 0 : ME_WARNING), + from->name().ptr(), type_handler()->name().ptr(), + field_name.str); + return error; + } + return false; +} + + /* Test if the given string contains important data: not spaces for character string, @@ -1108,19 +1153,27 @@ Field_longstr::pack_sort_string(uchar *to, const SORT_FIELD_ATTR *sort_field) relative position of the field value in the numeric interval [min,max] */ -double Field::pos_in_interval_val_real(Field *min, Field *max) +double pos_in_interval_for_double(double midp_val, double min_val, + double max_val) { double n, d; - n= val_real() - min->val_real(); + n= midp_val - min_val; if (n < 0) return 0.0; - d= max->val_real() - min->val_real(); + d= max_val - min_val; if (d <= 0) return 1.0; return MY_MIN(n/d, 1.0); } +double Field::pos_in_interval_val_real(Field *min, Field *max) +{ + return pos_in_interval_for_double(val_real(), min->val_real(), + max->val_real()); +} + + static inline ulonglong char_prefix_to_ulonglong(uchar *src) { @@ -1178,22 +1231,32 @@ static inline double safe_substract(ulonglong a, ulonglong b) double Field::pos_in_interval_val_str(Field *min, Field *max, uint data_offset) { + return pos_in_interval_for_string(charset(), + ptr + data_offset, data_length(), + min->ptr + data_offset, min->data_length(), + max->ptr + data_offset, max->data_length() + ); +} + + +double pos_in_interval_for_string(CHARSET_INFO *cset, + const uchar *midp_val, uint32 midp_len, + const uchar *min_val, uint32 min_len, + const uchar *max_val, uint32 max_len) +{ uchar mp_prefix[sizeof(ulonglong)]; uchar minp_prefix[sizeof(ulonglong)]; uchar maxp_prefix[sizeof(ulonglong)]; ulonglong mp, minp, maxp; - charset()->strnxfrm(mp_prefix, sizeof(mp), - ptr + data_offset, - data_length()); - charset()->strnxfrm(minp_prefix, sizeof(minp), - min->ptr + data_offset, - min->data_length()); - charset()->strnxfrm(maxp_prefix, sizeof(maxp), - max->ptr + data_offset, - max->data_length()); - mp= char_prefix_to_ulonglong(mp_prefix); + + cset->strnxfrm(mp_prefix, sizeof(mp), midp_val, midp_len); + cset->strnxfrm(minp_prefix, sizeof(minp), min_val, min_len); + cset->strnxfrm(maxp_prefix, sizeof(maxp), max_val, max_len); + + mp= char_prefix_to_ulonglong(mp_prefix); minp= char_prefix_to_ulonglong(minp_prefix); maxp= char_prefix_to_ulonglong(maxp_prefix); + double n, d; n= safe_substract(mp, minp); if (n < 0) @@ -1427,18 +1490,9 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) Item *expr_item; - if (!(expr_item= thd->sp_prepare_func_item(value, 1))) + if (!(expr_item= thd->sp_fix_func_item_for_assignment(this, value))) goto error; - /* - expr_item is now fixed, it's safe to call cmp_type() - */ - if (expr_item->cmp_type() == ROW_RESULT) - { - my_error(ER_OPERAND_COLUMNS, MYF(0), 1); - goto error; - } - /* Save the value in the field. Convert the value if needed. */ expr_item->save_in_field(this, 0); @@ -11127,6 +11181,14 @@ Field::set_warning(Sql_condition::enum_warning_level level, uint code, will have table == NULL. */ THD *thd= get_thd(); + + /* + In INPLACE ALTER, server can't know which row has generated + the warning, so the value of current row is supplied by the engine. + */ + if (current_row) + thd->get_stmt_da()->reset_current_row_for_warning(current_row); + if (thd->count_cuted_fields > CHECK_FIELD_EXPRESSION) { thd->cuted_fields+= cut_increment; diff --git a/sql/field.h b/sql/field.h index b146ded321d..c14a1fc4e7f 100644 --- a/sql/field.h +++ b/sql/field.h @@ -908,6 +908,12 @@ public: bool is_unsigned() const { return flags & UNSIGNED_FLAG; } + bool check_assignability_from(const Type_handler *from, bool ignore) const; + bool check_assignability_from(const Field *from, bool ignore) const + { + return check_assignability_from(from->type_handler(), ignore); + } + /** Convenience definition of a copy function returned by Field::get_copy_func() @@ -1525,11 +1531,20 @@ public: if (null_ptr) null_ptr=ADD_TO_PTR(null_ptr,ptr_diff,uchar*); } + + /* + Copy the Field's value to buff. The value will be in table->record[] + format. + */ void get_image(uchar *buff, uint length, CHARSET_INFO *cs) const { get_image(buff, length, ptr, cs); } virtual void get_image(uchar *buff, uint length, const uchar *ptr_arg, CHARSET_INFO *cs) const { memcpy(buff,ptr_arg,length); } + + /* + Set Field's value to the value in *buf. + */ virtual void set_image(const uchar *buff,uint length, CHARSET_INFO *cs) { memcpy(ptr,buff,length); } @@ -1869,6 +1884,7 @@ public: { return (double) 0.5; } + virtual bool pos_through_val_str() { return false;} /* Check if comparison between the field and an item unambiguously @@ -2154,6 +2170,8 @@ public: { return pos_in_interval_val_str(min, max, length_size()); } + bool pos_through_val_str() override {return true;} + bool test_if_equality_guarantees_uniqueness(const Item *const_item) const override; SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *param, KEY_PART *key_part, @@ -5901,4 +5919,12 @@ ulonglong TABLE::vers_start_id() const return static_cast<ulonglong>(vers_start_field()->val_int()); } +double pos_in_interval_for_string(CHARSET_INFO *cset, + const uchar *midp_val, uint32 midp_len, + const uchar *min_val, uint32 min_len, + const uchar *max_val, uint32 max_len); + +double pos_in_interval_for_double(double midp_val, + double min_val, double max_val); + #endif /* FIELD_INCLUDED */ diff --git a/sql/gcalc_slicescan.cc b/sql/gcalc_slicescan.cc index b079bd7a714..f94c7190532 100644 --- a/sql/gcalc_slicescan.cc +++ b/sql/gcalc_slicescan.cc @@ -172,7 +172,7 @@ static void GCALC_DBUG_PRINT_SLICE(const char *header, Gcalc_dyn_list::Gcalc_dyn_list(size_t blk_size, size_t sizeof_item): - m_blk_size(blk_size - ALLOC_ROOT_MIN_BLOCK_SIZE), + m_blk_size(blk_size), m_sizeof_item(ALIGN_SIZE(sizeof_item)), m_points_per_blk((uint)((m_blk_size - PH_DATA_OFFSET) / m_sizeof_item)), m_blk_hook(&m_first_blk), diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index f50aaa578fb..acdadb61df5 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -2143,7 +2143,12 @@ int ha_partition::change_partitions(HA_CREATE_INFO *create_info, } DBUG_ASSERT(m_new_file == 0); m_new_file= new_file_array; - if (unlikely((error= copy_partitions(copied, deleted)))) + for (i= 0; i < part_count; i++) + m_added_file[i]->extra(HA_EXTRA_BEGIN_ALTER_COPY); + error= copy_partitions(copied, deleted); + for (i= 0; i < part_count; i++) + m_added_file[i]->extra(HA_EXTRA_END_ALTER_COPY); + if (unlikely(error)) { /* Close and unlock the new temporary partitions. @@ -2733,6 +2738,7 @@ register_query_cache_dependant_tables(THD *thd, 2) MAX_ROWS, MIN_ROWS on partition 3) Index file name on partition 4) Data file name on partition + 5) Engine-defined attributes on partition */ int ha_partition::set_up_table_before_create(TABLE *tbl, @@ -2770,6 +2776,10 @@ int ha_partition::set_up_table_before_create(TABLE *tbl, if (info->connect_string.length) info->used_fields|= HA_CREATE_USED_CONNECTION; tbl->s->connect_string= part_elem->connect_string; + if (part_elem->option_list) + tbl->s->option_list= part_elem->option_list; + if (part_elem->option_struct) + tbl->s->option_struct= part_elem->option_struct; DBUG_RETURN(0); } @@ -3625,31 +3635,31 @@ bool ha_partition::init_partition_bitmaps() DBUG_ENTER("ha_partition::init_partition_bitmaps"); /* Initialize the bitmap we use to minimize ha_start_bulk_insert calls */ - if (my_bitmap_init(&m_bulk_insert_started, NULL, m_tot_parts + 1, FALSE)) + if (my_bitmap_init(&m_bulk_insert_started, NULL, m_tot_parts + 1)) DBUG_RETURN(true); /* Initialize the bitmap we use to keep track of locked partitions */ - if (my_bitmap_init(&m_locked_partitions, NULL, m_tot_parts, FALSE)) + if (my_bitmap_init(&m_locked_partitions, NULL, m_tot_parts)) DBUG_RETURN(true); /* Initialize the bitmap we use to keep track of partitions which may have something to reset in ha_reset(). */ - if (my_bitmap_init(&m_partitions_to_reset, NULL, m_tot_parts, FALSE)) + if (my_bitmap_init(&m_partitions_to_reset, NULL, m_tot_parts)) DBUG_RETURN(true); /* Initialize the bitmap we use to keep track of partitions which returned HA_ERR_KEY_NOT_FOUND from index_read_map. */ - if (my_bitmap_init(&m_key_not_found_partitions, NULL, m_tot_parts, FALSE)) + if (my_bitmap_init(&m_key_not_found_partitions, NULL, m_tot_parts)) DBUG_RETURN(true); - if (bitmap_init(&m_mrr_used_partitions, NULL, m_tot_parts, TRUE)) + if (my_bitmap_init(&m_mrr_used_partitions, NULL, m_tot_parts)) DBUG_RETURN(true); - if (my_bitmap_init(&m_opened_partitions, NULL, m_tot_parts, FALSE)) + if (my_bitmap_init(&m_opened_partitions, NULL, m_tot_parts)) DBUG_RETURN(true); m_file_sample= NULL; diff --git a/sql/handler.cc b/sql/handler.cc index c683a94e292..3081d4b0d3c 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2016, Oracle and/or its affiliates. - Copyright (c) 2009, 2022, MariaDB Corporation. + Copyright (c) 2009, 2023, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -581,6 +581,7 @@ static int hton_drop_table(handlerton *hton, const char *path) int ha_finalize_handlerton(st_plugin_int *plugin) { + int deinit_status= 0; handlerton *hton= (handlerton *)plugin->data; DBUG_ENTER("ha_finalize_handlerton"); @@ -595,18 +596,7 @@ int ha_finalize_handlerton(st_plugin_int *plugin) hton->panic(hton, HA_PANIC_CLOSE); if (plugin->plugin->deinit) - { - /* - Today we have no defined/special behavior for uninstalling - engine plugins. - */ - DBUG_PRINT("info", ("Deinitializing plugin: '%s'", plugin->name.str)); - if (plugin->plugin->deinit(NULL)) - { - DBUG_PRINT("warning", ("Plugin '%s' deinit function returned error.", - plugin->name.str)); - } - } + deinit_status= plugin->plugin->deinit(NULL); free_sysvar_table_options(hton); update_discovery_counters(hton, -1); @@ -627,7 +617,7 @@ int ha_finalize_handlerton(st_plugin_int *plugin) my_free(hton); end: - DBUG_RETURN(0); + DBUG_RETURN(deinit_status); } @@ -1779,6 +1769,13 @@ int ha_commit_trans(THD *thd, bool all) if (ha_info->ht()->prepare_commit_versioned) { trx_end_id= ha_info->ht()->prepare_commit_versioned(thd, &trx_start_id); + + if (trx_end_id == ULONGLONG_MAX) + { + my_error(ER_ERROR_DURING_COMMIT, MYF(0), 1); + goto err; + } + if (trx_end_id) break; // FIXME: use a common ID for cross-engine transactions } @@ -4168,6 +4165,9 @@ void handler::get_auto_increment(ulonglong offset, ulonglong increment, int error; MY_BITMAP *old_read_set; bool rnd_inited= (inited == RND); + bool rev= table->key_info[table->s->next_number_index]. + key_part[table->s->next_number_keypart].key_part_flag & + HA_REVERSE_SORT; if (rnd_inited && ha_rnd_end()) return; @@ -4189,7 +4189,8 @@ void handler::get_auto_increment(ulonglong offset, ulonglong increment, if (table->s->next_number_keypart == 0) { // Autoincrement at key-start - error= ha_index_last(table->record[1]); + error= rev ? ha_index_first(table->record[1]) + : ha_index_last(table->record[1]); /* MySQL implicitly assumes such method does locking (as MySQL decides to use nr+increment without checking again with the handler, in @@ -4204,9 +4205,8 @@ void handler::get_auto_increment(ulonglong offset, ulonglong increment, table->key_info + table->s->next_number_index, table->s->next_number_key_offset); error= ha_index_read_map(table->record[1], key, - make_prev_keypart_map(table->s-> - next_number_keypart), - HA_READ_PREFIX_LAST); + make_prev_keypart_map(table->s->next_number_keypart), + rev ? HA_READ_KEY_EXACT : HA_READ_PREFIX_LAST); /* MySQL needs to call us for next row: assume we are inserting ("a",null) here, we return 3, and next this statement will want to insert @@ -8154,7 +8154,7 @@ static Create_field *vers_init_sys_field(THD *thd, const char *field_name, int f f->set_handler(&type_handler_timestamp2); f->length= MAX_DATETIME_PRECISION; } - f->invisible= DBUG_EVALUATE_IF("sysvers_show", VISIBLE, INVISIBLE_SYSTEM); + f->invisible= DBUG_IF("sysvers_show") ? VISIBLE : INVISIBLE_SYSTEM; if (f->check(thd)) return NULL; @@ -8167,8 +8167,8 @@ bool Vers_parse_info::create_sys_field(THD *thd, const char *field_name, { DBUG_ASSERT(can_native >= 0); /* Requires vers_check_native() called */ Create_field *f= vers_init_sys_field(thd, field_name, flags, - DBUG_EVALUATE_IF("sysvers_force_trx", - (bool) can_native, false)); + DBUG_IF("sysvers_force_trx") && + can_native); if (!f) return true; @@ -8214,8 +8214,7 @@ bool Table_scope_and_contents_source_st::vers_fix_system_fields( { DBUG_ASSERT(!(alter_info->flags & ALTER_DROP_SYSTEM_VERSIONING)); - if (DBUG_EVALUATE_IF("sysvers_force", true, false) || - DBUG_EVALUATE_IF("sysvers_force_trx", true, false)) + if (DBUG_IF("sysvers_force") || DBUG_IF("sysvers_force_trx")) { alter_info->flags|= ALTER_ADD_SYSTEM_VERSIONING; options|= HA_VERSIONED_TABLE; @@ -8318,8 +8317,8 @@ bool Vers_parse_info::fix_alter_info(THD *thd, Alter_info *alter_info, if (!need_check(alter_info) && !share->versioned) return false; - if (DBUG_EVALUATE_IF("sysvers_force", 0, share->tmp_table) || - DBUG_EVALUATE_IF("sysvers_force_trx", 0, share->tmp_table)) + if (share->tmp_table && + !DBUG_IF("sysvers_force") && !DBUG_IF("sysvers_force_trx")) { my_error(ER_VERS_NOT_SUPPORTED, MYF(0), "CREATE TEMPORARY TABLE"); return true; diff --git a/sql/handler.h b/sql/handler.h index e3d968808ee..36f66b78de0 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -838,7 +838,9 @@ typedef bool Log_func(THD*, TABLE*, bool, const uchar*, const uchar*); // Set by Sql_cmd_alter_table_truncate_partition::execute() #define ALTER_PARTITION_TRUNCATE (1ULL << 11) // Set for REORGANIZE PARTITION -#define ALTER_PARTITION_TABLE_REORG (1ULL << 12) +#define ALTER_PARTITION_TABLE_REORG (1ULL << 12) +#define ALTER_PARTITION_CONVERT_IN (1ULL << 13) +#define ALTER_PARTITION_CONVERT_OUT (1ULL << 14) /* This is master database for most of system tables. However there @@ -1000,39 +1002,6 @@ struct xid_recovery_member #define MIN_XID_LIST_SIZE 128 #define MAX_XID_LIST_SIZE (1024*128) -/* - These structures are used to pass information from a set of SQL commands - on add/drop/change tablespace definitions to the proper hton. -*/ -#define UNDEF_NODEGROUP 65535 -enum ts_command_type -{ - TS_CMD_NOT_DEFINED = -1, - CREATE_TABLESPACE = 0, - ALTER_TABLESPACE = 1, - CREATE_LOGFILE_GROUP = 2, - ALTER_LOGFILE_GROUP = 3, - DROP_TABLESPACE = 4, - DROP_LOGFILE_GROUP = 5, - CHANGE_FILE_TABLESPACE = 6, - ALTER_ACCESS_MODE_TABLESPACE = 7 -}; - -enum ts_alter_tablespace_type -{ - TS_ALTER_TABLESPACE_TYPE_NOT_DEFINED = -1, - ALTER_TABLESPACE_ADD_FILE = 1, - ALTER_TABLESPACE_DROP_FILE = 2 -}; - -enum tablespace_access_mode -{ - TS_NOT_DEFINED= -1, - TS_READ_ONLY = 0, - TS_READ_WRITE = 1, - TS_NOT_ACCESSIBLE = 2 -}; - /* Statistics about batch operations like bulk_insert */ struct ha_copy_info { @@ -1043,50 +1012,6 @@ struct ha_copy_info ha_rows updated; }; -struct handlerton; -class st_alter_tablespace : public Sql_alloc -{ - public: - const char *tablespace_name; - const char *logfile_group_name; - enum ts_command_type ts_cmd_type; - enum ts_alter_tablespace_type ts_alter_tablespace_type; - const char *data_file_name; - const char *undo_file_name; - const char *redo_file_name; - ulonglong extent_size; - ulonglong undo_buffer_size; - ulonglong redo_buffer_size; - ulonglong initial_size; - ulonglong autoextend_size; - ulonglong max_size; - uint nodegroup_id; - handlerton *storage_engine; - bool wait_until_completed; - const char *ts_comment; - enum tablespace_access_mode ts_access_mode; - st_alter_tablespace() - { - tablespace_name= NULL; - logfile_group_name= "DEFAULT_LG"; //Default log file group - ts_cmd_type= TS_CMD_NOT_DEFINED; - data_file_name= NULL; - undo_file_name= NULL; - redo_file_name= NULL; - extent_size= 1024*1024; //Default 1 MByte - undo_buffer_size= 8*1024*1024; //Default 8 MByte - redo_buffer_size= 8*1024*1024; //Default 8 MByte - initial_size= 128*1024*1024; //Default 128 MByte - autoextend_size= 0; //No autoextension as default - max_size= 0; //Max size == initial size => no extension - storage_engine= NULL; - nodegroup_id= UNDEF_NODEGROUP; - wait_until_completed= TRUE; - ts_comment= NULL; - ts_access_mode= TS_NOT_DEFINED; - } -}; - /* The handler for a table type. Will be included in the TABLE structure */ struct TABLE; @@ -1150,6 +1075,7 @@ typedef bool (stat_print_fn)(THD *thd, const char *type, size_t type_len, enum ha_stat_type { HA_ENGINE_STATUS, HA_ENGINE_LOGS, HA_ENGINE_MUTEX }; extern MYSQL_PLUGIN_IMPORT st_plugin_int *hton2plugin[MAX_HA]; +struct handlerton; #define view_pseudo_hton ((handlerton *)1) /* @@ -1515,8 +1441,7 @@ struct handlerton bool (*show_status)(handlerton *hton, THD *thd, stat_print_fn *print, enum ha_stat_type stat); uint (*partition_flags)(); alter_table_operations (*alter_table_flags)(alter_table_operations flags); - int (*alter_tablespace)(handlerton *hton, THD *thd, st_alter_tablespace *ts_info); - int (*fill_is_table)(handlerton *hton, THD *thd, TABLE_LIST *tables, + int (*fill_is_table)(handlerton *hton, THD *thd, TABLE_LIST *tables, class Item *cond, enum enum_schema_tables); uint32 flags; /* global handler flags */ diff --git a/sql/hostname.cc b/sql/hostname.cc index e23560c7b8c..18164b32517 100644 --- a/sql/hostname.cc +++ b/sql/hostname.cc @@ -190,7 +190,7 @@ Host_entry *hostname_cache_first() static inline Host_entry *hostname_cache_search(const char *ip_key) { - return hostname_cache->search((uchar *) ip_key, 0); + return hostname_cache->search((uchar *) ip_key, HOST_ENTRY_KEY_SIZE); } static void add_hostname_impl(const char *ip_key, const char *hostname, diff --git a/sql/item.cc b/sql/item.cc index c10eb877179..764c7a3f530 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -56,8 +56,8 @@ const char *item_empty_name=""; const char *item_used_name= "\0"; static int save_field_in_field(Field *, bool *, Field *, bool); -const Item_bool_static Item_false("FALSE", 0); -const Item_bool_static Item_true("TRUE", 1); +Item_bool_static *Item_false; +Item_bool_static *Item_true; /** Compare two Items for List<Item>::add_unique() @@ -446,7 +446,7 @@ Item::Item(THD *thd): Item::Item(): name(null_clex_str), orig_name(0), is_expensive_cache(-1) { - DBUG_ASSERT(my_progname == NULL); // before main() + DBUG_ASSERT(!mysqld_server_started); // Created early base_flags= item_base_t::FIXED; with_flags= item_with_t::NONE; null_value= 0; @@ -4512,6 +4512,22 @@ bool Item_param::is_evaluable_expression() const } +bool Item_param::check_assignability_to(const Field *to, bool ignore) const +{ + switch (state) { + case SHORT_DATA_VALUE: + case LONG_DATA_VALUE: + case NULL_VALUE: + return to->check_assignability_from(type_handler(), ignore); + case NO_VALUE: + case IGNORE_VALUE: + case DEFAULT_VALUE: + break; + } + return false; +} + + bool Item_param::can_return_value() const { // There's no "default". See comments in Item_param::save_in_field(). @@ -9902,9 +9918,11 @@ void Item_trigger_field::set_required_privilege(bool rw) bool Item_trigger_field::set_value(THD *thd, sp_rcontext * /*ctx*/, Item **it) { - Item *item= thd->sp_prepare_func_item(it); + if (fix_fields_if_needed(thd, NULL)) + return true; - if (!item || fix_fields_if_needed(thd, NULL)) + Item *item= thd->sp_fix_func_item_for_assignment(field, it); + if (!item) return true; if (field->vers_sys_field()) return false; diff --git a/sql/item.h b/sql/item.h index 486ca098dda..07a3d476699 100644 --- a/sql/item.h +++ b/sql/item.h @@ -772,8 +772,8 @@ enum class item_base_t : item_flags_t FIXED= (1<<2), // Was fixed with fix_fields(). IS_EXPLICIT_NAME= (1<<3), // The name of this Item was set by the user // (or was auto generated otherwise) - IS_IN_WITH_CYCLE= (1<<4) // This item is in CYCLE clause - // of WITH. + IS_IN_WITH_CYCLE= (1<<4), // This item is in CYCLE clause of WITH. + AT_TOP_LEVEL= (1<<5) // At top (AND) level of item tree }; @@ -1327,6 +1327,25 @@ public: { set_maybe_null(maybe_null_arg); } + /* + Mark the item that it is a top level item, or part of a top level AND item, + for WHERE and ON clauses: + Example: ... WHERE a=5 AND b=6; Both a=5 and b=6 are top level items + + This is used to indicate that there is no distinction between if the + value of the item is FALSE or NULL.. + This enables Item_cond_and and subquery related items to do special + "top level" optimizations. + */ + virtual void top_level_item() + { + base_flags|= item_base_t::AT_TOP_LEVEL; + } + /* + Return TRUE if this item of top WHERE level (AND/OR) + */ + bool is_top_level_item() const + { return (bool) (base_flags & item_base_t::AT_TOP_LEVEL); } void set_typelib(const TYPELIB *typelib) override { @@ -1833,6 +1852,16 @@ public: */ virtual bool is_evaluable_expression() const { return true; } + virtual bool check_assignability_to(const Field *to, bool ignore) const + { + /* + "this" must be neither DEFAULT/IGNORE, + nor Item_param bound to DEFAULT/IGNORE. + */ + DBUG_ASSERT(is_evaluable_expression()); + return to->check_assignability_from(type_handler(), ignore); + } + /** * Check whether the item is a parameter ('?') of stored routine. * Default implementation returns false. Method is overridden in the class @@ -2049,25 +2078,6 @@ public: { return type_handler()->Item_update_null_value(this); } - - /* - Inform the item that there will be no distinction between its result - being FALSE or NULL. - - NOTE - This function will be called for eg. Items that are top-level AND-parts - of the WHERE clause. Items implementing this function (currently - Item_cond_and and subquery-related item) enable special optimizations - when they are "top level". - */ - virtual void top_level_item() {} - /* - Return TRUE if it is item of top WHERE level (AND/OR) and it is - important, return FALSE if it not important (we can not use to simplify - calculations) or not top level - */ - virtual bool is_top_level_item() const - { return FALSE; /* not important */} /* return IN/ALL/ANY subquery or NULL */ @@ -2677,18 +2687,27 @@ public: void register_in(THD *thd); bool depends_only_on(table_map view_map) - { return marker & MARKER_FULL_EXTRACTION; } - int get_extraction_flag() - { return marker & MARKER_EXTRACTION_MASK; } + { return get_extraction_flag() & MARKER_FULL_EXTRACTION; } + int get_extraction_flag() const + { + if (basic_const_item()) + return MARKER_FULL_EXTRACTION; + else + return marker & MARKER_EXTRACTION_MASK; + } void set_extraction_flag(int16 flags) { - marker &= ~MARKER_EXTRACTION_MASK; - marker|= flags; + if (!basic_const_item()) + { + marker= marker & ~MARKER_EXTRACTION_MASK; + marker|= flags; + } } void clear_extraction_flag() { - marker &= ~MARKER_EXTRACTION_MASK; - } + if (!basic_const_item()) + marker= marker & ~MARKER_EXTRACTION_MASK; + } void check_pushable_cond(Pushdown_checker excl_dep_func, uchar *arg); bool pushable_cond_checker_for_derived(uchar *arg) { @@ -4124,6 +4143,7 @@ class Item_param :public Item_basic_value, const String *value_query_val_str(THD *thd, String* str) const; Item *value_clone_item(THD *thd); bool is_evaluable_expression() const override; + bool check_assignability_to(const Field *field, bool ignore) const override; bool can_return_value() const; public: @@ -4470,11 +4490,16 @@ public: Item_bool_static(const char *str_arg, longlong i): Item_bool(str_arg, i) {}; + /* Don't mark static items as top level item */ + virtual void top_level_item() override {} void set_join_tab_idx(uint8 join_tab_idx_arg) override { DBUG_ASSERT(0); } + + void cleanup() override {} }; -extern const Item_bool_static Item_false, Item_true; +/* The following variablese are stored in a read only segment */ +extern Item_bool_static *Item_false, *Item_true; class Item_uint :public Item_int { @@ -6808,6 +6833,10 @@ public: { str->append(STRING_WITH_LEN("default")); } + bool check_assignability_to(const Field *to, bool ignore) const override + { + return false; + } int save_in_field(Field *field_arg, bool) override { return field_arg->save_in_field_default_value(false); @@ -6841,6 +6870,10 @@ public: { str->append(STRING_WITH_LEN("ignore")); } + bool check_assignability_to(const Field *to, bool ignore) const override + { + return false; + } int save_in_field(Field *field_arg, bool) override { return field_arg->save_in_field_ignore_value(false); diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 4c2f30c1c26..79c6eed930d 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -1079,14 +1079,14 @@ int Arg_comparator::compare_row() // NULL was compared switch (((Item_func*)owner)->functype()) { case Item_func::NE_FUNC: - break; // NE never aborts on NULL even if abort_on_null is set + break; // NE never aborts on NULL case Item_func::LT_FUNC: case Item_func::LE_FUNC: case Item_func::GT_FUNC: case Item_func::GE_FUNC: return -1; // <, <=, > and >= always fail on NULL case Item_func::EQ_FUNC: - if (((Item_func_eq*)owner)->abort_on_null) + if (owner->is_top_level_item()) return -1; // We do not need correct NULL returning break; default: @@ -1203,12 +1203,6 @@ longlong Item_func_truth::val_int() } -bool Item_in_optimizer::is_top_level_item() const -{ - return args[1]->is_top_level_item(); -} - - void Item_in_optimizer::fix_after_pullout(st_select_lex *new_parent, Item **ref, bool merge) { @@ -1406,7 +1400,8 @@ bool Item_in_optimizer::fix_fields(THD *thd, Item **ref) } base_flags|= (item_base_t::FIXED | - (args[1]->base_flags & item_base_t::MAYBE_NULL)); + (args[1]->base_flags & (item_base_t::MAYBE_NULL | + item_base_t::AT_TOP_LEVEL))); with_flags|= (item_with_t::SUBQUERY | args[1]->with_flags | (args[0]->with_flags & @@ -2083,7 +2078,7 @@ bool Item_func_between::eval_not_null_tables(void *opt_arg) return 1; /* not_null_tables_cache == union(T1(e),T1(e1),T1(e2)) */ - if (pred_level && !negated) + if (is_top_level_item() && !negated) return 0; /* not_null_tables_cache == union(T1(e), intersection(T1(e1),T1(e2))) */ @@ -2485,6 +2480,10 @@ bool Item_func_if::fix_fields(THD *thd, Item **ref) { DBUG_ASSERT(fixed() == 0); + /* + Mark that we don't care if args[0] is NULL or FALSE, we regard both cases as + false. + */ args[0]->top_level_item(); if (Item_func::fix_fields(thd, ref)) @@ -4410,7 +4409,7 @@ Item_func_in::eval_not_null_tables(void *opt_arg) return 1; /* not_null_tables_cache == union(T1(e),union(T1(ei))) */ - if (pred_level && negated) + if (is_top_level_item() && negated) return 0; /* not_null_tables_cache = union(T1(e),intersection(T1(ei))) */ @@ -4865,9 +4864,10 @@ bool Item_func_bit_and::fix_length_and_dec() Item_cond::Item_cond(THD *thd, Item_cond *item) :Item_bool_func(thd, item), - abort_on_null(item->abort_on_null), and_tables_cache(item->and_tables_cache) { + base_flags|= (item->base_flags & item_base_t::AT_TOP_LEVEL); + /* item->list will be copied by copy_andor_arguments() call */ @@ -4875,7 +4875,7 @@ Item_cond::Item_cond(THD *thd, Item_cond *item) Item_cond::Item_cond(THD *thd, Item *i1, Item *i2): - Item_bool_func(thd), abort_on_null(0) + Item_bool_func(thd) { list.push_back(i1, thd->mem_root); list.push_back(i2, thd->mem_root); @@ -4923,7 +4923,7 @@ Item_cond::fix_fields(THD *thd, Item **ref) { merge_sub_condition(li); item= *li.ref(); - if (abort_on_null) + if (is_top_level_item()) item->top_level_item(); /* @@ -4952,7 +4952,7 @@ Item_cond::fix_fields(THD *thd, Item **ref) if (item->can_eval_in_optimize() && !item->with_sp_var() && !cond_has_datetime_is_null(item)) { - if (item->eval_const_cond() == is_and_cond && top_level()) + if (item->eval_const_cond() == is_and_cond && is_top_level_item()) { /* a. This is "... AND true_cond AND ..." @@ -5058,7 +5058,7 @@ Item_cond::eval_not_null_tables(void *opt_arg) if (item->can_eval_in_optimize() && !item->with_sp_var() && !cond_has_datetime_is_null(item)) { - if (item->eval_const_cond() == is_and_cond && top_level()) + if (item->eval_const_cond() == is_and_cond && is_top_level_item()) { /* a. This is "... AND true_cond AND ..." @@ -5501,17 +5501,18 @@ void Item_cond_and::mark_as_condition_AND_part(TABLE_LIST *embedding) Evaluation of AND(expr, expr, expr ...). @note - abort_if_null is set for AND expressions for which we don't care if the - result is NULL or 0. This is set for: + There are AND expressions for which we don't care if the + result is NULL or 0. This is the case for: - WHERE clause - HAVING clause - IF(expression) + For these we mark them as "top_level_items" @retval 1 If all expressions are true @retval - 0 If all expressions are false or if we find a NULL expression and - 'abort_on_null' is set. + 0 If any of the expressions are false or if we find a NULL expression and + this is a top_level_item. @retval NULL if all expression are either 1 or NULL */ @@ -5527,8 +5528,8 @@ longlong Item_cond_and::val_int() { if (!item->val_bool()) { - if (abort_on_null || !(null_value= item->null_value)) - return 0; // return FALSE + if (is_top_level_item() || !(null_value= item->null_value)) + return 0; } } return null_value ? 0 : 1; diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index f11b668d546..c0cbcdcc081 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -408,7 +408,6 @@ public: void set_join_tab_idx(uint8 join_tab_idx_arg) override { args[1]->set_join_tab_idx(join_tab_idx_arg); } void get_cache_parameters(List<Item> ¶meters) override; - bool is_top_level_item() const override; bool eval_not_null_tables(void *opt_arg) override; bool find_not_null_fields(table_map allowed) override; void fix_after_pullout(st_select_lex *new_parent, Item **ref, @@ -628,12 +627,8 @@ public: class Item_func_not :public Item_bool_func { - bool abort_on_null; public: - Item_func_not(THD *thd, Item *a): - Item_bool_func(thd, a), abort_on_null(FALSE) {} - void top_level_item() override { abort_on_null= 1; } - bool is_top_level_item() const override { return abort_on_null; } + Item_func_not(THD *thd, Item *a): Item_bool_func(thd, a) {} longlong val_int() override; enum Functype functype() const override { return NOT_FUNC; } LEX_CSTRING func_name_cstring() const override @@ -754,11 +749,10 @@ public: class Item_func_eq :public Item_bool_rowready_func2 { - bool abort_on_null; public: Item_func_eq(THD *thd, Item *a, Item *b): Item_bool_rowready_func2(thd, a, b), - abort_on_null(false), in_equality_no(UINT_MAX) + in_equality_no(UINT_MAX) {} longlong val_int() override; enum Functype functype() const override { return EQ_FUNC; } @@ -769,7 +763,6 @@ public: static LEX_CSTRING name= {STRING_WITH_LEN("=") }; return name; } - void top_level_item() override { abort_on_null= true; } Item *negated_item(THD *thd) override; COND *build_equal_items(THD *thd, COND_EQUAL *inherited, bool link_item_fields, @@ -955,15 +948,12 @@ protected: DTCollation cmp_collation; public: bool negated; /* <=> the item represents NOT <func> */ - bool pred_level; /* <=> [NOT] <func> is used on a predicate level */ public: Item_func_opt_neg(THD *thd, Item *a, Item *b, Item *c): - Item_bool_func(thd, a, b, c), negated(0), pred_level(0) {} + Item_bool_func(thd, a, b, c), negated(0) {} Item_func_opt_neg(THD *thd, List<Item> &list): - Item_bool_func(thd, list), negated(0), pred_level(0) {} + Item_bool_func(thd, list), negated(0) {} public: - void top_level_item() override { pred_level= 1; } - bool is_top_level_item() const override { return pred_level; } Item *neg_transformer(THD *thd) override { negated= !negated; @@ -2802,11 +2792,9 @@ public: class Item_func_isnotnull :public Item_func_null_predicate { - bool abort_on_null; public: Item_func_isnotnull(THD *thd, Item *a): - Item_func_null_predicate(thd, a), abort_on_null(0) - { } + Item_func_null_predicate(thd, a) {} longlong val_int() override; enum Functype functype() const override { return ISNOTNULL_FUNC; } LEX_CSTRING func_name_cstring() const override @@ -2816,10 +2804,9 @@ public: } enum precedence precedence() const override { return CMP_PRECEDENCE; } table_map not_null_tables() const override - { return abort_on_null ? not_null_tables_cache : 0; } + { return is_top_level_item() ? not_null_tables_cache : 0; } Item *neg_transformer(THD *thd) override; void print(String *str, enum_query_type query_type) override; - void top_level_item() override { abort_on_null=1; } Item *get_copy(THD *thd) override { return get_item_copy<Item_func_isnotnull>(thd, this); } }; @@ -3130,17 +3117,19 @@ class Item_cond :public Item_bool_func { protected: List<Item> list; - bool abort_on_null; table_map and_tables_cache; public: - /* Item_cond() is only used to create top level items */ - Item_cond(THD *thd): Item_bool_func(thd), abort_on_null(1) - { const_item_cache=0; } + Item_cond(THD *thd): Item_bool_func(thd) + { + /* Item_cond() is only used to create top level items */ + top_level_item(); + const_item_cache=0; + } Item_cond(THD *thd, Item *i1, Item *i2); Item_cond(THD *thd, Item_cond *item); Item_cond(THD *thd, List<Item> &nlist): - Item_bool_func(thd), list(nlist), abort_on_null(0) {} + Item_bool_func(thd), list(nlist) {} bool add(Item *item, MEM_ROOT *root) { DBUG_ASSERT(item); @@ -3187,8 +3176,6 @@ public: List<Item> &fields, uint flags) override; friend int setup_conds(THD *thd, TABLE_LIST *tables, TABLE_LIST *leaves, COND **conds); - void top_level_item() override { abort_on_null=1; } - bool top_level() { return abort_on_null; } void copy_andor_arguments(THD *thd, Item_cond *item); bool walk(Item_processor processor, bool walk_subquery, void *arg) override; Item *do_transform(THD *thd, Item_transformer transformer, uchar *arg, @@ -3567,7 +3554,7 @@ public: } enum precedence precedence() const override { return AND_PRECEDENCE; } table_map not_null_tables() const override - { return abort_on_null ? not_null_tables_cache: and_tables_cache; } + { return is_top_level_item() ? not_null_tables_cache: and_tables_cache; } Item *copy_andor_structure(THD *thd) override; Item *neg_transformer(THD *thd) override; void mark_as_condition_AND_part(TABLE_LIST *embedding) override; diff --git a/sql/item_create.cc b/sql/item_create.cc index dcc64ef10e8..a8ca7d5cff8 100644 --- a/sql/item_create.cc +++ b/sql/item_create.cc @@ -519,10 +519,11 @@ protected: }; -class Create_func_crc32 : public Create_func_arg1 +class Create_func_crc32 : public Create_native_func { public: - virtual Item *create_1_arg(THD *thd, Item *arg1); + Item *create_native(THD *thd, const LEX_CSTRING *, List<Item> *item_list) + override; static Create_func_crc32 s_singleton; @@ -532,6 +533,20 @@ protected: }; +class Create_func_crc32c : public Create_native_func +{ +public: + Item *create_native(THD *thd, const LEX_CSTRING *, List<Item> *item_list) + override; + + static Create_func_crc32c s_singleton; + +protected: + Create_func_crc32c() = default; + virtual ~Create_func_crc32c() = default; +}; + + class Create_func_datediff : public Create_func_arg2 { public: @@ -918,6 +933,32 @@ protected: }; +class Create_func_json_normalize : public Create_func_arg1 +{ +public: + virtual Item *create_1_arg(THD *thd, Item *arg1); + + static Create_func_json_normalize s_singleton; + +protected: + Create_func_json_normalize() = default; + virtual ~Create_func_json_normalize() = default; +}; + + +class Create_func_json_equals : public Create_func_arg2 +{ +public: + virtual Item *create_2_arg(THD *thd, Item *arg1, Item *arg2); + + static Create_func_json_equals s_singleton; + +protected: + Create_func_json_equals() = default; + virtual ~Create_func_json_equals() = default; +}; + + class Create_func_json_exists : public Create_func_arg2 { public: @@ -1639,6 +1680,15 @@ protected: virtual ~Create_func_name_const() = default; }; +class Create_func_natural_sort_key : public Create_func_arg1 +{ +public: + virtual Item *create_1_arg(THD *thd, Item *arg1) override; + static Create_func_natural_sort_key s_singleton; +protected: + Create_func_natural_sort_key() = default; + virtual ~Create_func_natural_sort_key() = default; +}; class Create_func_nullif : public Create_func_arg2 { @@ -1944,6 +1994,16 @@ protected: virtual ~Create_func_sec_to_time() = default; }; +class Create_func_sformat : public Create_native_func +{ +public: + Item *create_native(THD *thd, const LEX_CSTRING *name, List<Item> *item_list) + override; + static Create_func_sformat s_singleton; +protected: + Create_func_sformat() = default; + virtual ~Create_func_sformat() = default; +}; class Create_func_sha : public Create_func_arg1 { @@ -2189,8 +2249,8 @@ public: static Create_func_to_char s_singleton; protected: - Create_func_to_char() {} - virtual ~Create_func_to_char() {} + Create_func_to_char() = default; + virtual ~Create_func_to_char() = default; }; @@ -2285,30 +2345,6 @@ protected: }; -class Create_func_uuid : public Create_func_arg0 -{ -public: - virtual Item *create_builder(THD *thd); - - static Create_func_uuid s_singleton; - -protected: - Create_func_uuid() = default; - virtual ~Create_func_uuid() = default; -}; - -class Create_func_sys_guid : public Create_func_arg0 -{ -public: - virtual Item *create_builder(THD *thd); - - static Create_func_sys_guid s_singleton; - -protected: - Create_func_sys_guid() {} - virtual ~Create_func_sys_guid() {} -}; - class Create_func_uuid_short : public Create_func_arg0 { public: @@ -3158,11 +3194,55 @@ Create_func_cot::create_1_arg(THD *thd, Item *arg1) Create_func_crc32 Create_func_crc32::s_singleton; Item* -Create_func_crc32::create_1_arg(THD *thd, Item *arg1) +Create_func_crc32::create_native(THD *thd, const LEX_CSTRING *name, + List<Item> *item_list) +{ + int argc= item_list ? item_list->elements : 0; + + if (unlikely(argc != 1 && argc != 2)) + { + my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name->str); + return nullptr; + } + + Item *arg1= item_list->pop(), *arg2= argc < 2 ? nullptr : item_list->pop(); + + /* This was checked in Create_native_func::create_func() */ + DBUG_ASSERT(!arg1->is_explicit_name()); + DBUG_ASSERT(!arg2 || !arg2->is_explicit_name()); + + return arg2 + ? new (thd->mem_root) Item_func_crc32(thd, false, arg1, arg2) + : new (thd->mem_root) Item_func_crc32(thd, false, arg1); +} + + +Create_func_crc32c Create_func_crc32c::s_singleton; + +Item* +Create_func_crc32c::create_native(THD *thd, const LEX_CSTRING *name, + List<Item> *item_list) { - return new (thd->mem_root) Item_func_crc32(thd, arg1); + int argc= item_list ? item_list->elements : 0; + + if (unlikely(argc != 1 && argc != 2)) + { + my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name->str); + return nullptr; + } + + Item *arg1= item_list->pop(), *arg2= argc < 2 ? nullptr : item_list->pop(); + + /* This was checked in Create_native_func::create_func() */ + DBUG_ASSERT(!arg1->is_explicit_name()); + DBUG_ASSERT(!arg2 || !arg2->is_explicit_name()); + + return arg2 + ? new (thd->mem_root) Item_func_crc32(thd, true, arg1, arg2) + : new (thd->mem_root) Item_func_crc32(thd, true, arg1); } + Create_func_datediff Create_func_datediff::s_singleton; Item* @@ -3664,6 +3744,25 @@ Create_func_isnull::create_1_arg(THD *thd, Item *arg1) return new (thd->mem_root) Item_func_isnull(thd, arg1); } +Create_func_json_normalize Create_func_json_normalize::s_singleton; + +Item* +Create_func_json_normalize::create_1_arg(THD *thd, Item *arg1) +{ + status_var_increment(thd->status_var.feature_json); + return new (thd->mem_root) Item_func_json_normalize(thd, arg1); +} + + +Create_func_json_equals Create_func_json_equals::s_singleton; + +Item* +Create_func_json_equals::create_2_arg(THD *thd, Item *arg1, Item *arg2) +{ + status_var_increment(thd->status_var.feature_json); + return new (thd->mem_root) Item_func_json_equals(thd, arg1, arg2); +} + Create_func_json_exists Create_func_json_exists::s_singleton; @@ -4665,6 +4764,12 @@ Create_func_md5::create_1_arg(THD *thd, Item *arg1) return new (thd->mem_root) Item_func_md5(thd, arg1); } +Create_func_natural_sort_key Create_func_natural_sort_key::s_singleton; + +Item *Create_func_natural_sort_key::create_1_arg(THD *thd, Item* arg1) +{ + return new (thd->mem_root) Item_func_natural_sort_key(thd, arg1); +} Create_func_monthname Create_func_monthname::s_singleton; @@ -5047,6 +5152,26 @@ Create_func_sec_to_time::create_1_arg(THD *thd, Item *arg1) return new (thd->mem_root) Item_func_sec_to_time(thd, arg1); } +Create_func_sformat Create_func_sformat::s_singleton; + +Item* +Create_func_sformat::create_native(THD *thd, const LEX_CSTRING *name, + List<Item> *item_list) +{ + int arg_count= 0; + + if (item_list != NULL) + arg_count= item_list->elements; + + if (unlikely(arg_count < 1)) + { + my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name->str); + return NULL; + } + + return new (thd->mem_root) Item_func_sformat(thd, *item_list); +} + Create_func_sha Create_func_sha::s_singleton; @@ -5365,29 +5490,6 @@ Create_func_unix_timestamp::create_native(THD *thd, const LEX_CSTRING *name, } -Create_func_uuid Create_func_uuid::s_singleton; - -Item* -Create_func_uuid::create_builder(THD *thd) -{ - DBUG_ENTER("Create_func_uuid::create"); - thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); - thd->lex->safe_to_cache_query= 0; - DBUG_RETURN(new (thd->mem_root) Item_func_uuid(thd, 0)); -} - -Create_func_sys_guid Create_func_sys_guid::s_singleton; - -Item* -Create_func_sys_guid::create_builder(THD *thd) -{ - DBUG_ENTER("Create_func_sys_guid::create"); - thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); - thd->lex->safe_to_cache_query= 0; - DBUG_RETURN(new (thd->mem_root) Item_func_uuid(thd, 1)); -} - - Create_func_uuid_short Create_func_uuid_short::s_singleton; Item* @@ -5594,6 +5696,7 @@ Native_func_registry func_array[] = { { STRING_WITH_LEN("COS") }, BUILDER(Create_func_cos)}, { { STRING_WITH_LEN("COT") }, BUILDER(Create_func_cot)}, { { STRING_WITH_LEN("CRC32") }, BUILDER(Create_func_crc32)}, + { { STRING_WITH_LEN("CRC32C") }, BUILDER(Create_func_crc32c)}, { { STRING_WITH_LEN("DATEDIFF") }, BUILDER(Create_func_datediff)}, { { STRING_WITH_LEN("DAYNAME") }, BUILDER(Create_func_dayname)}, { { STRING_WITH_LEN("DAYOFMONTH") }, BUILDER(Create_func_dayofmonth)}, @@ -5635,6 +5738,7 @@ Native_func_registry func_array[] = { { STRING_WITH_LEN("JSON_DEPTH") }, BUILDER(Create_func_json_depth)}, { { STRING_WITH_LEN("JSON_DETAILED") }, BUILDER(Create_func_json_detailed)}, { { STRING_WITH_LEN("JSON_PRETTY") }, BUILDER(Create_func_json_detailed)}, + { { STRING_WITH_LEN("JSON_EQUALS") }, BUILDER(Create_func_json_equals)}, { { STRING_WITH_LEN("JSON_EXISTS") }, BUILDER(Create_func_json_exists)}, { { STRING_WITH_LEN("JSON_EXTRACT") }, BUILDER(Create_func_json_extract)}, { { STRING_WITH_LEN("JSON_INSERT") }, BUILDER(Create_func_json_insert)}, @@ -5644,6 +5748,7 @@ Native_func_registry func_array[] = { { STRING_WITH_LEN("JSON_MERGE") }, BUILDER(Create_func_json_merge)}, { { STRING_WITH_LEN("JSON_MERGE_PATCH") }, BUILDER(Create_func_json_merge_patch)}, { { STRING_WITH_LEN("JSON_MERGE_PRESERVE") }, BUILDER(Create_func_json_merge)}, + { { STRING_WITH_LEN("JSON_NORMALIZE") }, BUILDER(Create_func_json_normalize)}, { { STRING_WITH_LEN("JSON_QUERY") }, BUILDER(Create_func_json_query)}, { { STRING_WITH_LEN("JSON_QUOTE") }, BUILDER(Create_func_json_quote)}, { { STRING_WITH_LEN("JSON_OBJECT") }, BUILDER(Create_func_json_object)}, @@ -5684,6 +5789,7 @@ Native_func_registry func_array[] = { { STRING_WITH_LEN("MD5") }, BUILDER(Create_func_md5)}, { { STRING_WITH_LEN("MONTHNAME") }, BUILDER(Create_func_monthname)}, { { STRING_WITH_LEN("NAME_CONST") }, BUILDER(Create_func_name_const)}, + { {STRING_WITH_LEN("NATURAL_SORT_KEY")}, BUILDER(Create_func_natural_sort_key)}, { { STRING_WITH_LEN("NVL") }, BUILDER(Create_func_ifnull)}, { { STRING_WITH_LEN("NVL2") }, BUILDER(Create_func_nvl2)}, { { STRING_WITH_LEN("NULLIF") }, BUILDER(Create_func_nullif)}, @@ -5713,6 +5819,7 @@ Native_func_registry func_array[] = { { STRING_WITH_LEN("RTRIM") }, BUILDER(Create_func_rtrim)}, { { STRING_WITH_LEN("RTRIM_ORACLE") }, BUILDER(Create_func_rtrim_oracle)}, { { STRING_WITH_LEN("SEC_TO_TIME") }, BUILDER(Create_func_sec_to_time)}, + { { STRING_WITH_LEN("SFORMAT") }, BUILDER(Create_func_sformat)}, { { STRING_WITH_LEN("SHA") }, BUILDER(Create_func_sha)}, { { STRING_WITH_LEN("SHA1") }, BUILDER(Create_func_sha)}, { { STRING_WITH_LEN("SHA2") }, BUILDER(Create_func_sha2)}, @@ -5728,7 +5835,6 @@ Native_func_registry func_array[] = BUILDER(Create_func_substr_oracle)}, { { STRING_WITH_LEN("SUBSTRING_INDEX") }, BUILDER(Create_func_substr_index)}, { { STRING_WITH_LEN("SUBTIME") }, BUILDER(Create_func_subtime)}, - { { STRING_WITH_LEN("SYS_GUID") }, BUILDER(Create_func_sys_guid)}, { { STRING_WITH_LEN("TAN") }, BUILDER(Create_func_tan)}, { { STRING_WITH_LEN("TIMEDIFF") }, BUILDER(Create_func_timediff)}, { { STRING_WITH_LEN("TIME_FORMAT") }, BUILDER(Create_func_time_format)}, @@ -5744,7 +5850,6 @@ Native_func_registry func_array[] = { { STRING_WITH_LEN("UNIX_TIMESTAMP") }, BUILDER(Create_func_unix_timestamp)}, { { STRING_WITH_LEN("UPDATEXML") }, BUILDER(Create_func_xml_update)}, { { STRING_WITH_LEN("UPPER") }, BUILDER(Create_func_ucase)}, - { { STRING_WITH_LEN("UUID") }, BUILDER(Create_func_uuid)}, { { STRING_WITH_LEN("UUID_SHORT") }, BUILDER(Create_func_uuid_short)}, { { STRING_WITH_LEN("VERSION") }, BUILDER(Create_func_version)}, { { STRING_WITH_LEN("WEEKDAY") }, BUILDER(Create_func_weekday)}, diff --git a/sql/item_func.cc b/sql/item_func.cc index e3f2eb500e0..0f7b26a678d 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -1007,7 +1007,8 @@ err: push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE, ER_THD(thd, ER_WARN_DATA_OUT_OF_RANGE), - name.str, 1L); + name.str, + thd->get_stmt_da()->current_row_for_warning()); return dec; } diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc index b8a743c2b6f..7e331e287f2 100644 --- a/sql/item_geofunc.cc +++ b/sql/item_geofunc.cc @@ -1083,6 +1083,7 @@ Item_func_spatial_rel::get_mm_leaf(RANGE_OPT_PARAM *param, DBUG_RETURN(0); // out of memory field->get_key_image(str, key_part->length, key_part->image_type); SEL_ARG *tree; + if (!(tree= new (param->mem_root) SEL_ARG(field, str, str))) DBUG_RETURN(0); // out of memory diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 3425f620dda..d32d2d30d4a 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -545,6 +545,66 @@ longlong Item_func_json_valid::val_int() } +bool Item_func_json_equals::fix_length_and_dec() +{ + if (Item_bool_func::fix_length_and_dec()) + return TRUE; + set_maybe_null(); + return FALSE; +} + + +longlong Item_func_json_equals::val_int() +{ + longlong result= 0; + + String a_tmp, b_tmp; + + String *a= args[0]->val_json(&a_tmp); + String *b= args[1]->val_json(&b_tmp); + + DYNAMIC_STRING a_res; + if (init_dynamic_string(&a_res, NULL, 0, 0)) + { + null_value= 1; + return 1; + } + + DYNAMIC_STRING b_res; + if (init_dynamic_string(&b_res, NULL, 0, 0)) + { + dynstr_free(&a_res); + null_value= 1; + return 1; + } + + if ((null_value= args[0]->null_value || args[1]->null_value)) + { + null_value= 1; + goto end; + } + + if (json_normalize(&a_res, a->ptr(), a->length(), a->charset())) + { + null_value= 1; + goto end; + } + + if (json_normalize(&b_res, b->ptr(), b->length(), b->charset())) + { + null_value= 1; + goto end; + } + + result= strcmp(a_res.str, b_res.str) ? 0 : 1; + +end: + dynstr_free(&b_res); + dynstr_free(&a_res); + return result; +} + + bool Item_func_json_exists::fix_length_and_dec() { if (Item_bool_func::fix_length_and_dec()) @@ -1134,7 +1194,7 @@ my_decimal *Item_func_json_extract::val_decimal(my_decimal *to) case JSON_VALUE_OBJECT: case JSON_VALUE_ARRAY: case JSON_VALUE_FALSE: - case JSON_VALUE_UNINITALIZED: + case JSON_VALUE_UNINITIALIZED: case JSON_VALUE_NULL: int2my_decimal(E_DEC_FATAL_ERROR, 0, false/*unsigned_flag*/, to); return to; @@ -2917,7 +2977,7 @@ longlong Item_func_json_depth::val_int() bool Item_func_json_type::fix_length_and_dec() { collation.set(&my_charset_utf8mb3_general_ci); - max_length= 12; + max_length= 12 * collation.collation->mbmaxlen; set_maybe_null(); return FALSE; } @@ -4104,3 +4164,48 @@ String* Item_func_json_objectagg::val_str(String* str) } +String *Item_func_json_normalize::val_str(String *buf) +{ + String tmp; + String *raw_json= args[0]->val_str(&tmp); + + DYNAMIC_STRING normalized_json; + if (init_dynamic_string(&normalized_json, NULL, 0, 0)) + { + null_value= 1; + return NULL; + } + + null_value= args[0]->null_value; + if (null_value) + goto end; + + if (json_normalize(&normalized_json, + raw_json->ptr(), raw_json->length(), + raw_json->charset())) + { + null_value= 1; + goto end; + } + + buf->length(0); + if (buf->append(normalized_json.str, normalized_json.length)) + { + null_value= 1; + goto end; + } + +end: + dynstr_free(&normalized_json); + return null_value ? NULL : buf; +} + + +bool Item_func_json_normalize::fix_length_and_dec() +{ + collation.set(&my_charset_utf8mb4_bin); + /* 0 becomes 0.0E0, thus one character becomes 5 chars */ + fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 5); + set_maybe_null(); + return FALSE; +} diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 0be796c5825..7c5e0bfa6e0 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -107,6 +107,23 @@ public: }; +class Item_func_json_equals: public Item_bool_func +{ +public: + Item_func_json_equals(THD *thd, Item *a, Item *b): + Item_bool_func(thd, a, b) {} + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("json_equals") }; + return name; + } + bool fix_length_and_dec() override; + Item *get_copy(THD *thd) override + { return get_item_copy<Item_func_json_equals>(thd, this); } + longlong val_int() override; +}; + + class Item_func_json_exists: public Item_bool_func { protected: @@ -440,6 +457,24 @@ public: { return get_item_copy<Item_func_json_merge_patch>(thd, this); } }; + +class Item_func_json_normalize: public Item_json_func +{ +public: + Item_func_json_normalize(THD *thd, Item *a): + Item_json_func(thd, a) {} + String *val_str(String *) override; + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("json_normalize") }; + return name; + } + bool fix_length_and_dec() override; + Item *get_copy(THD *thd) override + { return get_item_copy<Item_func_json_normalize>(thd, this); } +}; + + class Item_func_json_length: public Item_long_func { bool check_arguments() const override diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index f044be35b47..c6f00db6563 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2017, Oracle and/or its affiliates. - Copyright (c) 2009, 2021, MariaDB Corporation. + Copyright (c) 2009, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -55,6 +55,11 @@ C_MODE_END #include <sql_repl.h> #include "sql_statistics.h" +/* fmtlib include (https://fmt.dev/). */ +#define FMT_STATIC_THOUSANDS_SEPARATOR ',' +#define FMT_HEADER_ONLY 1 +#include "fmt/format-inl.h" + size_t username_char_length= USERNAME_CHAR_LENGTH; /* @@ -498,7 +503,7 @@ err: const char *histogram_types[] = - {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", 0}; + {"SINGLE_PREC_HB", "DOUBLE_PREC_HB", "JSON_HB", 0}; static TYPELIB histogram_types_typelib= { array_elements(histogram_types), "histogram_types", @@ -528,6 +533,14 @@ String *Item_func_decode_histogram::val_str(String *str) null_value= 1; return 0; } + + if (type == JSON_HB) + { + // It's a JSON histogram. Return it as-is. + null_value= 0; + return res; + } + if (type == DOUBLE_PREC_HB && res->length() % 2 != 0) res->length(res->length() - 1); // one byte is unused @@ -1303,6 +1316,138 @@ bool Item_func_replace::fix_length_and_dec() return FALSE; } +/* + this is done in the constructor to be in the same memroot as + the item itself +*/ +Item_func_sformat::Item_func_sformat(THD *thd, List<Item> &list) + : Item_str_func(thd, list) +{ + val_arg= new (thd->mem_root) String[arg_count]; +} + + +bool Item_func_sformat::fix_length_and_dec() +{ + if (!val_arg) + return TRUE; + + ulonglong char_length= 0; + + uint flags= MY_COLL_ALLOW_SUPERSET_CONV | + MY_COLL_ALLOW_COERCIBLE_CONV | + MY_COLL_ALLOW_NUMERIC_CONV; + + if (Type_std_attributes::agg_item_collations(collation, func_name_cstring(), + args, arg_count, flags, 1)) + return TRUE; + + DTCollation c= collation; + if (c.collation->mbminlen > 1) + c.collation= &my_charset_utf8mb4_bin; + + for (uint i=0 ; i < arg_count ; i++) + { + if (args[i]->result_type() == STRING_RESULT && + Type_std_attributes::agg_item_set_converter(c, func_name_cstring(), + args+i, 1, flags, 1)) + return TRUE; + } + + char_length= MAX_BLOB_WIDTH; + fix_char_length_ulonglong(char_length); + return FALSE; +} + +/* + allow fmt to take String arguments directly. + Inherit from string_view, so all string formatting works. + but {:p} doesn't, because it's not char*, not a pointer. +*/ +namespace fmt { + template <> struct formatter<String>: formatter<string_view> { + template <typename FormatContext> + auto format(String c, FormatContext& ctx) -> decltype(ctx.out()) { + string_view name = { c.ptr(), c.length() }; + return formatter<string_view>::format(name, ctx); + }; + }; +}; + +/* + SFORMAT(format_string, ...) + This function receives a formatting specification string and N parameters + (N >= 0), and it returns string formatted using the rules the user passed + in the specification. It uses fmtlib (https://fmt.dev/). +*/ +String *Item_func_sformat::val_str(String *res) +{ + DBUG_ASSERT(fixed()); + using ctx= fmt::format_context; + String *fmt_arg= NULL; + String *parg= NULL; + fmt::format_args::format_arg *vargs= NULL; + + null_value= true; + if (!(fmt_arg= args[0]->val_str(res))) + return NULL; + + if (!(vargs= new fmt::format_args::format_arg[arg_count - 1])) + return NULL; + + /* Creates the array of arguments for vformat */ + for (uint carg= 1; carg < arg_count; carg++) + { + switch (args[carg]->result_type()) + { + case INT_RESULT: + vargs[carg-1]= fmt::detail::make_arg<ctx>(args[carg]->val_int()); + break; + case DECIMAL_RESULT: // TODO + case REAL_RESULT: + if (args[carg]->field_type() == MYSQL_TYPE_FLOAT) + vargs[carg-1]= fmt::detail::make_arg<ctx>((float)args[carg]->val_real()); + else + vargs[carg-1]= fmt::detail::make_arg<ctx>(args[carg]->val_real()); + break; + case STRING_RESULT: + if (!(parg= args[carg]->val_str(&val_arg[carg-1]))) + { + delete [] vargs; + return NULL; + } + vargs[carg-1]= fmt::detail::make_arg<ctx>(*parg); + break; + case TIME_RESULT: // TODO + case ROW_RESULT: // TODO + default: + DBUG_ASSERT(0); + delete [] vargs; + return NULL; + } + } + + null_value= false; + /* Create the string output */ + try + { + auto text = fmt::vformat(fmt_arg->c_ptr_safe(), + fmt::format_args(vargs, arg_count-1)); + res->length(0); + res->set_charset(collation.collation); + res->append(text.c_str(), text.size(), fmt_arg->charset()); + } + catch (const fmt::format_error &ex) + { + THD *thd= current_thd; + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + WARN_SFORMAT_ERROR, + ER_THD(thd, WARN_SFORMAT_ERROR), ex.what()); + null_value= true; + } + delete [] vargs; + return null_value ? NULL : res; +} /*********************************************************************/ bool Item_func_regexp_replace::fix_length_and_dec() @@ -4256,14 +4401,32 @@ longlong Item_func_uncompressed_length::val_int() longlong Item_func_crc32::val_int() { DBUG_ASSERT(fixed()); - String *res=args[0]->val_str(&value); + DBUG_ASSERT(arg_count == 1 || arg_count == 2); + String *res; + longlong crc; + if (arg_count > 1) + { + crc= args[0]->val_int(); + null_value= args[0]->null_value; + if (null_value) + return 0; + res= args[1]->val_str(&value); + } + else + { + crc= 0; + null_value= 0; + res= args[0]->val_str(&value); + } + if (!res) { null_value=1; return 0; /* purecov: inspected */ } - null_value=0; - return (longlong) my_checksum(0L, (uchar*)res->ptr(), res->length()); + + return static_cast<longlong> + (ulonglong{crc_func(uint32_t(crc), res->ptr(), res->length())}); } #ifdef HAVE_COMPRESS @@ -4396,26 +4559,6 @@ err: #endif -String *Item_func_uuid::val_str(String *str) -{ - DBUG_ASSERT(fixed()); - uchar guid[MY_UUID_SIZE]; - size_t length= (without_separators ? - MY_UUID_ORACLE_STRING_LENGTH : - MY_UUID_STRING_LENGTH); - - str->alloc(length+1); - str->length(length); - str->set_charset(system_charset_info); - my_uuid(guid); - if (without_separators) - my_uuid2str_oracle(guid, (char *)str->ptr()); - else - my_uuid2str(guid, (char *)str->ptr()); - return str; -} - - Item_func_dyncol_create::Item_func_dyncol_create(THD *thd, List<Item> &args, DYNCALL_CREATE_DEF *dfs): Item_str_func(thd, args), defs(dfs), vals(0), keys_num(NULL), keys_str(NULL), @@ -5321,6 +5464,282 @@ String *Item_temptable_rowid::val_str(String *str) return &str_value; } +/** + Helper routine to encode length prefix + in natsort_encode_numeric_string(). + + The idea is so that bigger input numbers correspond + lexicographically bigger output strings. + + Note, that in real use the number would typically + small, as it only computes variable *length prefixes*. + + @param[in] n - the number + @param[in] s - output string + + @return - length of encoding + + Here is how encoding works + + - n is from 0 to 8 + Output string calculated as '0'+n (range '0' - '8') + + - n is from 9 to 17 + Output calculated as concat('9', '0' + n -9)' + Output range: '90'-'98' + + -n is from 18 to 26 + Output calculated as concat('99', '0' + n -18)' + Output range '990'-'998' + + - n is from 27 to SIZE_T_MAX + Output starts with '999', + then log10(n) is encoded as 2-digit decimal number + then the number itself is added. + Example : for 28 key is concat('999', '01' , '28') + i.e '9990128' + + Key length is 5 + ceil(log10(n)) + + Output range is + (64bit)'9990128' - '9991918446744073709551615' + (32bit)'9990128' - '999094294967295' +*/ + +/* Largest length of encoded string.*/ +static size_t natsort_encode_length_max(size_t n) +{ + return (n < 27) ? n/9+1 : 26; +} + +static void natsort_encode_length(size_t n, String* out) +{ + if (n < 27) + { + if (n >= 9) + out->fill(out->length() + n/9,'9'); + out->append(char(n % 9 + '0')); + return; + } + + size_t log10n= 0; + for (size_t tmp= n / 10; tmp; tmp/= 10) + log10n++; + out->fill(out->length() + 3, '9'); + out->append('0' + (char) (log10n / 10)); + out->append('0' + (char) (log10n % 10)); + out->append_ulonglong(n); +} + +enum class NATSORT_ERR +{ + SUCCESS= 0, + KEY_TOO_LARGE= 1, + ALLOC_ERROR= 2 +}; + +/* + Encode numeric string for natural sorting. + + @param[in] in - start of the numeric string + skipping leading zeros + + @param[in] n_digits - length of the string, + in characters, not counting leading zeros. + + @param[out] out - String to write to. The string should + have enough preallocated space to fit the encoded key. + + @return + NATSORT_ERR::SUCCESS - success + NATSORT_ERR::KEY_TOO_LARGE - out string does not have enough + space left to accomodate the key. + + + The resulting encoding of the numeric string is then + + CONCAT(natsort_encode_length(n_digits), in) +*/ +static NATSORT_ERR natsort_encode_numeric_string(const char *in, + size_t n_digits, + String *out) +{ + DBUG_ASSERT(in); + DBUG_ASSERT(n_digits); + + if (out->length() + natsort_encode_length_max(n_digits - 1) + n_digits > + out->alloced_length()) + return NATSORT_ERR::KEY_TOO_LARGE; + + natsort_encode_length(n_digits - 1, out); + out->append(in, n_digits); + return NATSORT_ERR::SUCCESS; +} + +/* + Calculate max size of the natsort key. + + A digit in string expands to 2 chars length_prefix , and the digit + + With even length L=2N, the largest key corresponds to input string + in form REPEAT(<digit><letter>,N) and the length of a key is + 2N + N = 3N + + With odd input length L=2N+1, largest key is built by appending + a digit at the end, with key length 3N+2 + +*/ +static size_t natsort_max_key_size(size_t input_size) +{ + return input_size + (input_size + 1)/2 ; +} + +/** + Convert a string to natural sort key. + @param[in] in - input string + @param[out] out - output string + @param[in] max_key_size - the maximum size of the output + key, in bytes. + @return NATSORT_ERR::SUCCESS - successful completion + NATSORT_ERR::ALLOC_ERROR - memory allocation error + NATSORT_ERR::KEY_TOO_LARGE - resulting key would exceed max_key_size +*/ +static NATSORT_ERR to_natsort_key(const String *in, String *out, + size_t max_key_size) +{ + size_t n_digits= 0; + size_t n_lead_zeros= 0; + size_t num_start; + size_t reserve_length= std::min( + natsort_max_key_size(in->length()) + MAX_BIGINT_WIDTH + 2, max_key_size); + + out->length(0); + out->set_charset(in->charset()); + + if (out->alloc((uint32) reserve_length)) + return NATSORT_ERR::ALLOC_ERROR; + + for (size_t pos= 0;; pos++) + { + char c= pos < in->length() ? (*in)[pos] : 0; + bool is_digit= (c >= '0' && c <= '9'); + if (!is_digit && (n_digits || n_lead_zeros)) + { + /* Handle end of digits run.*/ + if (!n_digits) + { + /*We only have zeros.*/ + n_lead_zeros--; + num_start= pos - 1; + n_digits= 1; + } + NATSORT_ERR err= natsort_encode_numeric_string( + in->ptr() + num_start, n_digits, out); + if (err != NATSORT_ERR::SUCCESS) + return err; + + /* Reset state.*/ + n_digits= 0; + num_start= size_t(-1); + n_lead_zeros= 0; + } + + if (pos == in->length()) + break; + + if (!is_digit) + { + if (out->length() == max_key_size) + return NATSORT_ERR::KEY_TOO_LARGE; + out->append(c); + } + else if (c == '0' && !n_digits) + n_lead_zeros++; + else if (!n_digits++) + num_start= pos; + } + return NATSORT_ERR::SUCCESS; +} + +String *Item_func_natural_sort_key::val_str(String *out) +{ + String *in= args[0]->val_str(); + if (args[0]->null_value || !in) + { + null_value= true; + return nullptr; + } + NATSORT_ERR err= NATSORT_ERR::SUCCESS; + CHARSET_INFO *cs= in->charset(); + ulong max_allowed_packet= current_thd->variables.max_allowed_packet; + uint errs; + String tmp; + /* + to_natsort_key() only support charsets where digits are represented by + a single byte in range 0x30-0x39. Almost everything is OK, just utf16/32 + won't do. Full ASCII compatibility is not required, so that SJIS and SWE7 + are fine. + */ + if (cs->mbminlen != 1) + { + if (tmp.copy(in, &my_charset_utf8mb4_bin, &errs)) + goto error_exit; + in= &tmp; + } + + err= to_natsort_key(in, out, max_allowed_packet / cs->mbminlen); + + if (err != NATSORT_ERR::SUCCESS) + { + if (err == NATSORT_ERR::KEY_TOO_LARGE) + { + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_WARN_ALLOWED_PACKET_OVERFLOWED, + ER(ER_WARN_ALLOWED_PACKET_OVERFLOWED), func_name(), + max_allowed_packet); + } + goto error_exit; + } + + if (cs->mbminlen != 1) + { + /* output string is now utf8, convert to input charset.*/ + if (tmp.copy(out, cs, &errs) || out->copy(tmp)) + goto error_exit; + } + null_value= false; + return out; + +error_exit: + null_value= true; + return nullptr; +} + +bool Item_func_natural_sort_key::fix_length_and_dec(void) +{ + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return true; + DBUG_ASSERT(collation.collation != NULL); + uint32 max_char_len= + (uint32) natsort_max_key_size(args[0]->max_char_length()); + fix_char_length(max_char_len); + + set_maybe_null(args[0]->maybe_null() || + max_char_len * collation.collation->mbmaxlen > + current_thd->variables.max_allowed_packet); + return false; +} + +/** + Disable use in stored virtual functions. Temporarily(?), until + the encoding is stable. +*/ +bool Item_func_natural_sort_key::check_vcol_func_processor(void *arg) +{ + return mark_unsupported_function(func_name(), "()", arg, + VCOL_NON_DETERMINISTIC); +} + #ifdef WITH_WSREP #include "wsrep_mysqld.h" #include "wsrep_server_state.h" diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 258c0e26bad..86a095a39e6 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -3,7 +3,7 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2009, 2021, MariaDB + Copyright (c) 2009, 2022, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -271,6 +271,25 @@ public: { return get_item_copy<Item_func_aes_decrypt>(thd, this); } }; +class Item_func_natural_sort_key : public Item_str_func +{ +public: + Item_func_natural_sort_key(THD *thd, Item *a) + : Item_str_func(thd, a){}; + String *val_str(String *) override; + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("natural_sort_key")}; + return name; + } + bool fix_length_and_dec(void) override; + Item *get_copy(THD *thd) override + { + return get_item_copy<Item_func_natural_sort_key>(thd, this); + } + + bool check_vcol_func_processor(void *arg) override; +}; class Item_func_concat :public Item_str_func { @@ -587,6 +606,23 @@ public: { return get_item_copy<Item_func_substr>(thd, this); } }; +class Item_func_sformat :public Item_str_func +{ + String *val_arg; +public: + Item_func_sformat(THD *thd, List<Item> &list); + ~Item_func_sformat() { delete [] val_arg; } + String *val_str(String*) override; + bool fix_length_and_dec() override; + LEX_CSTRING func_name_cstring() const override + { + static LEX_CSTRING name= {STRING_WITH_LEN("sformat") }; + return name; + } + Item *get_copy(THD *thd) override + { return get_item_copy<Item_func_sformat>(thd, this); } +}; + class Item_func_substr_oracle :public Item_func_substr { protected: @@ -1909,15 +1945,27 @@ public: class Item_func_crc32 :public Item_long_func { bool check_arguments() const override - { return args[0]->check_type_can_return_str(func_name_cstring()); } + { + return args[0]->check_type_can_return_str(func_name_cstring()) && + (arg_count == 1 || + args[1]->check_type_can_return_int(func_name_cstring())); + } String value; + uint32 (*const crc_func)(uint32, const void*, size_t); public: - Item_func_crc32(THD *thd, Item *a): Item_long_func(thd, a) + Item_func_crc32(THD *thd, bool Castagnoli, Item *a) : + Item_long_func(thd, a), + crc_func(Castagnoli ? my_crc32c : my_checksum) + { unsigned_flag= 1; } + Item_func_crc32(THD *thd, bool Castagnoli, Item *a, Item *b) : + Item_long_func(thd, a, b), + crc_func(Castagnoli ? my_crc32c : my_checksum) { unsigned_flag= 1; } LEX_CSTRING func_name_cstring() const override { - static LEX_CSTRING name= {STRING_WITH_LEN("crc32") }; - return name; + static LEX_CSTRING crc32_name= {STRING_WITH_LEN("crc32") }; + static LEX_CSTRING crc32c_name= {STRING_WITH_LEN("crc32c") }; + return crc_func == my_crc32c ? crc32c_name : crc32_name; } bool fix_length_and_dec() override { max_length=10; return FALSE; } longlong val_int() override; @@ -1996,40 +2044,6 @@ public: }; -class Item_func_uuid: public Item_str_func -{ - /* Set if uuid should be returned without separators (Oracle sys_guid) */ - bool without_separators; -public: -Item_func_uuid(THD *thd, bool without_separators_arg): Item_str_func(thd), - without_separators(without_separators_arg) - {} - bool fix_length_and_dec() override - { - collation.set(DTCollation_numeric()); - fix_char_length(without_separators ? MY_UUID_ORACLE_STRING_LENGTH : - MY_UUID_STRING_LENGTH); - return FALSE; - } - bool const_item() const override { return false; } - table_map used_tables() const override { return RAND_TABLE_BIT; } - LEX_CSTRING func_name_cstring() const override - { - static LEX_CSTRING mariadb_name= {STRING_WITH_LEN("uuid") }; - static LEX_CSTRING oracle_name= {STRING_WITH_LEN("sys_guid") }; - return without_separators ? oracle_name : mariadb_name; - } - String *val_str(String *) override; - bool check_vcol_func_processor(void *arg) override - { - return mark_unsupported_function(func_name(), "()", arg, - VCOL_NON_DETERMINISTIC); - } - Item *get_copy(THD *thd) override - { return get_item_copy<Item_func_uuid>(thd, this); } -}; - - class Item_func_dyncol_create: public Item_str_func { protected: diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index eade1356da0..e99c968efba 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1577,7 +1577,7 @@ bool Item_singlerow_subselect::get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t Item_exists_subselect::Item_exists_subselect(THD *thd, st_select_lex *select_lex): - Item_subselect(thd), upper_not(NULL), abort_on_null(0), + Item_subselect(thd), upper_not(NULL), emb_on_expr_nest(NULL), optimizer(0), exists_transformed(0) { DBUG_ENTER("Item_exists_subselect::Item_exists_subselect"); @@ -1662,7 +1662,6 @@ Item_allany_subselect::Item_allany_subselect(THD *thd, Item * left_exp, func= func_creator(all_arg); init(select_lex, new (thd->mem_root) select_exists_subselect(thd, this)); max_columns= 1; - abort_on_null= 0; reset(); //if test_limit will fail then error will be reported to client test_limit(select_lex->master_unit()); @@ -2237,8 +2236,11 @@ bool Item_allany_subselect::is_maxmin_applicable(JOIN *join) Check if max/min optimization applicable: It is top item of WHERE condition. */ - return (abort_on_null || (upper_item && upper_item->is_top_level_item())) && - !(join->select_lex->master_unit()->uncacheable & ~UNCACHEABLE_EXPLAIN) && !func->eqne_op(); + return ((is_top_level_item() || + (upper_item && upper_item->is_top_level_item())) && + !(join->select_lex->master_unit()->uncacheable & + ~UNCACHEABLE_EXPLAIN) && + !func->eqne_op()); } @@ -2308,7 +2310,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, ref_pointer_array[0], {STRING_WITH_LEN("<ref>")}, field_name)); - if (!abort_on_null && left_expr->maybe_null()) + if (!is_top_level_item() && left_expr->maybe_null()) { /* We can encounter "NULL IN (SELECT ...)". Wrap the added condition @@ -2341,7 +2343,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, Item *orig_item= item; item= func->create(thd, expr, item); - if (!abort_on_null && orig_item->maybe_null()) + if (!is_top_level_item() && orig_item->maybe_null()) { having= new (thd->mem_root) Item_is_not_null_test(thd, this, having); if (left_expr->maybe_null()) @@ -2363,7 +2365,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, If we may encounter NULL IN (SELECT ...) and care whether subquery result is NULL or FALSE, wrap condition in a trig_cond. */ - if (!abort_on_null && left_expr->maybe_null()) + if (!is_top_level_item() && left_expr->maybe_null()) { disable_cond_guard_for_const_null_left_expr(0); if (!(item= new (thd->mem_root) Item_func_trig_cond(thd, item, @@ -2393,7 +2395,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, &select_lex->ref_pointer_array[0], no_matter_name, field_name)); - if (!abort_on_null && left_expr->maybe_null()) + if (!is_top_level_item() && left_expr->maybe_null()) { disable_cond_guard_for_const_null_left_expr(0); if (!(new_having= new (thd->mem_root) @@ -2592,7 +2594,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, list_ref)); Item *col_item= new (thd->mem_root) Item_cond_or(thd, item_eq, item_isnull); - if (!abort_on_null && left_expr->element_index(i)->maybe_null() && + if (!is_top_level_item() && left_expr->element_index(i)->maybe_null() && get_cond_guard(i)) { disable_cond_guard_for_const_null_left_expr(i); @@ -2611,7 +2613,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, ref_pointer_array[i], no_matter_name, list_ref)); - if (!abort_on_null && left_expr->element_index(i)->maybe_null() && + if (!is_top_level_item() && left_expr->element_index(i)->maybe_null() && get_cond_guard(i) ) { disable_cond_guard_for_const_null_left_expr(i); @@ -2652,7 +2654,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, ref_pointer_array[i], no_matter_name, list_ref)); - if (!abort_on_null && select_lex->ref_pointer_array[i]->maybe_null()) + if (!is_top_level_item() && select_lex->ref_pointer_array[i]->maybe_null()) { Item *having_col_item= new (thd->mem_root) @@ -2684,7 +2686,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, } *having_item= and_items(thd, *having_item, having_col_item); } - if (!abort_on_null && left_expr->element_index(i)->maybe_null() && + if (!is_top_level_item() && left_expr->element_index(i)->maybe_null() && get_cond_guard(i)) { if (!(item= new (thd->mem_root) @@ -3657,7 +3659,7 @@ bool Item_in_subselect::init_cond_guards() { DBUG_ASSERT(thd); uint cols_num= left_expr->cols(); - if (!abort_on_null && !pushed_cond_guards && + if (!is_top_level_item() && !pushed_cond_guards && (left_expr->maybe_null() || cols_num > 1)) { if (!(pushed_cond_guards= (bool*)thd->alloc(sizeof(bool) * cols_num))) @@ -4243,9 +4245,9 @@ bool subselect_uniquesubquery_engine::copy_ref_key(bool skip_constants) - NULL if select produces empty row set - FALSE otherwise. - In some cases (IN subselect is a top level item, i.e. abort_on_null==TRUE) - the caller doesn't distinguish between NULL and FALSE result and we just - return FALSE. + In some cases (IN subselect is a top level item, i.e. + is_top_level_item() == TRUE, the caller doesn't distinguish between NULL and + FALSE result and we just return FALSE. Otherwise we make a full table scan to see if there is at least one matching row. @@ -5106,7 +5108,7 @@ my_bitmap_init_memroot(MY_BITMAP *map, uint n_bits, MEM_ROOT *mem_root) if (!(bitmap_buf= (my_bitmap_map*) alloc_root(mem_root, bitmap_buffer_size(n_bits))) || - my_bitmap_init(map, bitmap_buf, n_bits, FALSE)) + my_bitmap_init(map, bitmap_buf, n_bits)) return TRUE; bitmap_clear_all(map); return FALSE; @@ -5995,7 +5997,7 @@ bool Ordered_key::alloc_keys_buffers() lookup offset. */ /* Notice that max_null_row is max array index, we need count, so +1. */ - if (my_bitmap_init(&null_key, NULL, (uint)(max_null_row + 1), FALSE)) + if (my_bitmap_init(&null_key, NULL, (uint)(max_null_row + 1))) return TRUE; cur_key_idx= HA_POS_ERROR; diff --git a/sql/item_subselect.h b/sql/item_subselect.h index a304c63c458..8d58e16bb28 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -372,7 +372,6 @@ class Item_exists_subselect :public Item_subselect protected: Item_func_not *upper_not; bool value; /* value of this item (boolean: exists/not-exists) */ - bool abort_on_null; void init_length_and_dec(); bool select_prepare_to_be_in(); @@ -396,7 +395,7 @@ public: Item_exists_subselect(THD *thd_arg, st_select_lex *select_lex); Item_exists_subselect(THD *thd_arg): - Item_subselect(thd_arg), upper_not(NULL), abort_on_null(0), + Item_subselect(thd_arg), upper_not(NULL), emb_on_expr_nest(NULL), optimizer(0), exists_transformed(0) {} @@ -423,8 +422,6 @@ public: bool fix_length_and_dec() override; void print(String *str, enum_query_type query_type) override; bool select_transformer(JOIN *join) override; - void top_level_item() override { abort_on_null=1; } - bool is_top_level_item() const override { return abort_on_null; } bool exists2in_processor(void *opt_arg) override; Item* expr_cache_insert_transformer(THD *thd, uchar *unused) override; diff --git a/sql/item_xmlfunc.cc b/sql/item_xmlfunc.cc index 2f4d34afc6d..a130be4f973 100644 --- a/sql/item_xmlfunc.cc +++ b/sql/item_xmlfunc.cc @@ -1232,13 +1232,13 @@ my_xpath_keyword(MY_XPATH *x, static Item *create_func_true(MY_XPATH *xpath, Item **args, uint nargs) { - return (Item*) &Item_true; + return (Item*) Item_true; } static Item *create_func_false(MY_XPATH *xpath, Item **args, uint nargs) { - return (Item*) &Item_false; + return (Item*) Item_false; } diff --git a/sql/json_table.cc b/sql/json_table.cc index 65fe3c9a659..21782a9f14b 100644 --- a/sql/json_table.cc +++ b/sql/json_table.cc @@ -886,8 +886,7 @@ TABLE *create_table_for_function(THD *thd, TABLE_LIST *sql_table) my_bitmap_map* bitmaps= (my_bitmap_map*) thd->alloc(bitmap_buffer_size(field_count)); - my_bitmap_init(&table->def_read_set, (my_bitmap_map*) bitmaps, field_count, - FALSE); + my_bitmap_init(&table->def_read_set, (my_bitmap_map*) bitmaps, field_count); table->read_set= &table->def_read_set; bitmap_clear_all(table->read_set); table->alias_name_used= true; diff --git a/sql/key.cc b/sql/key.cc index 06845aaf919..4e40a3354ce 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. - Copyright (c) 2018, 2020, MariaDB + Copyright (c) 2018, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -495,6 +495,7 @@ int key_cmp(KEY_PART_INFO *key_part, const uchar *key, uint key_length) { int cmp; store_length= key_part->store_length; + int sort_order = (key_part->key_part_flag & HA_REVERSE_SORT) ? -1 : 1; if (key_part->null_bit) { /* This key part allows null values; NULL is lower than everything */ @@ -503,19 +504,19 @@ int key_cmp(KEY_PART_INFO *key_part, const uchar *key, uint key_length) { /* the range is expecting a null value */ if (!field_is_null) - return 1; // Found key is > range + return sort_order; // Found key is > range /* null -- exact match, go to next key part */ continue; } else if (field_is_null) - return -1; // NULL is less than any value + return -sort_order; // NULL is less than any value key++; // Skip null byte store_length--; } if ((cmp=key_part->field->key_cmp(key, key_part->length)) < 0) - return -1; + return -sort_order; if (cmp > 0) - return 1; + return sort_order; } return 0; // Keys are equal } @@ -573,6 +574,9 @@ int key_rec_cmp(void *key_p, uchar *first_rec, uchar *second_rec) /* loop over every key part */ do { + const int GREATER= key_part->key_part_flag & HA_REVERSE_SORT ? -1 : +1; + const int LESS= -GREATER; + field= key_part->field; if (key_part->null_bit) @@ -593,12 +597,12 @@ int key_rec_cmp(void *key_p, uchar *first_rec, uchar *second_rec) ; /* Fall through, no NULL fields */ else { - DBUG_RETURN(+1); + DBUG_RETURN(GREATER); } } else if (!sec_is_null) { - DBUG_RETURN(-1); + DBUG_RETURN(LESS); } else goto next_loop; /* Both were NULL */ @@ -613,7 +617,7 @@ int key_rec_cmp(void *key_p, uchar *first_rec, uchar *second_rec) if ((result= field->cmp_prefix(field->ptr+first_diff, field->ptr+sec_diff, key_part->length / field->charset()->mbmaxlen))) - DBUG_RETURN(result); + DBUG_RETURN(result * GREATER); next_loop: key_part++; key_part_num++; diff --git a/sql/lex.h b/sql/lex.h index cbf9d9d51b2..4ce88ccc2ee 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -110,6 +110,7 @@ SYMBOL symbols[] = { { "CHAIN", SYM(CHAIN_SYM)}, { "CHANGE", SYM(CHANGE)}, { "CHANGED", SYM(CHANGED)}, + { "CHANNEL", SYM(CHANNEL_SYM)}, { "CHAR", SYM(CHAR_SYM)}, { "CHARACTER", SYM(CHAR_SYM)}, { "CHARSET", SYM(CHARSET)}, @@ -567,6 +568,8 @@ SYMBOL symbols[] = { { "ROWTYPE", SYM(ROWTYPE_MARIADB_SYM)}, { "ROW_COUNT", SYM(ROW_COUNT_SYM)}, { "ROW_FORMAT", SYM(ROW_FORMAT_SYM)}, + /** sql_function and condition_property_name for GET DIAGNOSTICS */ + { "ROW_NUMBER", SYM(ROW_NUMBER_SYM)}, { "RTREE", SYM(RTREE_SYM)}, { "SAVEPOINT", SYM(SAVEPOINT_SYM)}, { "SCHEDULE", SYM(SCHEDULE_SYM)}, @@ -781,7 +784,6 @@ SYMBOL sql_functions[] = { { "PERCENTILE_CONT", SYM(PERCENTILE_CONT_SYM)}, { "PERCENTILE_DISC", SYM(PERCENTILE_DISC_SYM)}, { "RANK", SYM(RANK_SYM)}, - { "ROW_NUMBER", SYM(ROW_NUMBER_SYM)}, { "SESSION_USER", SYM(USER_SYM)}, { "STD", SYM(STD_SYM)}, { "STDDEV", SYM(STD_SYM)}, diff --git a/sql/lock.cc b/sql/lock.cc index 1099a5c2fb1..0767b787bec 100644 --- a/sql/lock.cc +++ b/sql/lock.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2020, MariaDB + Copyright (c) 2020, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1141,6 +1141,9 @@ void Global_read_lock::unlock_global_read_lock(THD *thd) else if (WSREP_NNULL(thd) && server_state.state() == Wsrep_server_state::s_synced) { + THD_STAGE_INFO(thd, stage_waiting_flow); + WSREP_DEBUG("unlock_global_read_lock: waiting for flow control for %s", + wsrep_thd_query(thd)); server_state.resume_and_resync(); wsrep_locked_seqno= WSREP_SEQNO_UNDEFINED; } diff --git a/sql/log.cc b/sql/log.cc index 811921d8322..ac86441df32 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -57,6 +57,7 @@ #include "semisync_master.h" #include "sp_rcontext.h" #include "sp_head.h" +#include "sql_table.h" #include "wsrep_mysqld.h" #ifdef WITH_WSREP @@ -576,13 +577,137 @@ public: ulong binlog_id; /* Set if we get an error during commit that must be returned from unlog(). */ bool delayed_error; - + //Will be reset when gtid is written into binlog + uchar gtid_flags3; + decltype (rpl_gtid::seq_no) sa_seq_no; private: binlog_cache_mngr& operator=(const binlog_cache_mngr& info); binlog_cache_mngr(const binlog_cache_mngr& info); }; +/** + The function handles the first phase of two-phase binlogged ALTER. + On master binlogs START ALTER when that is configured to do so. + On slave START ALTER gets binlogged and its gtid committed into gtid slave pos + table. + + @param thd Thread handle. + @param start_alter_id Start Alter identifier or zero. + @param[out] + partial_alter Is set to true when Start Alter phase is completed. + @param if_exists True indicates the binary logging of the query + should be done with "if exists" option. + + @return false on success, true on failure + @return @c partial_alter set to @c true when START ALTER phase + has been completed +*/ +bool write_bin_log_start_alter(THD *thd, bool& partial_alter, + uint64 start_alter_id, bool if_exists) +{ +#if defined(HAVE_REPLICATION) + if (thd->variables.option_bits & OPTION_BIN_TMP_LOG_OFF) + return false; + + if (start_alter_id) + { + if (thd->rgi_slave->get_finish_event_group_called()) + return false; // can get here through retrying + + DBUG_EXECUTE_IF("at_write_start_alter", { + debug_sync_set_action(thd, + STRING_WITH_LEN("now wait_for alter_cont")); + }); + + Master_info *mi= thd->rgi_slave->rli->mi; + start_alter_info *info= thd->rgi_slave->sa_info; + bool is_shutdown= false; + + info->sa_seq_no= start_alter_id; + info->domain_id= thd->variables.gtid_domain_id; + mysql_mutex_lock(&mi->start_alter_list_lock); + // possible stop-slave's marking of the whole alter state list is checked + is_shutdown= mi->is_shutdown; + mi->start_alter_list.push_back(info, &mi->mem_root); + mysql_mutex_unlock(&mi->start_alter_list_lock); + info->state= start_alter_state::REGISTERED; + thd->rgi_slave->commit_orderer.wait_for_prior_commit(thd); + thd->rgi_slave->start_alter_ev->update_pos(thd->rgi_slave); + if (mysql_bin_log.is_open()) + { + Write_log_with_flags wlwf (thd, Gtid_log_event::FL_START_ALTER_E1); + if (write_bin_log(thd, true, thd->query(), thd->query_length())) + { + DBUG_ASSERT(thd->is_error()); + return true; + } + } + thd->rgi_slave->mark_start_commit(); + thd->wakeup_subsequent_commits(0); + thd->rgi_slave->finish_start_alter_event_group(); + + if (is_shutdown) + { + /* SA exists abruptly and will notify any CA|RA waiter. */ + mysql_mutex_lock(&mi->start_alter_lock); + /* + If there is (or will be) unlikely any CA it will execute + the whole query before to stop itself. + */ + info->direct_commit_alter= true; + info->state= start_alter_state::ROLLBACK_ALTER; + mysql_mutex_unlock(&mi->start_alter_lock); + + return true; + } + + return false; + } +#endif + +#ifndef WITH_WSREP + rpl_group_info *rgi= thd->rgi_slave ? thd->rgi_slave : thd->rgi_fake; +#else + rpl_group_info *rgi= thd->slave_thread ? thd->rgi_slave : + WSREP(thd) ? (thd->wsrep_rgi ? thd->wsrep_rgi : thd->rgi_fake) : + thd->rgi_fake; +#endif + + if (!rgi && thd->variables.binlog_alter_two_phase) + { + /* slave applier can handle here only regular ALTER */ + DBUG_ASSERT(!rgi || !(rgi->gtid_ev_flags_extra & + (Gtid_log_event::FL_START_ALTER_E1 | + Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1))); + + /* + After logging binlog state stays flagged with SA flags3 an seq_no. + The state is not reset after write_bin_log() is done which is + deferred for the second logging phase. + */ + thd->set_binlog_flags_for_alter(Gtid_log_event::FL_START_ALTER_E1); + if(write_bin_log_with_if_exists(thd, false, false, if_exists, false)) + { + DBUG_ASSERT(thd->is_error()); + + thd->set_binlog_flags_for_alter(0); + return true; + } + partial_alter= true; + } + else if (rgi && rgi->direct_commit_alter) + { + DBUG_ASSERT(rgi->gtid_ev_flags_extra & + Gtid_log_event::FL_COMMIT_ALTER_E1); + + partial_alter= true; + } + + return false; +} + bool LOGGER::is_log_table_enabled(uint log_table_type) { switch (log_table_type) { @@ -2665,12 +2790,11 @@ static void setup_windows_event_source() static int find_uniq_filename(char *name, ulong min_log_number_to_use, ulong *last_used_log_number) { - uint i; char buff[FN_REFLEN], ext_buf[FN_REFLEN]; struct st_my_dir *dir_info; struct fileinfo *file_info; ulong max_found= 0, next= 0, number= 0; - size_t buf_length, length; + size_t i, buf_length, length; char *start, *end; int error= 0; DBUG_ENTER("find_uniq_filename"); @@ -3001,7 +3125,7 @@ int MYSQL_BIN_LOG::generate_new_name(char *new_name, const char *log_name, fn_format(new_name, log_name, mysql_data_home, "", 4); if (!fn_ext(log_name)[0]) { - if (DBUG_EVALUATE_IF("binlog_inject_new_name_error", TRUE, FALSE) || + if (DBUG_IF("binlog_inject_new_name_error") || unlikely(find_uniq_filename(new_name, next_log_number, &last_used_log_number))) { @@ -3572,7 +3696,7 @@ bool MYSQL_BIN_LOG::open_index_file(const char *index_file_name_arg, mysql_file_seek(index_file_nr, 0L, MY_SEEK_END, MYF(0)), 0, MYF(MY_WME | MY_WAIT_IF_FULL), m_key_file_log_index_cache) || - DBUG_EVALUATE_IF("fault_injection_openning_index", 1, 0)) + DBUG_IF("fault_injection_openning_index")) { /* TODO: all operations creating/deleting the index file or a log, should @@ -3598,7 +3722,7 @@ bool MYSQL_BIN_LOG::open_index_file(const char *index_file_name_arg, open_purge_index_file(FALSE) || purge_index_entry(NULL, NULL, need_mutex) || close_purge_index_file() || - DBUG_EVALUATE_IF("fault_injection_recovering_index", 1, 0)) + DBUG_IF("fault_injection_recovering_index")) { sql_print_error("MYSQL_BIN_LOG::open_index_file failed to sync the index " "file."); @@ -3667,7 +3791,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, if (open_purge_index_file(TRUE) || register_create_index_entry(log_file_name) || sync_purge_index_file() || - DBUG_EVALUATE_IF("fault_injection_registering_index", 1, 0)) + DBUG_IF("fault_injection_registering_index")) { /** TODO: @@ -3949,7 +4073,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, As this is a new log file, we write the file name to the index file. As every time we write to the index file, we sync it. */ - if (DBUG_EVALUATE_IF("fault_injection_updating_index", 1, 0) || + if (DBUG_IF("fault_injection_updating_index") || my_b_write(&index_file, (uchar*) log_file_name, strlen(log_file_name)) || my_b_write(&index_file, (uchar*) "\n", 1) || @@ -5362,8 +5486,8 @@ int MYSQL_BIN_LOG::new_file_impl() r.checksum_alg= relay_log_checksum_alg; DBUG_ASSERT(!is_relay_log || relay_log_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF); - if (DBUG_EVALUATE_IF("fault_injection_new_file_rotate_event", - (error= close_on_error= TRUE), FALSE) || + if ((DBUG_IF("fault_injection_new_file_rotate_event") && + (error= close_on_error= TRUE)) || (error= write_event(&r))) { DBUG_EXECUTE_IF("fault_injection_new_file_rotate_event", errno= 2;); @@ -5799,6 +5923,39 @@ binlog_cache_mngr *THD::binlog_setup_trx_data() DBUG_RETURN(cache_mngr); } + +/* + Two phase logged ALTER getter and setter methods. +*/ +uchar THD::get_binlog_flags_for_alter() +{ + return mysql_bin_log.is_open() ? binlog_setup_trx_data()->gtid_flags3 : 0; +} + +void THD::set_binlog_flags_for_alter(uchar flags) +{ + if (mysql_bin_log.is_open()) + { + // SA must find the flag set empty + DBUG_ASSERT(flags != Gtid_log_event::FL_START_ALTER_E1 || + binlog_setup_trx_data()->gtid_flags3 == 0); + + binlog_setup_trx_data()->gtid_flags3= flags; + } +} + +uint64 THD::get_binlog_start_alter_seq_no() +{ + return mysql_bin_log.is_open() ? binlog_setup_trx_data()->sa_seq_no : 0; +} + +void THD::set_binlog_start_alter_seq_no(uint64 s_no) +{ + if (mysql_bin_log.is_open()) + binlog_setup_trx_data()->sa_seq_no= s_no; +} + + /* Function to start a statement and optionally a transaction for the binary log. @@ -6329,6 +6486,8 @@ MYSQL_BIN_LOG::write_gtid_event(THD *thd, bool standalone, DBUG_RETURN(true); thd->set_last_commit_gtid(gtid); + if (thd->get_binlog_flags_for_alter() & Gtid_log_event::FL_START_ALTER_E1) + thd->set_binlog_start_alter_seq_no(gtid.seq_no); Gtid_log_event gtid_event(thd, seq_no, domain_id, standalone, LOG_EVENT_SUPPRESS_USE_F, is_transactional, @@ -6776,7 +6935,7 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info, my_bool *with_annotate) Write the event. */ if (write_event(event_info, cache_data, file) || - DBUG_EVALUATE_IF("injecting_fault_writing", 1, 0)) + DBUG_IF("injecting_fault_writing")) goto err; error= 0; @@ -7558,9 +7717,9 @@ bool MYSQL_BIN_LOG::write_incident(THD *thd) if (likely(is_open())) { prev_binlog_id= current_binlog_id; - if (likely( - !(error= DBUG_EVALUATE_IF("incident_event_write_error", 1, - write_incident_already_locked(thd)))) && + if (likely(!(error= DBUG_IF("incident_event_write_error") + ? 1 + : write_incident_already_locked(thd))) && likely(!(error= flush_and_sync(0)))) { update_binlog_end_pos(); @@ -8553,7 +8712,7 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) DEBUG_SYNC(leader->thd, "commit_loop_entry_commit_ordered"); ++num_commits; if (current->cache_mngr->using_xa && likely(!current->error) && - DBUG_EVALUATE_IF("skip_commit_ordered", 0, 1)) + !DBUG_IF("skip_commit_ordered")) { mysql_mutex_lock(¤t->thd->LOCK_thd_data); run_commit_ordered(current->thd, current->all); diff --git a/sql/log.h b/sql/log.h index 7b62a1a5477..c20f0fe5a57 100644 --- a/sql/log.h +++ b/sql/log.h @@ -1264,5 +1264,8 @@ get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list); int binlog_commit(THD *thd, bool all, bool is_ro_1pc= false); int binlog_commit_by_xid(handlerton *hton, XID *xid); int binlog_rollback_by_xid(handlerton *hton, XID *xid); +bool write_bin_log_start_alter(THD *thd, bool& partial_alter, + uint64 start_alter_id, bool log_if_exists); + #endif /* LOG_H */ diff --git a/sql/log_event.cc b/sql/log_event.cc index 15f06861928..932a8d1588f 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1378,6 +1378,7 @@ code_name(int code) case Q_MASTER_DATA_WRITTEN_CODE: return "Q_MASTER_DATA_WRITTEN_CODE"; case Q_HRNOW: return "Q_HRNOW"; case Q_XID: return "XID"; + case Q_GTID_FLAGS3: return "Q_GTID_FLAGS3"; } sprintf(buf, "CODE#%d", code); return buf; @@ -1426,7 +1427,8 @@ Query_log_event::Query_log_event(const uchar *buf, uint event_len, flags2_inited(0), sql_mode_inited(0), charset_inited(0), flags2(0), auto_increment_increment(1), auto_increment_offset(1), time_zone_len(0), lc_time_names_number(0), charset_database_number(0), - table_map_for_update(0), xid(0), master_data_written(0) + table_map_for_update(0), xid(0), master_data_written(0), gtid_flags_extra(0), + sa_seq_no(0) { ulong data_len; uint32 tmp; @@ -1442,28 +1444,28 @@ Query_log_event::Query_log_event(const uchar *buf, uint event_len, post_header_len= description_event->post_header_len[event_type-1]; DBUG_PRINT("info",("event_len: %u common_header_len: %d post_header_len: %d", event_len, common_header_len, post_header_len)); - + /* We test if the event's length is sensible, and if so we compute data_len. We cannot rely on QUERY_HEADER_LEN here as it would not be format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same for 3.23, 4.0 & 5.0. */ if (event_len < (uint)(common_header_len + post_header_len)) - DBUG_VOID_RETURN; + DBUG_VOID_RETURN; data_len= event_len - (common_header_len + post_header_len); buf+= common_header_len; - - thread_id= slave_proxy_id= uint4korr(buf + Q_THREAD_ID_OFFSET); - exec_time= uint4korr(buf + Q_EXEC_TIME_OFFSET); - db_len= buf[Q_DB_LEN_OFFSET]; // TODO: add a check of all *_len vars - error_code= uint2korr(buf + Q_ERR_CODE_OFFSET); + + thread_id = slave_proxy_id = uint4korr(buf + Q_THREAD_ID_OFFSET); + exec_time = uint4korr(buf + Q_EXEC_TIME_OFFSET); + db_len = (uchar)buf[Q_DB_LEN_OFFSET]; // TODO: add a check of all *_len vars + error_code = uint2korr(buf + Q_ERR_CODE_OFFSET); /* 5.0 format starts here. Depending on the format, we may or not have affected/warnings etc The remnent post-header to be parsed has length: */ - tmp= post_header_len - QUERY_HEADER_MINIMAL_LEN; + tmp= post_header_len - QUERY_HEADER_MINIMAL_LEN; if (tmp) { status_vars_len= uint2korr(buf + Q_STATUS_VARS_LEN_OFFSET); @@ -1610,13 +1612,26 @@ Query_log_event::Query_log_event(const uchar *buf, uint event_len, pos+= 3; break; } - case Q_XID: + case Q_XID: { CHECK_SPACE(pos, end, 8); xid= uint8korr(pos); pos+= 8; break; } + case Q_GTID_FLAGS3: + { + CHECK_SPACE(pos, end, 1); + gtid_flags_extra= *pos++; + if (gtid_flags_extra & (Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1)) + { + CHECK_SPACE(pos, end, 8); + sa_seq_no = uint8korr(pos); + pos+= 8; + } + break; + } default: /* That's why you must write status vars in growing order of code */ DBUG_PRINT("info",("Query_log_event has unknown status vars (first has\ @@ -2635,7 +2650,7 @@ Gtid_log_event::Gtid_log_event(const uchar *buf, uint event_len, extra engines flags presence is identifed by non-zero byte value at this point */ - if (flags_extra & FL_EXTRA_MULTI_ENGINE) + if (flags_extra & FL_EXTRA_MULTI_ENGINE_E1) { DBUG_ASSERT(static_cast<uint>(buf - buf_0) < event_len); @@ -2643,6 +2658,11 @@ Gtid_log_event::Gtid_log_event(const uchar *buf, uint event_len, DBUG_ASSERT(extra_engines > 0); } + if (flags_extra & (FL_COMMIT_ALTER_E1 | FL_ROLLBACK_ALTER_E1)) + { + sa_seq_no= uint8korr(buf); + buf+= 8; + } } /* the strict '<' part of the assert corresponds to extra zero-padded @@ -2659,6 +2679,20 @@ Gtid_log_event::Gtid_log_event(const uchar *buf, uint event_len, buf_0[event_len - 1] == 0); } +int compare_glle_gtids(const void * _gtid1, const void *_gtid2) +{ + rpl_gtid *gtid1= (rpl_gtid *) _gtid1; + rpl_gtid *gtid2= (rpl_gtid *) _gtid2; + + int ret; + if (*gtid1 < *gtid2) + ret= -1; + else if (*gtid1 > *gtid2) + ret= 1; + else + ret= 0; + return ret; +} /* GTID list. */ @@ -3347,8 +3381,7 @@ Rows_log_event::Rows_log_event(const uchar *buf, uint event_len, /* if my_bitmap_init fails, caught in is_valid() */ if (likely(!my_bitmap_init(&m_cols, m_width <= sizeof(m_bitbuf)*8 ? m_bitbuf : NULL, - m_width, - false))) + m_width))) { DBUG_PRINT("debug", ("Reading from %p", ptr_after_width)); memcpy(m_cols.bitmap, ptr_after_width, (m_width + 7) / 8); @@ -3372,8 +3405,7 @@ Rows_log_event::Rows_log_event(const uchar *buf, uint event_len, /* if my_bitmap_init fails, caught in is_valid() */ if (likely(!my_bitmap_init(&m_cols_ai, m_width <= sizeof(m_bitbuf_ai)*8 ? m_bitbuf_ai : NULL, - m_width, - false))) + m_width))) { DBUG_PRINT("debug", ("Reading from %p", ptr_after_width)); memcpy(m_cols_ai.bitmap, ptr_after_width, (m_width + 7) / 8); diff --git a/sql/log_event.h b/sql/log_event.h index 88db7984714..ad575ee7244 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -240,7 +240,8 @@ class String; 1 + 8 /* type, table_map_for_update */ + \ 1 + 4 /* type, master_data_written */ + \ 1 + 3 /* type, sec_part of NOW() */ + \ - 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */) + 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */ + \ + 1 + 2 + 8 /* type, flags3, seq_no */) #define MAX_LOG_EVENT_HEADER ( /* in order of Query_log_event::write */ \ LOG_EVENT_HEADER_LEN + /* write_header */ \ QUERY_HEADER_LEN + /* write_data */ \ @@ -321,6 +322,7 @@ class String; #define Q_HRNOW 128 #define Q_XID 129 +#define Q_GTID_FLAGS3 130 /* Intvar event post-header */ /* Intvar event data */ @@ -903,6 +905,20 @@ typedef struct st_print_event_info IO_CACHE review_sql_cache; #endif FILE *file; + + + + /* + Used to include the events within a GTID start/stop boundary + */ + my_bool m_is_event_group_active; + + /* + Tracks whether or not output events must be explicitly activated in order + to be printed + */ + my_bool m_is_event_group_filtering_enabled; + st_print_event_info(); ~st_print_event_info() { @@ -925,6 +941,40 @@ typedef struct st_print_event_info copy_event_cache_to_file_and_reinit(&body_cache, file); fflush(file); } + + /* + Notify that all events part of the current group should be printed + */ + void activate_current_event_group() + { + m_is_event_group_active= TRUE; + } + void deactivate_current_event_group() + { + m_is_event_group_active= FALSE; + } + + /* + Used for displaying events part of an event group. + Returns TRUE when both event group filtering is enabled and the current + event group should be displayed, OR if event group filtering is + disabled. More specifically, if filtering is disabled, all events + should be shown. + Returns FALSE when event group filtering is enabled and the current event + group is filtered out. + */ + my_bool is_event_group_active() + { + return m_is_event_group_filtering_enabled ? m_is_event_group_active : TRUE; + } + + /* + Notify that events must be explicitly activated in order to be printed + */ + void enable_event_group_filtering() + { + m_is_event_group_filtering_enabled= TRUE; + } } PRINT_EVENT_INFO; #endif @@ -2128,6 +2178,12 @@ public: Q_MASTER_DATA_WRITTEN_CODE to the slave's server binlog. */ uint32 master_data_written; + /* + A copy of Gtid event's extra flags that is relevant for two-phase + logged ALTER. + */ + uchar gtid_flags_extra; + decltype(rpl_gtid::seq_no) sa_seq_no; /* data part for CA/RA flags */ #ifdef MYSQL_SERVER @@ -2139,6 +2195,7 @@ public: #endif /* HAVE_REPLICATION */ #else bool print_query_header(IO_CACHE* file, PRINT_EVENT_INFO* print_event_info); + bool print_verbose(IO_CACHE* cache, PRINT_EVENT_INFO* print_event_info); bool print(FILE* file, PRINT_EVENT_INFO* print_event_info); #endif @@ -2152,8 +2209,10 @@ public: my_free(data_buf); } Log_event_type get_type_code() { return QUERY_EVENT; } - static int dummy_event(String *packet, ulong ev_offset, enum enum_binlog_checksum_alg checksum_alg); - static int begin_event(String *packet, ulong ev_offset, enum enum_binlog_checksum_alg checksum_alg); + static int dummy_event(String *packet, ulong ev_offset, + enum enum_binlog_checksum_alg checksum_alg); + static int begin_event(String *packet, ulong ev_offset, + enum enum_binlog_checksum_alg checksum_alg); #ifdef MYSQL_SERVER bool write(); virtual bool write_post_header_for_derived() { return FALSE; } @@ -2179,6 +2238,9 @@ public: /* !!! Public in this patch to allow old usage */ size_t event_len, enum enum_binlog_checksum_alg checksum_alg); + int handle_split_alter_query_log_event(rpl_group_info *rgi, + bool &skip_error_check); + #endif /* HAVE_REPLICATION */ /* If true, the event always be applied by slave SQL thread or be printed by @@ -3587,13 +3649,19 @@ public: uint64 seq_no; uint64 commit_id; uint32 domain_id; + uint64 sa_seq_no; // start alter identifier for CA/RA #ifdef MYSQL_SERVER event_xid_t xid; #else event_mysql_xid_t xid; #endif uchar flags2; - uint flags_extra; // more flags area placed after the regular flags2's one + /* + More flags area placed after the regular flags2's area. The type + is declared to be in agreement with Query_log_event's member that + may copy the flags_extra value. + */ + decltype(Query_log_event::gtid_flags_extra) flags_extra; /* Number of engine participants in transaction minus 1. When zero the event does not contain that information. @@ -3631,14 +3699,19 @@ public: /* FL_"COMMITTED or ROLLED-BACK"_XA is set for XA transaction. */ static const uchar FL_COMPLETED_XA= 128; - /* Flags_extra. */ - /* - FL_EXTRA_MULTI_ENGINE is set for event group comprising a transaction + flags_extra 's bit values. + _E1 suffix below stands for Extra to infer the extra flags, + their "1st" generation (more *generations* can come when necessary). + + FL_EXTRA_MULTI_ENGINE_E1 is set for event group comprising a transaction involving multiple storage engines. No flag and extra data are added to the event when the transaction involves only one engine. */ - static const uchar FL_EXTRA_MULTI_ENGINE= 1; + static const uchar FL_EXTRA_MULTI_ENGINE_E1= 1; + static const uchar FL_START_ALTER_E1= 2; + static const uchar FL_COMMIT_ALTER_E1= 4; + static const uchar FL_ROLLBACK_ALTER_E1= 8; #ifdef MYSQL_SERVER Gtid_log_event(THD *thd_arg, uint64 seq_no, uint32 domain_id, bool standalone, @@ -5808,4 +5881,12 @@ int row_log_event_uncompress(const Format_description_log_event uchar* buf, ulong buf_size, bool *is_malloc, uchar **dst, ulong *newlen); +bool is_parallel_retry_error(rpl_group_info *rgi, int err); + +/* + Compares two GTIDs to facilitate sorting a GTID list log event by domain id + (ascending) and sequence number (ascending) +*/ +int compare_glle_gtids(const void * _gtid1, const void *_gtid2); + #endif /* _log_event_h */ diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index 51667f4fcce..03e319076c7 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -17,6 +17,7 @@ */ +#include "log_event.h" #ifndef MYSQL_CLIENT #error MYSQL_CLIENT must be defined here #endif @@ -1191,7 +1192,7 @@ void Rows_log_event::change_to_flashback_event(PRINT_EVENT_INFO *print_event_inf } /* Copying rows from the end to the begining into event */ - for (uint i= rows_arr.elements; i > 0; --i) + for (size_t i= rows_arr.elements; i > 0; --i) { LEX_STRING *one_row= dynamic_element(&rows_arr, i - 1, LEX_STRING*); @@ -2015,6 +2016,19 @@ err: return 1; } +bool Query_log_event::print_verbose(IO_CACHE* cache, PRINT_EVENT_INFO* print_event_info) +{ + if (my_b_printf(cache, "### ") || + my_b_write(cache, (uchar *) query, q_len) || + my_b_printf(cache, "\n")) + { + goto err; + } + return 0; + +err: + return 1; +} bool Query_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) { @@ -2030,9 +2044,42 @@ bool Query_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) goto err; if (!is_flashback) { - if (my_b_write(&cache, (uchar*) query, q_len) || - my_b_printf(&cache, "\n%s\n", print_event_info->delimiter)) - goto err; + if (gtid_flags_extra & (Gtid_log_event::FL_START_ALTER_E1 | + Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1)) + { + bool do_print_encoded= + print_event_info->base64_output_mode != BASE64_OUTPUT_NEVER && + print_event_info->base64_output_mode != BASE64_OUTPUT_DECODE_ROWS && + !print_event_info->short_form; + bool comment_mode= do_print_encoded && + gtid_flags_extra & (Gtid_log_event::FL_START_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1); + + if(comment_mode) + my_b_printf(&cache, "/*!100600 "); + if (do_print_encoded) + my_b_printf(&cache, "BINLOG '\n"); + if (print_base64(&cache, print_event_info, do_print_encoded)) + goto err; + if (do_print_encoded) + { + if(comment_mode) + my_b_printf(&cache, "' */%s\n", print_event_info->delimiter); + else + my_b_printf(&cache, "'%s\n", print_event_info->delimiter); + } + if (print_event_info->verbose && print_verbose(&cache, print_event_info)) + { + goto err; + } + } + else + { + if (my_b_write(&cache, (uchar*) query, q_len) || + my_b_printf(&cache, "\n%s\n", print_event_info->delimiter)) + goto err; + } } else // is_flashback == 1 { @@ -2305,6 +2352,8 @@ Gtid_list_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) char buf[21]; uint32 i; + qsort(list, count, sizeof(rpl_gtid), compare_glle_gtids); + if (print_header(&cache, print_event_info, FALSE) || my_b_printf(&cache, "\tGtid list [")) goto err; @@ -3801,6 +3850,8 @@ st_print_event_info::st_print_event_info() printed_fd_event=FALSE; file= 0; base64_output_mode=BASE64_OUTPUT_UNSPEC; + m_is_event_group_active= TRUE; + m_is_event_group_filtering_enabled= FALSE; open_cached_file(&head_cache, NULL, NULL, 0, flags); open_cached_file(&body_cache, NULL, NULL, 0, flags); open_cached_file(&tail_cache, NULL, NULL, 0, flags); @@ -3864,6 +3915,15 @@ Gtid_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) if (flags2 & FL_WAITED) if (my_b_write_string(&cache, " waited")) goto err; + if (flags_extra & FL_START_ALTER_E1) + if (my_b_write_string(&cache, " START ALTER")) + goto err; + if (flags_extra & FL_COMMIT_ALTER_E1) + if (my_b_printf(&cache, " COMMIT ALTER id= %lu", sa_seq_no)) + goto err; + if (flags_extra & FL_ROLLBACK_ALTER_E1) + if (my_b_printf(&cache, " ROLLBACK ALTER id= %lu", sa_seq_no)) + goto err; if (my_b_printf(&cache, "\n")) goto err; diff --git a/sql/log_event_old.cc b/sql/log_event_old.cc index 4e6b9e3f1c8..1990103598e 100644 --- a/sql/log_event_old.cc +++ b/sql/log_event_old.cc @@ -1156,8 +1156,7 @@ Old_rows_log_event::Old_rows_log_event(THD *thd_arg, TABLE *tbl_arg, ulong tid, /* if my_bitmap_init fails, caught in is_valid() */ if (likely(!my_bitmap_init(&m_cols, m_width <= sizeof(m_bitbuf)*8 ? m_bitbuf : NULL, - m_width, - false))) + m_width))) { /* Cols can be zero if this is a dummy binrows event */ if (likely(cols != NULL)) @@ -1232,8 +1231,7 @@ Old_rows_log_event::Old_rows_log_event(const uchar *buf, uint event_len, /* if my_bitmap_init fails, caught in is_valid() */ if (likely(!my_bitmap_init(&m_cols, m_width <= sizeof(m_bitbuf)*8 ? m_bitbuf : NULL, - m_width, - false))) + m_width))) { DBUG_PRINT("debug", ("Reading from %p", ptr_after_width)); memcpy(m_cols.bitmap, ptr_after_width, (m_width + 7) / 8); diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index f80cf68623a..b3fca6de0ac 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -52,6 +52,7 @@ #include "compat56.h" #include "wsrep_mysqld.h" #include "sql_insert.h" +#include "sql_table.h" #include <my_bitmap.h> #include "rpl_utility.h" @@ -139,7 +140,7 @@ static const char *HA_ERR(int i) deadlocks; such errors are handled automatically by rolling back re-trying the transactions, so should not pollute the error log. */ -static bool +bool is_parallel_retry_error(rpl_group_info *rgi, int err) { if (!rgi->is_parallel_exec) @@ -484,7 +485,7 @@ static void cleanup_load_tmpdir(LEX_CSTRING *connection_name) { MY_DIR *dirp; FILEINFO *file; - uint i; + size_t i; char dir[FN_REFLEN], fname[FN_REFLEN]; char prefbuf[31 + MAX_CONNECTION_NAME* MAX_FILENAME_MBWIDTH + 1]; DBUG_ENTER("cleanup_load_tmpdir"); @@ -505,7 +506,7 @@ static void cleanup_load_tmpdir(LEX_CSTRING *connection_name) load_data_tmp_prefix(prefbuf, connection_name); DBUG_PRINT("enter", ("dir: '%s' prefix: '%s'", dir, prefbuf)); - for (i=0 ; i < (uint)dirp->number_of_files; i++) + for (i=0 ; i < dirp->number_of_files; i++) { file=dirp->dir_entry+i; if (is_prefix(file->name, prefbuf)) @@ -1317,11 +1318,25 @@ bool Query_log_event::write() start+= 8; } + if (gtid_flags_extra) + { + *start++= Q_GTID_FLAGS3; + *start++= gtid_flags_extra; + if (gtid_flags_extra & + (Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1)) + { + int8store(start, sa_seq_no); + start+= 8; + } + } + + /* NOTE: When adding new status vars, please don't forget to update the MAX_SIZE_LOG_EVENT_STATUS in log_event.h and update the function code_name() in this file. - + Here there could be code like if (command-line-option-which-says-"log_this_variable" && inited) { @@ -1429,7 +1444,9 @@ Query_log_event::Query_log_event(THD* thd_arg, const char* query_arg, lc_time_names_number(thd_arg->variables.lc_time_names->number), charset_database_number(0), table_map_for_update((ulonglong)thd_arg->table_map_for_update), - master_data_written(0) + master_data_written(0), + gtid_flags_extra(thd_arg->get_binlog_flags_for_alter()), + sa_seq_no(0) { /* status_vars_len is set just before writing the event */ @@ -1565,11 +1582,15 @@ Query_log_event::Query_log_event(THD* thd_arg, const char* query_arg, use_cache= trx_cache= TRUE; break; default: - use_cache= sqlcom_can_generate_row_events(thd); + use_cache= (gtid_flags_extra) ? false : sqlcom_can_generate_row_events(thd); break; } } + if (gtid_flags_extra & (Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1)) + sa_seq_no= thd_arg->get_binlog_start_alter_seq_no(); + if (!use_cache || direct) { cache_type= Log_event::EVENT_NO_CACHE; @@ -1641,6 +1662,223 @@ bool test_if_equal_repl_errors(int expected_error, int actual_error) } +static start_alter_info *get_new_start_alter_info(THD *thd) +{ + /* + Why on global memory ?- So that process_commit/rollback_alter should not get + error when spawned threads exits too early. + */ + start_alter_info *info; + if (!(info= (start_alter_info *)my_malloc(PSI_INSTRUMENT_ME, + sizeof(start_alter_info), MYF(MY_WME)))) + { + sql_print_error("Failed to allocate memory for ddl log free list"); + return 0; + } + info->sa_seq_no= 0; + info->domain_id= 0; + info->direct_commit_alter= false; + info->state= start_alter_state::INVALID; + mysql_cond_init(0, &info->start_alter_cond, NULL); + info->error= 0; + + return info; +} + + +/* + Perform necessary actions for two-phase-logged ALTER parts, to + return + + 0 when the event's query proceeds normal parsing and execution + 1 when the event skips parsing and execution + -1 as error. +*/ +int Query_log_event::handle_split_alter_query_log_event(rpl_group_info *rgi, + bool &skip_error_check) +{ + int rc= 0; + + rgi->gtid_ev_flags_extra= gtid_flags_extra; + if (gtid_flags_extra & Gtid_log_event::FL_START_ALTER_E1) + { + //No Slave, Normal Slave, Start Alter under Worker 1 will simple binlog and exit + if(!rgi->rpt || rgi->reserved_start_alter_thread || WSREP(thd)) + { + rc= 1; + /* + We will just write the binlog and move to next event , because COMMIT + Alter will take care of actual work + */ + rgi->reserved_start_alter_thread= false; + thd->lex->sql_command= SQLCOM_ALTER_TABLE; + Write_log_with_flags wlwf(thd, Gtid_log_event::FL_START_ALTER_E1, + true /* wsrep to isolation end */); +#ifdef WITH_WSREP + if (WSREP(thd) && wsrep_thd_is_local(thd) && + // no need to supply other than db in this case + wsrep_to_isolation_begin(thd, db, NULL,NULL,NULL,NULL,NULL)) + return -1; +#endif + if (write_bin_log(thd, false, thd->query(), thd->query_length())) + return -1; + + my_ok(thd); + return rc; + } + if (!rgi->sa_info) + rgi->sa_info= get_new_start_alter_info(thd); + else + { + /* Not send Start-Alter into query execution when it's to rollback */ + mysql_mutex_lock(&rgi->rli->mi->start_alter_lock); + if (rgi->sa_info->state == start_alter_state::ROLLBACK_ALTER) + mysql_cond_broadcast(&rgi->sa_info->start_alter_cond); + mysql_mutex_unlock(&rgi->rli->mi->start_alter_lock); + } + + return rc; + } + + bool is_CA= (gtid_flags_extra & Gtid_log_event::FL_COMMIT_ALTER_E1) ? true : false; + if (is_CA) + { + DBUG_EXECUTE_IF("rpl_slave_stop_CA_before_binlog", + { + // the awake comes from STOP-SLAVE running driver (sql) thread + debug_sync_set_action(thd, + STRING_WITH_LEN("now WAIT_FOR proceed_CA_1")); + }); + } + start_alter_info *info=NULL; + Master_info *mi= NULL; + + rgi->gtid_ev_sa_seq_no= sa_seq_no; + // is set for both the direct execution and the write to binlog + thd->set_binlog_start_alter_seq_no(sa_seq_no); + mi= rgi->rli->mi; + mysql_mutex_lock(&mi->start_alter_list_lock); + { + List_iterator<start_alter_info> info_iterator(mi->start_alter_list); + while ((info= info_iterator++)) + { + if(info->sa_seq_no == rgi->gtid_ev_sa_seq_no && + info->domain_id == rgi->current_gtid.domain_id) + { + info_iterator.remove(); + break; + } + } + } + mysql_mutex_unlock(&mi->start_alter_list_lock); + + if (!info) + { + if (is_CA) + { + /* + error handeling, direct_commit_alter is turned on, so that we dont + wait for master reply in mysql_alter_table (in wait_for_master) + */ + rgi->direct_commit_alter= true; +#ifdef WITH_WSREP + if (WSREP(thd)) + thd->set_binlog_flags_for_alter(Gtid_log_event::FL_COMMIT_ALTER_E1); +#endif + goto cleanup; + } + else + { + //Just write the binlog because there is nothing to be done + goto write_binlog; + } + } + + mysql_mutex_lock(&mi->start_alter_lock); + if (info->state != start_alter_state::COMPLETED) + { + if (is_CA) + info->state= start_alter_state::COMMIT_ALTER; + else + info->state= start_alter_state::ROLLBACK_ALTER; + mysql_cond_broadcast(&info->start_alter_cond); + mysql_mutex_unlock(&mi->start_alter_lock); + /* + Wait till Start Alter worker has changed the state to ::COMPLETED + when start alter worker reaches the old code write_bin_log(), it will + change state to COMMITTED. + COMMITTED and `direct_commit_alter == true` at the same time indicates + the query needs re-execution by the CA running thread. + */ + mysql_mutex_lock(&mi->start_alter_lock); + + DBUG_ASSERT(info->state == start_alter_state::COMPLETED || + !info->direct_commit_alter); + + while(info->state != start_alter_state::COMPLETED) + mysql_cond_wait(&info->start_alter_cond, &mi->start_alter_lock); + } + else + { + // SA has completed and left being kicked out by deadlock or ftwrl + DBUG_ASSERT(info->direct_commit_alter); + } + mysql_mutex_unlock(&mi->start_alter_lock); + + if (info->direct_commit_alter) + { + rgi->direct_commit_alter= true; // execute the query as if there was no SA + if (is_CA) + goto cleanup; + } + +write_binlog: + rc= 1; + + if(!is_CA) + { + if(((info && info->error) || error_code) && + global_system_variables.log_warnings > 2) + { + sql_print_information("Query '%s' having %d error code on master " + "is rolled back%s", query, error_code, + !(info && info->error) ? "." : ";"); + if (info && info->error) + sql_print_information("its execution on slave %sproduced %d error.", + info->error == error_code ? "re":"", info->error); + } + } + { + thd->lex->sql_command= SQLCOM_ALTER_TABLE; + Write_log_with_flags wlwf(thd, is_CA ? Gtid_log_event::FL_COMMIT_ALTER_E1 : + Gtid_log_event::FL_ROLLBACK_ALTER_E1, + true); +#ifdef WITH_WSREP + if (WSREP(thd) && wsrep_thd_is_local(thd) && + wsrep_to_isolation_begin(thd, db, NULL,NULL,NULL,NULL,NULL)) + rc= -1; +#endif + if (rc != -1 && + write_bin_log(thd, false, thd->query(), thd->query_length())) + rc= -1; + } + + if (!thd->is_error()) + { + skip_error_check= true; + my_ok(thd); + } + +cleanup: + if (info) + { + mysql_cond_destroy(&info->start_alter_cond); + my_free(info); + } + return rc; +} + + /** @todo Compare the values of "affected rows" around here. Something @@ -1669,6 +1907,7 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, Relay_log_info const *rli= rgi->rli; Rpl_filter *rpl_filter= rli->mi->rpl_filter; bool current_stmt_is_commit; + bool skip_error_check= false; DBUG_ENTER("Query_log_event::do_apply_event"); /* @@ -1679,6 +1918,7 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, you. */ thd->catalog= catalog_len ? (char *) catalog : (char *)""; + rgi->start_alter_ev= this; size_t valid_len= Well_formed_prefix(system_charset_info, db, db_len, NAME_LEN).length(); @@ -1723,13 +1963,15 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, */ if (is_trans_keyword() || rpl_filter->db_ok(thd->db.str)) { + bool is_rb_alter= gtid_flags_extra & Gtid_log_event::FL_ROLLBACK_ALTER_E1; + thd->set_time(when, when_sec_part); thd->set_query_and_id((char*)query_arg, q_len_arg, thd->charset(), next_query_id()); thd->variables.pseudo_thread_id= thread_id; // for temp tables DBUG_PRINT("query",("%s", thd->query())); - if (unlikely(!(expected_error= error_code)) || + if (unlikely(!(expected_error= !is_rb_alter ? error_code : 0)) || ignored_error_code(expected_error) || !unexpected_error_code(expected_error)) { @@ -1760,7 +2002,7 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, if (charset_inited) { rpl_sql_thread_info *sql_info= thd->system_thread_info.rpl_sql_info; - if (sql_info->cached_charset_compare(charset)) + if (thd->slave_thread && sql_info->cached_charset_compare(charset)) { /* Verify that we support the charsets found in the event. */ if (!(thd->variables.character_set_client= @@ -1898,47 +2140,69 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, thd->variables.option_bits|= OPTION_MASTER_SQL_ERROR; thd->variables.option_bits&= ~OPTION_GTID_BEGIN; } - /* Execute the query (note that we bypass dispatch_command()) */ - Parser_state parser_state; - if (!parser_state.init(thd, thd->query(), thd->query_length())) + + int sa_result= 0; + bool is_2p_alter= gtid_flags_extra & + (Gtid_log_event::FL_START_ALTER_E1 | + Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1); + if (is_2p_alter) + sa_result= handle_split_alter_query_log_event(rgi, skip_error_check); + if (sa_result == 0) { - DBUG_ASSERT(thd->m_digest == NULL); - thd->m_digest= & thd->m_digest_state; - DBUG_ASSERT(thd->m_statement_psi == NULL); - thd->m_statement_psi= MYSQL_START_STATEMENT(&thd->m_statement_state, - stmt_info_rpl.m_key, - thd->db.str, thd->db.length, - thd->charset(), NULL); - THD_STAGE_INFO(thd, stage_starting); - MYSQL_SET_STATEMENT_TEXT(thd->m_statement_psi, thd->query(), thd->query_length()); - if (thd->m_digest != NULL) - thd->m_digest->reset(thd->m_token_array, max_digest_length); - - if (thd->slave_thread) - { - /* - To be compatible with previous releases, the slave thread uses the global - log_slow_disabled_statements value, wich can be changed dynamically, so we - have to set the sql_log_slow respectively. - */ - thd->variables.sql_log_slow= !MY_TEST(global_system_variables.log_slow_disabled_statements & LOG_SLOW_DISABLE_SLAVE); - } - - mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); - /* Finalize server status flags after executing a statement. */ - thd->update_server_status(); - log_slow_statement(thd); - thd->lex->restore_set_statement_var(); + /* Execute the query (note that we bypass dispatch_command()) */ + Parser_state parser_state; + if (!parser_state.init(thd, thd->query(), thd->query_length())) + { + DBUG_ASSERT(thd->m_digest == NULL); + thd->m_digest= & thd->m_digest_state; + DBUG_ASSERT(thd->m_statement_psi == NULL); + thd->m_statement_psi= MYSQL_START_STATEMENT(&thd->m_statement_state, + stmt_info_rpl.m_key, + thd->db.str, thd->db.length, + thd->charset(), NULL); + THD_STAGE_INFO(thd, stage_starting); + MYSQL_SET_STATEMENT_TEXT(thd->m_statement_psi, thd->query(), thd->query_length()); + if (thd->m_digest != NULL) + thd->m_digest->reset(thd->m_token_array, max_digest_length); + + if (thd->slave_thread) + { + /* + To be compatible with previous releases, the slave thread uses the global + log_slow_disabled_statements value, wich can be changed dynamically, so we + have to set the sql_log_slow respectively. + */ + thd->variables.sql_log_slow= !MY_TEST(global_system_variables.log_slow_disabled_statements & LOG_SLOW_DISABLE_SLAVE); + } + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); + /* Finalize server status flags after executing a statement. */ + thd->update_server_status(); + log_slow_statement(thd); + thd->lex->restore_set_statement_var(); - /* - When THD::slave_expected_error gets reset inside execution stack - that is the case of to be ignored event. In this case the expected - error must change to the reset value as well. - */ - expected_error= thd->slave_expected_error; + /* + When THD::slave_expected_error gets reset inside execution stack + that is the case of to be ignored event. In this case the expected + error must change to the reset value as well. + */ + expected_error= thd->slave_expected_error; + } + } + else if (sa_result == -1) + { + rli->report(ERROR_LEVEL, expected_error, rgi->gtid_info(), + "TODO start alter error"); + thd->is_slave_error= 1; + goto end; } - thd->variables.option_bits&= ~OPTION_MASTER_SQL_ERROR; + if (is_2p_alter && !rgi->is_parallel_exec) + { + rgi->gtid_ev_flags_extra= 0; + rgi->direct_commit_alter= 0; + rgi->gtid_ev_sa_seq_no= 0; + } } else { @@ -2001,7 +2265,8 @@ compare_errors: If we expected a non-zero error code, and we don't get the same error code, and it should be ignored or is related to a concurrency issue. */ - actual_error= thd->is_error() ? thd->get_stmt_da()->sql_errno() : 0; + actual_error= thd->is_error() ? thd->get_stmt_da()->sql_errno() : + skip_error_check? expected_error : 0; DBUG_PRINT("info",("expected_error: %d sql_errno: %d", expected_error, actual_error)); @@ -2397,6 +2662,39 @@ bool Format_description_log_event::write() } #if defined(HAVE_REPLICATION) +/* + Auxiliary function to conduct cleanup of unfinished two-phase logged ALTERs. +*/ +static void check_and_remove_stale_alter(Relay_log_info *rli) +{ + Master_info *mi= rli->mi; + start_alter_info *info=NULL; + + mysql_mutex_lock(&mi->start_alter_list_lock); + List_iterator<start_alter_info> info_iterator(mi->start_alter_list); + while ((info= info_iterator++)) + { + DBUG_ASSERT(info->state == start_alter_state::REGISTERED); + + sql_print_warning("ALTER query started at %u-%u-%llu could not " + "be completed because of unexpected master server " + "or its binlog change", info->sa_seq_no, // todo:gtid + 0, 0); + info_iterator.remove(); + mysql_mutex_lock(&mi->start_alter_lock); + info->state= start_alter_state::ROLLBACK_ALTER; + mysql_mutex_unlock(&mi->start_alter_lock); + mysql_cond_broadcast(&info->start_alter_cond); + mysql_mutex_lock(&mi->start_alter_lock); + while(info->state != start_alter_state::COMPLETED) + mysql_cond_wait(&info->start_alter_cond, &mi->start_alter_lock); + mysql_mutex_unlock(&mi->start_alter_lock); + mysql_cond_destroy(&info->start_alter_cond); + my_free(info); + } + mysql_mutex_unlock(&mi->start_alter_list_lock); +} + int Format_description_log_event::do_apply_event(rpl_group_info *rgi) { int ret= 0; @@ -2414,16 +2712,21 @@ int Format_description_log_event::do_apply_event(rpl_group_info *rgi) original place when it comes to us; we'll know this by checking log_pos ("artificial" events have log_pos == 0). */ - if (!thd->rli_fake && - !is_artificial_event() && created && thd->transaction->all.ha_list) + if (!is_artificial_event() && created && !thd->rli_fake && !thd->rgi_fake) { - /* This is not an error (XA is safe), just an information */ - rli->report(INFORMATION_LEVEL, 0, NULL, - "Rolling back unfinished transaction (no COMMIT " - "or ROLLBACK in relay log). A probable cause is that " - "the master died while writing the transaction to " - "its binary log, thus rolled back too."); - rgi->cleanup_context(thd, 1); + // check_and_remove stale Start Alter:s + if (flags & LOG_EVENT_BINLOG_IN_USE_F) + check_and_remove_stale_alter(rli); + if (thd->transaction->all.ha_list) + { + /* This is not an error (XA is safe), just an information */ + rli->report(INFORMATION_LEVEL, 0, NULL, + "Rolling back unfinished transaction (no COMMIT " + "or ROLLBACK in relay log). A probable cause is that " + "the master died while writing the transaction to " + "its binary log, thus rolled back too."); + rgi->cleanup_context(thd, 1); + } } /* @@ -3343,7 +3646,14 @@ Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg, extra_engines= count > 1 ? 0 : UCHAR_MAX; } if (extra_engines > 0) - flags_extra|= FL_EXTRA_MULTI_ENGINE; + flags_extra|= FL_EXTRA_MULTI_ENGINE_E1; + } + if (thd->get_binlog_flags_for_alter()) + { + flags_extra |= thd->get_binlog_flags_for_alter(); + if (flags_extra & (FL_COMMIT_ALTER_E1 | FL_ROLLBACK_ALTER_E1)) + sa_seq_no= thd->get_binlog_start_alter_seq_no(); + flags2|= FL_DDL; } } @@ -3416,12 +3726,18 @@ Gtid_log_event::write() buf[write_len]= flags_extra; write_len++; } - if (flags_extra & FL_EXTRA_MULTI_ENGINE) + if (flags_extra & FL_EXTRA_MULTI_ENGINE_E1) { buf[write_len]= extra_engines; write_len++; } + if (flags_extra & (FL_COMMIT_ALTER_E1 | FL_ROLLBACK_ALTER_E1)) + { + int8store(buf + write_len, sa_seq_no); + write_len+= 8; + } + if (write_len < GTID_HEADER_LEN) { bzero(buf+write_len, GTID_HEADER_LEN-write_len); @@ -3485,6 +3801,20 @@ Gtid_log_event::pack_info(Protocol *protocol) p= strmov(p, " cid="); p= longlong10_to_str(commit_id, p, 10); } + if (flags_extra & FL_START_ALTER_E1) + { + p= strmov(p, " START ALTER"); + } + if (flags_extra & FL_COMMIT_ALTER_E1) + { + p= strmov(p, " COMMIT ALTER id="); + p= longlong10_to_str(sa_seq_no, p, 10); + } + if (flags_extra & FL_ROLLBACK_ALTER_E1) + { + p= strmov(p, " ROLLBACK ALTER id="); + p= longlong10_to_str(sa_seq_no, p, 10); + } protocol->store(buf, p-buf, &my_charset_bin); } @@ -3499,6 +3829,9 @@ Gtid_log_event::do_apply_event(rpl_group_info *rgi) thd->variables.gtid_domain_id= this->domain_id; thd->variables.gtid_seq_no= this->seq_no; rgi->gtid_ev_flags2= flags2; + + rgi->gtid_ev_flags_extra= flags_extra; + rgi->gtid_ev_sa_seq_no= sa_seq_no; thd->reset_for_next_command(); if (opt_gtid_strict_mode && opt_bin_log && opt_log_slave_updates) @@ -3769,6 +4102,12 @@ Gtid_list_log_event::pack_info(Protocol *protocol) uint32 i; bool first; + /* + For output consistency and ease of reading, we sort the GTID list in + ascending order + */ + qsort(list, count, sizeof(rpl_gtid), compare_glle_gtids); + buf.length(0); buf.append(STRING_WITH_LEN("[")); first= true; @@ -5258,8 +5597,7 @@ Rows_log_event::Rows_log_event(THD *thd_arg, TABLE *tbl_arg, ulong tid, /* if my_bitmap_init fails, caught in is_valid() */ if (likely(!my_bitmap_init(&m_cols, m_width <= sizeof(m_bitbuf)*8 ? m_bitbuf : NULL, - m_width, - false))) + m_width))) { /* Cols can be zero if this is a dummy binrows event */ if (likely(cols != NULL)) @@ -6490,7 +6828,7 @@ int Table_map_log_event::do_apply_event(rpl_group_info *rgi) LEX_CSTRING tmp_tbl_name= {tname_mem, tname_mem_length }; table_list->init_one_table(&tmp_db_name, &tmp_tbl_name, 0, TL_WRITE); - table_list->table_id= DBUG_EVALUATE_IF("inject_tblmap_same_id_maps_diff_table", 0, m_table_id); + table_list->table_id= DBUG_IF("inject_tblmap_same_id_maps_diff_table") ? 0 : m_table_id; table_list->updating= 1; table_list->required_type= TABLE_TYPE_NORMAL; @@ -6730,7 +7068,7 @@ void Table_map_log_event::init_metadata_fields() if (binlog_row_metadata == BINLOG_ROW_METADATA_FULL) { - if (DBUG_EVALUATE_IF("dont_log_column_name", 0, init_column_name_field()) || + if ((!DBUG_IF("dont_log_column_name") && init_column_name_field()) || init_charset_field(&is_enum_or_set_field, ENUM_AND_SET_DEFAULT_CHARSET, ENUM_AND_SET_COLUMN_CHARSET) || init_set_str_value_field() || @@ -8353,8 +8691,7 @@ void Update_rows_log_event::init(MY_BITMAP const *cols) /* if my_bitmap_init fails, caught in is_valid() */ if (likely(!my_bitmap_init(&m_cols_ai, m_width <= sizeof(m_bitbuf_ai)*8 ? m_bitbuf_ai : NULL, - m_width, - false))) + m_width))) { /* Cols can be zero if this is a dummy binrows event */ if (likely(cols != NULL)) diff --git a/sql/mdl.cc b/sql/mdl.cc index 8fd1fa7580a..6cb27efba12 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -2241,8 +2241,8 @@ bool MDL_lock::check_if_conflicting_replication_locks(MDL_context *ctx) /* If the conflicting thread is another parallel replication - thread for the same master and it's not in commit stage, then - the current transaction has started too early and something is + thread for the same master and it's not in commit or post-commit stages, + then the current transaction has started too early and something is seriously wrong. */ if (conflicting_rgi_slave && @@ -2250,7 +2250,9 @@ bool MDL_lock::check_if_conflicting_replication_locks(MDL_context *ctx) conflicting_rgi_slave->rli == rgi_slave->rli && conflicting_rgi_slave->current_gtid.domain_id == rgi_slave->current_gtid.domain_id && - !conflicting_rgi_slave->did_mark_start_commit) + !((conflicting_rgi_slave->did_mark_start_commit || + conflicting_rgi_slave->worker_error) || + conflicting_rgi_slave->finish_event_group_called)) return 1; // Fatal error } } @@ -2325,6 +2327,20 @@ MDL_context::acquire_lock(MDL_request *mdl_request, double lock_wait_timeout) DBUG_RETURN(TRUE); } +#ifdef WITH_WSREP + if (WSREP(get_thd())) + { + THD* requester= get_thd(); + bool requester_toi= wsrep_thd_is_toi(requester) || wsrep_thd_is_applying(requester); + WSREP_DEBUG("::acquire_lock is TOI %d for %s", requester_toi, + wsrep_thd_query(requester)); + if (requester_toi) + THD_STAGE_INFO(requester, stage_waiting_ddl); + else + THD_STAGE_INFO(requester, stage_waiting_isolation); + } +#endif /* WITH_WSREP */ + lock->m_waiting.add_ticket(ticket); /* diff --git a/sql/my_json_writer.cc b/sql/my_json_writer.cc index 9470ba57855..54eb8423caf 100644 --- a/sql/my_json_writer.cc +++ b/sql/my_json_writer.cc @@ -13,12 +13,11 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ -#include "mariadb.h" -#include "sql_priv.h" -#include "sql_string.h" +#include "my_global.h" #include "my_json_writer.h" #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) + bool Json_writer::named_item_expected() const { return named_items_expectation.size() @@ -39,7 +38,13 @@ inline void Json_writer::on_start_object() #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) if(!fmt_helper.is_making_writer_calls()) { - VALIDITY_ASSERT(got_name == named_item_expected()); + if (got_name != named_item_expected()) + { + sql_print_error(got_name + ? "Json_writer got a member name which is not expected.\n" + : "Json_writer: a member name was expected.\n"); + VALIDITY_ASSERT(got_name == named_item_expected()); + } named_items_expectation.push_back(true); } #endif @@ -60,6 +65,7 @@ void Json_writer::start_object() document_start= false; #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) got_name= false; + named_items.emplace(); #endif } @@ -71,6 +77,8 @@ void Json_writer::start_array() VALIDITY_ASSERT(got_name == named_item_expected()); named_items_expectation.push_back(false); got_name= false; + if (document_start) + named_items.emplace(); } #endif @@ -95,6 +103,8 @@ void Json_writer::end_object() named_items_expectation.pop_back(); VALIDITY_ASSERT(!got_name); got_name= false; + VALIDITY_ASSERT(named_items.size()); + named_items.pop(); #endif indent_level-=INDENT_SIZE; if (!first_child) @@ -140,7 +150,19 @@ Json_writer& Json_writer::add_member(const char *name, size_t len) } #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) if (!fmt_helper.is_making_writer_calls()) + { + VALIDITY_ASSERT(!got_name); got_name= true; + VALIDITY_ASSERT(named_items.size()); + auto& named_items_keys= named_items.top(); + auto emplaced= named_items_keys.emplace(name, len); + auto is_uniq_key= emplaced.second; + if(!is_uniq_key) + { + sql_print_error("Duplicated key: %s\n", emplaced.first->c_str()); + VALIDITY_ASSERT(is_uniq_key); + } + } #endif return *this; } diff --git a/sql/my_json_writer.h b/sql/my_json_writer.h index c2e70962514..87d1a7facf1 100644 --- a/sql/my_json_writer.h +++ b/sql/my_json_writer.h @@ -17,6 +17,7 @@ #define JSON_WRITER_INCLUDED #include "my_base.h" +#include "sql_string.h" #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) || defined ENABLED_JSON_WRITER_CONSISTENCY_CHECKS #include <set> @@ -26,12 +27,11 @@ #endif #ifdef JSON_WRITER_UNIT_TEST -#include "sql_string.h" -constexpr uint FAKE_SELECT_LEX_ID= UINT_MAX; // Also, mock objects are defined in my_json_writer-t.cc #define VALIDITY_ASSERT(x) if (!(x)) this->invalid_json= true; #else -#include "sql_select.h" +#include "sql_class.h" // For class THD +#include "log.h" // for sql_print_error #define VALIDITY_ASSERT(x) DBUG_ASSERT(x) #endif @@ -40,8 +40,10 @@ constexpr uint FAKE_SELECT_LEX_ID= UINT_MAX; class Opt_trace_stmt; class Opt_trace_context; class Json_writer; -struct TABLE_LIST; +struct TABLE; +struct st_join_table; +using JOIN_TAB= struct st_join_table; /* Single_line_formatting_helper is used by Json_writer to do better formatting @@ -216,6 +218,7 @@ class Json_writer produce an invalid JSON document (e.g. JSON array having named elements). */ std::vector<bool> named_items_expectation; + std::stack<std::set<std::string> > named_items; bool named_item_expected() const; @@ -235,6 +238,8 @@ public: Json_writer& add_member(const char *name, size_t len); /* Add atomic values */ + + /* Note: the add_str methods do not do escapes. Should this change? */ void add_str(const char* val); void add_str(const char* val, size_t num_bytes); void add_str(const String &str); diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index aa83ad87588..1543cb5ca56 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -21,6 +21,7 @@ #include "mariadb.h" #include <my_getopt.h> #include <m_string.h> +#include <password.h> #include <windows.h> #include <shellapi.h> @@ -30,6 +31,7 @@ #include <sddl.h> struct IUnknown; #include <shlwapi.h> +#include <winservice.h> #include <string> @@ -432,16 +434,14 @@ static int create_myini() } -static const char update_root_passwd_part1[]= +static constexpr const char* update_root_passwd= "UPDATE mysql.global_priv SET priv=json_set(priv," "'$.password_last_changed', UNIX_TIMESTAMP()," "'$.plugin','mysql_native_password'," - "'$.authentication_string',PASSWORD("; -static const char update_root_passwd_part2[]= - ")) where User='root';\n"; -static const char remove_default_user_cmd[]= + "'$.authentication_string','%s') where User='root';\n"; +static constexpr char remove_default_user_cmd[]= "DELETE FROM mysql.user where User='';\n"; -static const char allow_remote_root_access_cmd[]= +static constexpr char allow_remote_root_access_cmd[]= "CREATE TEMPORARY TABLE tmp_user LIKE global_priv;\n" "INSERT INTO tmp_user SELECT * from global_priv where user='root' " " AND host='localhost';\n" @@ -920,18 +920,10 @@ static int create_db_instance(const char *datadir) /* Change root password if requested. */ if (opt_password && opt_password[0]) { - verbose("Setting root password",remove_default_user_cmd); - fputs(update_root_passwd_part1, in); - - /* Use hex encoding for password, to avoid escaping problems.*/ - fputc('0', in); - fputc('x', in); - for(int i= 0; opt_password[i]; i++) - { - fprintf(in,"%02x",opt_password[i]); - } - - fputs(update_root_passwd_part2, in); + verbose("Setting root password"); + char buf[2 * MY_SHA1_HASH_SIZE + 2]; + my_make_scrambled_password(buf, opt_password, strlen(opt_password)); + fprintf(in, update_root_passwd, buf); fflush(in); } @@ -966,7 +958,7 @@ end: auto sc_manager = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS); if (sc_manager) { - auto sc_handle= OpenServiceA(sc_manager,opt_service, DELETE); + auto sc_handle= OpenService(sc_manager,opt_service, DELETE); if (sc_handle) { DeleteService(sc_handle); diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index 7438ab131ea..02fae11a260 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -374,13 +374,17 @@ static void change_service_config() Write datadir to my.ini, after converting backslashes to unix style slashes. */ - strcpy_s(buf, MAX_PATH, service_properties.datadir); - for(i= 0; buf[i]; i++) + if (service_properties.datadir[0]) { - if (buf[i] == '\\') - buf[i]= '/'; + strcpy_s(buf, MAX_PATH, service_properties.datadir); + for (i= 0; buf[i]; i++) + { + if (buf[i] == '\\') + buf[i]= '/'; + } + WritePrivateProfileString("mysqld", "datadir", buf, + service_properties.inifile); } - WritePrivateProfileString("mysqld", "datadir",buf, service_properties.inifile); /* Remove basedir from defaults file, otherwise the service wont come up in @@ -465,13 +469,8 @@ int main(int argc, char **argv) } } - old_mysqld_exe_exists = (GetFileAttributes(service_properties.mysqld_exe) != INVALID_FILE_ATTRIBUTES); - log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases, my_ini_exists ? "" : "(skipped)"); - - snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK", service_properties.inifile); - CopyFile(service_properties.inifile, my_ini_bck, FALSE); - upgrade_config_file(service_properties.inifile); - + old_mysqld_exe_exists= (GetFileAttributes(service_properties.mysqld_exe) != + INVALID_FILE_ATTRIBUTES); bool do_start_stop_server = old_mysqld_exe_exists && initial_service_state != SERVICE_RUNNING; log("Phase %d/%d: Start and stop server in the old version, to avoid crash recovery %s", ++phase, max_phases, @@ -526,6 +525,14 @@ int main(int argc, char **argv) start_duration_ms += 500; } } + + log("Phase %d/%d: Fixing server config file%s", ++phase, max_phases, + my_ini_exists ? "" : "(skipped)"); + snprintf(my_ini_bck, sizeof(my_ini_bck), "%s.BCK", + service_properties.inifile); + CopyFile(service_properties.inifile, my_ini_bck, FALSE); + upgrade_config_file(service_properties.inifile); + /* Start mysqld.exe as non-service skipping privileges (so we do not care about the password). But disable networking and enable pipe diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 3163ff36c9a..e0c830f8618 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -128,6 +128,7 @@ #ifdef _WIN32 #include <handle_connections_win.h> #include <sddl.h> +#include <winservice.h> /* SERVICE_STOPPED, SERVICE_RUNNING etc */ #endif #include <my_service_manager.h> @@ -372,6 +373,7 @@ uint volatile global_disable_checkpoint; ulong slow_start_timeout; #endif static MEM_ROOT startup_root; +MEM_ROOT read_only_root; /** @brief 'grant_option' is used to indicate if privileges needs @@ -643,7 +645,23 @@ struct system_variables max_system_variables; struct system_status_var global_status_var; MY_TMPDIR mysql_tmpdir_list; -MY_BITMAP temp_pool; +static MY_BITMAP temp_pool; +static mysql_mutex_t LOCK_temp_pool; + +void temp_pool_clear_bit(uint bit) +{ + mysql_mutex_lock(&LOCK_temp_pool); + bitmap_clear_bit(&temp_pool, bit); + mysql_mutex_unlock(&LOCK_temp_pool); +} + +uint temp_pool_set_next() +{ + mysql_mutex_lock(&LOCK_temp_pool); + uint res= bitmap_set_next(&temp_pool); + mysql_mutex_unlock(&LOCK_temp_pool); + return res; +} CHARSET_INFO *system_charset_info, *files_charset_info ; CHARSET_INFO *national_charset_info, *table_alias_charset; @@ -890,11 +908,13 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_manager, key_LOCK_backup_log, key_LOCK_prepared_stmt_count, key_LOCK_rpl_status, key_LOCK_server_started, - key_LOCK_status, + key_LOCK_status, key_LOCK_temp_pool, key_LOCK_system_variables_hash, key_LOCK_thd_data, key_LOCK_thd_kill, key_LOCK_user_conn, key_LOCK_uuid_short_generator, key_LOG_LOCK_log, key_master_info_data_lock, key_master_info_run_lock, key_master_info_sleep_lock, key_master_info_start_stop_lock, + key_master_info_start_alter_lock, + key_master_info_start_alter_list_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_rpl_group_info_sleep_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, @@ -948,6 +968,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_hash_filo_lock, "hash_filo::lock", 0}, { &key_LOCK_active_mi, "LOCK_active_mi", PSI_FLAG_GLOBAL}, { &key_LOCK_backup_log, "LOCK_backup_log", PSI_FLAG_GLOBAL}, + { &key_LOCK_temp_pool, "LOCK_temp_pool", PSI_FLAG_GLOBAL}, { &key_LOCK_thread_id, "LOCK_thread_id", PSI_FLAG_GLOBAL}, { &key_LOCK_crypt, "LOCK_crypt", PSI_FLAG_GLOBAL}, { &key_LOCK_delayed_create, "LOCK_delayed_create", PSI_FLAG_GLOBAL}, @@ -978,6 +999,8 @@ static PSI_mutex_info all_server_mutexes[]= { &key_master_info_start_stop_lock, "Master_info::start_stop_lock", 0}, { &key_master_info_run_lock, "Master_info::run_lock", 0}, { &key_master_info_sleep_lock, "Master_info::sleep_lock", 0}, + { &key_master_info_start_alter_lock, "Master_info::start_alter_lock", 0}, + { &key_master_info_start_alter_list_lock, "Master_info::start_alter_lock", 0}, { &key_mutex_slave_reporting_capability_err_lock, "Slave_reporting_capability::err_lock", 0}, { &key_relay_log_info_data_lock, "Relay_log_info::data_lock", 0}, { &key_relay_log_info_log_space_lock, "Relay_log_info::log_space_lock", 0}, @@ -1509,6 +1532,16 @@ static void end_ssl(); #ifndef EMBEDDED_LIBRARY +extern Atomic_counter<uint32_t> local_connection_thread_count; + +uint THD_count::connection_thd_count() +{ + return value() - + binlog_dump_thread_count - + local_connection_thread_count; +} + + /**************************************************************************** ** Code to end mysqld ****************************************************************************/ @@ -1536,10 +1569,9 @@ static my_bool kill_thread_phase_1(THD *thd, int *n_threads_awaiting_ack) ++(*n_threads_awaiting_ack))) return 0; - if (DBUG_EVALUATE_IF("only_kill_system_threads", !thd->system_thread, 0)) + if (DBUG_IF("only_kill_system_threads") && !thd->system_thread) return 0; - if (DBUG_EVALUATE_IF("only_kill_system_threads_no_loop", - !thd->system_thread, 0)) + if (DBUG_IF("only_kill_system_threads_no_loop") && !thd->system_thread) return 0; thd->awake(KILL_SERVER_HARD); @@ -1771,12 +1803,13 @@ static void close_connections(void) */ DBUG_PRINT("info", ("THD_count: %u", THD_count::value())); - for (int i= 0; (THD_count::value() - binlog_dump_thread_count - - n_threads_awaiting_ack) && - i < 1000 && - DBUG_EVALUATE_IF("only_kill_system_threads_no_loop", 0, 1); - i++) + for (int i= 0; THD_count::connection_thd_count() - n_threads_awaiting_ack + && i < 1000; i++) + { + if (DBUG_IF("only_kill_system_threads_no_loop")) + break; my_sleep(20000); + } if (global_system_variables.log_warnings) server_threads.iterate(warn_threads_active_after_phase_1); @@ -1790,19 +1823,18 @@ static void close_connections(void) #endif /* All threads has now been aborted */ DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)", - THD_count::value() - binlog_dump_thread_count - - n_threads_awaiting_ack)); + THD_count::connection_thd_count() - n_threads_awaiting_ack)); - while ((THD_count::value() - binlog_dump_thread_count - - n_threads_awaiting_ack) && - DBUG_EVALUATE_IF("only_kill_system_threads_no_loop", 0, 1)) + while (THD_count::connection_thd_count() - n_threads_awaiting_ack) { + if (DBUG_IF("only_kill_system_threads_no_loop")) + break; my_sleep(1000); } /* Kill phase 2 */ server_threads.iterate(kill_thread_phase_2); - for (uint64 i= 0; THD_count::value(); i++) + for (uint64 i= 0; THD_count::value() > local_connection_thread_count; i++) { /* This time the warnings are emitted within the loop to provide a @@ -2014,6 +2046,8 @@ static void clean_up(bool print_message) mysql_library_end(); finish_client_errs(); free_root(&startup_root, MYF(0)); + protect_root(&read_only_root, PROT_READ | PROT_WRITE); + free_root(&read_only_root, MYF(0)); cleanup_errmsgs(); free_error_messages(); /* Tell main we are ready */ @@ -2093,6 +2127,7 @@ static void clean_up_mutexes() mysql_mutex_destroy(&LOCK_active_mi); mysql_rwlock_destroy(&LOCK_ssl_refresh); mysql_mutex_destroy(&LOCK_backup_log); + mysql_mutex_destroy(&LOCK_temp_pool); mysql_rwlock_destroy(&LOCK_sys_init_connect); mysql_rwlock_destroy(&LOCK_sys_init_slave); mysql_mutex_destroy(&LOCK_global_system_variables); @@ -3286,7 +3321,7 @@ void my_message_sql(uint error, const char *str, myf MyFlags) { if (unlikely(MyFlags & ME_FATAL)) thd->is_fatal_error= 1; - (void) thd->raise_condition(error, NULL, level, str); + (void) thd->raise_condition(error, "\0\0\0\0\0", level, str); } else mysql_audit_general(0, MYSQL_AUDIT_GENERAL_ERROR, error, str); @@ -3400,7 +3435,6 @@ SHOW_VAR com_status_vars[]= { {"alter_server", STMT_STATUS(SQLCOM_ALTER_SERVER)}, {"alter_sequence", STMT_STATUS(SQLCOM_ALTER_SEQUENCE)}, {"alter_table", STMT_STATUS(SQLCOM_ALTER_TABLE)}, - {"alter_tablespace", STMT_STATUS(SQLCOM_ALTER_TABLESPACE)}, {"alter_user", STMT_STATUS(SQLCOM_ALTER_USER)}, {"analyze", STMT_STATUS(SQLCOM_ANALYZE)}, {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, @@ -3774,6 +3808,8 @@ static int init_early_variables() set_malloc_size_cb(my_malloc_size_cb_func); global_status_var.global_memory_used= 0; init_alloc_root(PSI_NOT_INSTRUMENTED, &startup_root, 1024, 0, MYF(0)); + init_alloc_root(PSI_NOT_INSTRUMENTED, &read_only_root, 1024, 0, + MYF(MY_ROOT_USE_MPROTECT)); return 0; } @@ -4273,7 +4309,7 @@ static int init_common_variables() #endif /* defined(ENABLED_DEBUG_SYNC) */ #if (ENABLE_TEMP_POOL) - if (use_temp_pool && my_bitmap_init(&temp_pool,0,1024,1)) + if (use_temp_pool && my_bitmap_init(&temp_pool,0,1024)) return 1; #else use_temp_pool= 0; @@ -4400,6 +4436,7 @@ static int init_thread_environment() mysql_mutex_init(key_LOCK_commit_ordered, &LOCK_commit_ordered, MY_MUTEX_INIT_SLOW); mysql_mutex_init(key_LOCK_backup_log, &LOCK_backup_log, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_LOCK_temp_pool, &LOCK_temp_pool, MY_MUTEX_INIT_FAST); #ifdef HAVE_OPENSSL mysql_mutex_init(key_LOCK_des_key_file, @@ -5092,6 +5129,7 @@ static int init_server_components() init_global_table_stats(); init_global_index_stats(); + init_update_queries(); /* Allow storage engine to give real error messages */ if (unlikely(ha_init_errors())) @@ -5099,6 +5137,9 @@ static int init_server_components() tc_log= 0; // ha_initialize_handlerton() needs that + if (!opt_abort && ddl_log_initialize()) + unireg_abort(1); + if (plugin_init(&remaining_argc, remaining_argv, (opt_noacl ? PLUGIN_INIT_SKIP_PLUGIN_TABLE : 0) | (opt_abort ? PLUGIN_INIT_SKIP_INITIALIZATION : 0))) @@ -5201,6 +5242,7 @@ static int init_server_components() MARIADB_REMOVED_OPTION("innodb-log-compressed-pages"), MARIADB_REMOVED_OPTION("innodb-log-files-in-group"), MARIADB_REMOVED_OPTION("innodb-log-optimize-ddl"), + MARIADB_REMOVED_OPTION("innodb-log-write-ahead-size"), MARIADB_REMOVED_OPTION("innodb-page-cleaners"), MARIADB_REMOVED_OPTION("innodb-replication-delay"), MARIADB_REMOVED_OPTION("innodb-scrub-log"), @@ -5342,9 +5384,6 @@ static int init_server_components() } #endif - if (ddl_log_initialize()) - unireg_abort(1); - tc_log= get_tc_log_implementation(); if (tc_log->open(opt_bin_log ? opt_bin_logname : opt_tc_log_file)) @@ -5428,12 +5467,15 @@ static int init_server_components() ft_init_stopwords(); init_max_user_conn(); - init_update_queries(); init_global_user_stats(); init_global_client_stats(); if (!opt_bootstrap) servers_init(0); init_status_vars(); + Item_false= new (&read_only_root) Item_bool_static("FALSE", 0); + Item_true= new (&read_only_root) Item_bool_static("TRUE", 1); + DBUG_ASSERT(Item_false); + DBUG_RETURN(0); } @@ -5797,6 +5839,9 @@ int mysqld_main(int argc, char **argv) } #endif /* WITH_WSREP */ + /* Protect read_only_root against writes */ + protect_root(&read_only_root, PROT_READ); + if (opt_bootstrap) { select_thread_in_use= 0; // Allow 'kill' to work @@ -9215,6 +9260,14 @@ PSI_stage_info stage_starting= { 0, "starting", 0}; PSI_stage_info stage_waiting_for_flush= { 0, "Waiting for non trans tables to be flushed", 0}; PSI_stage_info stage_waiting_for_ddl= { 0, "Waiting for DDLs", 0}; +#ifdef WITH_WSREP +// Aditional Galera thread states +PSI_stage_info stage_waiting_isolation= { 0, "Waiting to execute in isolation", 0}; +PSI_stage_info stage_waiting_certification= {0, "Waiting for certification", 0}; +PSI_stage_info stage_waiting_ddl= {0, "Waiting for TOI DDL", 0}; +PSI_stage_info stage_waiting_flow= {0, "Waiting for flow control", 0}; +#endif /* WITH_WSREP */ + PSI_memory_key key_memory_DATE_TIME_FORMAT; PSI_memory_key key_memory_DDL_LOG_MEMORY_ENTRY; PSI_memory_key key_memory_Event_queue_element_for_exec_names; @@ -9434,6 +9487,13 @@ PSI_stage_info *all_server_stages[]= & stage_reading_semi_sync_ack, & stage_waiting_for_deadlock_kill, & stage_starting +#ifdef WITH_WSREP + , + & stage_waiting_isolation, + & stage_waiting_certification, + & stage_waiting_ddl, + & stage_waiting_flow +#endif /* WITH_WSREP */ }; PSI_socket_key key_socket_tcpip, key_socket_unix, key_socket_client_connection; diff --git a/sql/mysqld.h b/sql/mysqld.h index 3014da58b3d..36b17e28eab 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -1,5 +1,5 @@ /* Copyright (c) 2006, 2016, Oracle and/or its affiliates. - Copyright (c) 2010, 2020, MariaDB Corporation. + Copyright (c) 2010, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -100,7 +100,9 @@ extern CHARSET_INFO *error_message_charset_info; extern CHARSET_INFO *character_set_filesystem; -extern MY_BITMAP temp_pool; +void temp_pool_clear_bit(uint bit); +uint temp_pool_set_next(); + extern bool opt_large_files; extern bool opt_update_log, opt_bin_log, opt_error_log, opt_bin_log_compress; extern uint opt_bin_log_compress_min_len; @@ -330,6 +332,8 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_user_conn, key_LOG_LOCK_log, key_master_info_data_lock, key_master_info_run_lock, key_master_info_sleep_lock, key_master_info_start_stop_lock, + key_master_info_start_alter_lock, + key_master_info_start_alter_list_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, key_rpl_group_info_sleep_lock, @@ -677,6 +681,13 @@ extern PSI_stage_info stage_slave_background_process_request; extern PSI_stage_info stage_slave_background_wait_request; extern PSI_stage_info stage_waiting_for_deadlock_kill; extern PSI_stage_info stage_starting; +#ifdef WITH_WSREP +// Aditional Galera thread states +extern PSI_stage_info stage_waiting_isolation; +extern PSI_stage_info stage_waiting_certification; +extern PSI_stage_info stage_waiting_ddl; +extern PSI_stage_info stage_waiting_flow; +#endif /* WITH_WSREP */ #ifdef HAVE_PSI_STATEMENT_INTERFACE /** diff --git a/sql/opt_histogram_json.cc b/sql/opt_histogram_json.cc new file mode 100644 index 00000000000..bea18050a59 --- /dev/null +++ b/sql/opt_histogram_json.cc @@ -0,0 +1,1200 @@ +/* + Copyright (c) 2021, 2022, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "mariadb.h" +#include "sql_base.h" +#include "my_json_writer.h" +#include "sql_statistics.h" +#include "opt_histogram_json.h" + + +/* + @brief + Un-escape a JSON string and save it into *out. + + @detail + There's no way to tell how much space is needed for the output. + Start with a small string and increase its size until json_unescape() + succeeds. +*/ + +static bool json_unescape_to_string(const char *val, int val_len, String* out) +{ + // Make sure 'out' has some memory allocated. + if (!out->alloced_length() && out->alloc(128)) + return true; + + while (1) + { + uchar *buf= (uchar*)out->ptr(); + out->length(out->alloced_length()); + + int res= json_unescape(&my_charset_utf8mb4_bin, + (const uchar*)val, + (const uchar*)val + val_len, + out->charset(), + buf, buf + out->length()); + if (res >= 0) + { + out->length(res); + return false; // Ok + } + + // We get here if the unescaped string didn't fit into memory. + if (out->alloc(out->alloced_length()*2)) + return true; + } +} + + +/* + @brief + Escape a JSON string and save it into *out. + + @detail + There's no way to tell how much space is needed for the output. + Start with a small string and increase its size until json_escape() + succeeds. +*/ + +static int json_escape_to_string(const String *str, String* out) +{ + // Make sure 'out' has some memory allocated. + if (!out->alloced_length() && out->alloc(128)) + return JSON_ERROR_OUT_OF_SPACE; + + while (1) + { + uchar *buf= (uchar*)out->ptr(); + out->length(out->alloced_length()); + const uchar *str_ptr= (const uchar*)str->ptr(); + + int res= json_escape(str->charset(), + str_ptr, + str_ptr + str->length(), + &my_charset_utf8mb4_bin, + buf, buf + out->length()); + if (res >= 0) + { + out->length(res); + return 0; // Ok + } + + if (res != JSON_ERROR_OUT_OF_SPACE) + return res; // Some conversion error + + // Out of space error. Try with a bigger buffer + if (out->alloc(out->alloced_length()*2)) + return JSON_ERROR_OUT_OF_SPACE; + } +} + + +class Histogram_json_builder : public Histogram_builder +{ + Histogram_json_hb *histogram; + /* Number of buckets in the histogram */ + uint hist_width; + + /* + Number of rows that we intend to have in the bucket. That is, this is + + n_rows_in_table / hist_width + + Actual number of rows in the buckets we produce may vary because of + "popular values" and rounding. + */ + longlong bucket_capacity; + + /* Number of the buckets already collected */ + uint n_buckets_collected; + + /* + TRUE means do not try to represent values as UTF-8 text in histogram + storage. Use start_hex/end_hex for all values. + */ + bool force_binary; + + /* Data about the bucket we are filling now */ + struct CurBucket + { + /* Number of values in the bucket so far. */ + longlong size; + + /* Number of distinct values in the bucket */ + int ndv; + }; + CurBucket bucket; + + /* Used to create the JSON representation of the histogram. */ + Json_writer writer; + +public: + + Histogram_json_builder(Histogram_json_hb *hist, Field *col, uint col_len, + ha_rows rows) + : Histogram_builder(col, col_len, rows), histogram(hist) + { + /* + When computing number of rows in the bucket, round it UP. This way, we + will not end up with a histogram that has more buckets than intended. + + We may end up producing a histogram with fewer buckets than intended, but + this is considered tolerable. + */ + bucket_capacity= (longlong)round(rows2double(records) / histogram->get_width() + 0.5); + if (bucket_capacity == 0) + bucket_capacity= 1; + hist_width= histogram->get_width(); + n_buckets_collected= 0; + bucket.ndv= 0; + bucket.size= 0; + force_binary= (col->type() == MYSQL_TYPE_BIT); + + writer.start_object(); + append_histogram_params(); + + writer.add_member(Histogram_json_hb::JSON_NAME).start_array(); + } + + ~Histogram_json_builder() override = default; + +private: + bool bucket_is_empty() { return bucket.ndv == 0; } + + void append_histogram_params() + { + char buf[128]; + String str(buf, sizeof(buf), system_charset_info); + THD *thd= current_thd; + timeval tv= {thd->query_start(), 0}; // we do not need microseconds + + Timestamp(tv).to_datetime(thd).to_string(&str, 0); + writer.add_member("target_histogram_size").add_ull(hist_width); + writer.add_member("collected_at").add_str(str.ptr()); + writer.add_member("collected_by").add_str(server_version); + } + /* + Flush the current bucket out (to JSON output), and set it to be empty. + */ + void finalize_bucket() + { + double fract= (double) bucket.size / records; + writer.add_member("size").add_double(fract); + writer.add_member("ndv").add_ll(bucket.ndv); + writer.end_object(); + n_buckets_collected++; + + bucket.ndv= 0; + bucket.size= 0; + } + + /* + Same as finalize_bucket() but also provide the bucket's end value. + */ + bool finalize_bucket_with_end_value(void *elem) + { + if (append_column_value(elem, false)) + return true; + finalize_bucket(); + return false; + } + + /* + Write the first value group to the bucket. + @param elem The value we are writing + @param cnt The number of such values. + */ + bool start_bucket(void *elem, longlong cnt) + { + DBUG_ASSERT(bucket.size == 0); + writer.start_object(); + if (append_column_value(elem, true)) + return true; + + bucket.ndv= 1; + bucket.size= cnt; + return false; + } + + /* + Append the passed value into the JSON writer as string value + */ + bool append_column_value(void *elem, bool is_start) + { + StringBuffer<MAX_FIELD_WIDTH> val; + + // Get the text representation of the value + column->store_field_value((uchar*) elem, col_length); + String *str= column->val_str(&val); + + // Escape the value for JSON + StringBuffer<MAX_FIELD_WIDTH> escaped_val; + int rc= JSON_ERROR_ILLEGAL_SYMBOL; + if (!force_binary) + { + rc= json_escape_to_string(str, &escaped_val); + if (!rc) + { + writer.add_member(is_start? "start": "end"); + writer.add_str(escaped_val.c_ptr_safe()); + return false; + } + } + if (rc == JSON_ERROR_ILLEGAL_SYMBOL) + { + escaped_val.set_hex(val.ptr(), val.length()); + writer.add_member(is_start? "start_hex": "end_hex"); + writer.add_str(escaped_val.c_ptr_safe()); + return false; + } + return true; + } + + /* + Append a value group of cnt values. + */ + void append_to_bucket(longlong cnt) + { + bucket.ndv++; + bucket.size += cnt; + } + +public: + /* + @brief + Add data to the histogram. + + @detail + The call signals to add a "value group" of elem_cnt rows, each of which + has the same value that is provided in *elem. + + Subsequent next() calls will add values that are greater than the + current one. + + @return + 0 - OK + */ + int next(void *elem, element_count elem_cnt) override + { + counters.next(elem, elem_cnt); + ulonglong count= counters.get_count(); + + /* + Ok, we've got a "value group" of elem_cnt identical values. + + If we take the values from the value group and put them into + the current bucket, how many values will be left after we've + filled the bucket? + */ + longlong overflow= bucket.size + elem_cnt - bucket_capacity; + + /* + Case #1: This value group should be put into a separate bucket, if + A. It fills the current bucket and also fills the next bucket, OR + B. It fills the current bucket, which was empty. + */ + if (overflow >= bucket_capacity || (bucket_is_empty() && overflow >= 0)) + { + // Finalize the current bucket + if (!bucket_is_empty()) + finalize_bucket(); + + // Start/end the separate bucket for this value group. + if (start_bucket(elem, elem_cnt)) + return 1; // OOM + + if (records == count) + { + if (finalize_bucket_with_end_value(elem)) + return 1; + } + else + finalize_bucket(); + } + else if (overflow >= 0) + { + /* + Case #2: is when Case#1 doesn't hold, but we can still fill the + current bucket. + */ + + // If the bucket was empty, it would have been case #1. + DBUG_ASSERT(!bucket_is_empty()); + + /* + Finalize the current bucket. Put there enough values to make it hold + bucket_capacity values. + */ + append_to_bucket(bucket_capacity - bucket.size); + if (records == count && !overflow) + { + if (finalize_bucket_with_end_value(elem)) + return 1; + } + else + finalize_bucket(); + + if (overflow > 0) + { + // Then, start the new bucket with the remaining values. + if (start_bucket(elem, overflow)) + return 1; + } + } + else + { + // Case #3: there's not enough values to fill the current bucket. + if (bucket_is_empty()) + { + if (start_bucket(elem, elem_cnt)) + return 1; + } + else + append_to_bucket(elem_cnt); + } + + if (records == count) + { + // This is the final value group. + if (!bucket_is_empty()) + { + if (finalize_bucket_with_end_value(elem)) + return 1; + } + } + return 0; + } + + /* + @brief + Finalize the creation of histogram + */ + void finalize() override + { + writer.end_array(); + writer.end_object(); + Binary_string *json_string= (Binary_string *) writer.output.get_string(); + histogram->set_json_text(n_buckets_collected, + json_string->c_ptr(), + (size_t)json_string->length()); + } +}; + + +Histogram_builder *Histogram_json_hb::create_builder(Field *col, uint col_len, + ha_rows rows) +{ + return new Histogram_json_builder(this, col, col_len, rows); +} + + +void Histogram_json_hb::init_for_collection(MEM_ROOT *mem_root, + Histogram_type htype_arg, + ulonglong size_arg) +{ + DBUG_ASSERT(htype_arg == JSON_HB); + size= (size_t)size_arg; +} + + +/* + A syntax sugar interface to json_string_t +*/ +class Json_string +{ + json_string_t str; +public: + explicit Json_string(const char *name) + { + json_string_set_str(&str, (const uchar*)name, + (const uchar*)name + strlen(name)); + json_string_set_cs(&str, system_charset_info); + } + json_string_t *get() { return &str; } +}; + + +/* + This [partially] saves the JSON parser state and then can rollback the parser + to it. + + The goal of this is to be able to make multiple json_key_matches() calls: + + Json_saved_parser_state save(je); + if (json_key_matches(je, KEY_NAME_1)) { + ... + return; + } + save.restore_to(je); + if (json_key_matches(je, KEY_NAME_2)) { + ... + } + + This allows one to parse JSON objects where [optional] members come in any + order. +*/ + +class Json_saved_parser_state +{ + const uchar *c_str; + my_wc_t c_next; + int state; +public: + explicit Json_saved_parser_state(const json_engine_t *je) : + c_str(je->s.c_str), + c_next(je->s.c_next), + state(je->state) + {} + void restore_to(json_engine_t *je) + { + je->s.c_str= c_str; + je->s.c_next= c_next; + je->state= state; + } +}; + + +/* + @brief + Read a constant from JSON document and save it in *out. + + @detail + The JSON document stores constant in text form, we need to save it in + KeyTupleFormat. String constants in JSON may be escaped. +*/ + +bool read_bucket_endpoint(json_engine_t *je, Field *field, String *out, + const char **err) +{ + if (json_read_value(je)) + return true; + + if (je->value_type != JSON_VALUE_STRING && + je->value_type != JSON_VALUE_NUMBER) + { + *err= "String or number expected"; + return true; + } + + const char* je_value= (const char*)je->value; + if (je->value_type == JSON_VALUE_STRING && je->value_escaped) + { + StringBuffer<128> unescape_buf; + if (json_unescape_to_string(je_value, je->value_len, &unescape_buf)) + { + *err= "Un-escape error"; + return true; + } + field->store_text(unescape_buf.ptr(), unescape_buf.length(), + unescape_buf.charset()); + } + else + field->store_text(je_value, je->value_len, &my_charset_utf8mb4_bin); + + out->alloc(field->pack_length()); + uint bytes= field->get_key_image((uchar*)out->ptr(), + field->key_length(), Field::itRAW); + out->length(bytes); + return false; +} + + +bool read_hex_bucket_endpoint(json_engine_t *je, Field *field, String *out, + const char **err) +{ + if (json_read_value(je)) + return true; + + if (je->value_type != JSON_VALUE_STRING || je->value_escaped || + (je->value_len & 1)) + { + *err= "Expected a hex string"; + return true; + } + StringBuffer<128> buf; + + for (auto pc= je->value; pc < je->value + je->value_len; pc+=2) + { + int hex_char1= hexchar_to_int(pc[0]); + int hex_char2= hexchar_to_int(pc[1]); + if (hex_char1 == -1 || hex_char2 == -1) + { + *err= "Expected a hex string"; + return true; + } + buf.append((hex_char1 << 4) | hex_char2); + } + + field->store_text(buf.ptr(), buf.length(), field->charset()); + out->alloc(field->pack_length()); + uint bytes= field->get_key_image((uchar*)out->ptr(), + field->key_length(), Field::itRAW); + out->length(bytes); + return false; +} + + +/* + @brief Parse a JSON reprsentation for one histogram bucket + + @param je The JSON parser object + @param field Table field we are using histogram (used to convert + endpoints from text representation to binary) + @param total_size INOUT Fraction of the table rows in the buckets parsed so + far. + @param assigned_last_end OUT TRUE<=> The bucket had "end" members, the + function has saved it in + this->last_bucket_end_endp + @param err OUT If function returns 1, this *may* be set to point to text + describing the error. + + @detail + + Parse a JSON object in this form: + + { "start": "value", "size":nnn.nn, "ndv": nnn, "end": "value"} + + Unknown members are ignored. + + @return + 0 OK + 1 Parse Error + -1 EOF +*/ +int Histogram_json_hb::parse_bucket(json_engine_t *je, Field *field, + double *total_size, + bool *assigned_last_end, + const char **err) +{ + *assigned_last_end= false; + if (json_scan_next(je)) + return 1; + if (je->state != JST_VALUE) + { + if (je->state == JST_ARRAY_END) + return -1; // EOF + else + return 1; // An error + } + + if (json_scan_next(je) || je->state != JST_OBJ_START) + { + *err= "Expected an object in the buckets array"; + return 1; + } + + bool have_start= false; + bool have_size= false; + bool have_ndv= false; + + double size_d; + longlong ndv_ll= 0; + StringBuffer<128> value_buf; + int rc; + + while (!(rc= json_scan_next(je)) && je->state != JST_OBJ_END) + { + Json_saved_parser_state save1(je); + Json_string start_str("start"); + if (json_key_matches(je, start_str.get())) + { + if (read_bucket_endpoint(je, field, &value_buf, err)) + return 1; + + have_start= true; + continue; + } + save1.restore_to(je); + + Json_string size_str("size"); + if (json_key_matches(je, size_str.get())) + { + if (json_read_value(je)) + return 1; + + const char *size= (const char*)je->value_begin; + char *size_end= (char*)je->value_end; + int conv_err; + size_d= my_strtod(size, &size_end, &conv_err); + if (conv_err) + { + *err= ".size member must be a floating-point value"; + return 1; + } + have_size= true; + continue; + } + save1.restore_to(je); + + Json_string ndv_str("ndv"); + if (json_key_matches(je, ndv_str.get())) + { + if (json_read_value(je)) + return 1; + + const char *ndv= (const char*)je->value_begin; + char *ndv_end= (char*)je->value_end; + int conv_err; + ndv_ll= my_strtoll10(ndv, &ndv_end, &conv_err); + if (conv_err) + { + *err= ".ndv member must be an integer value"; + return 1; + } + have_ndv= true; + continue; + } + save1.restore_to(je); + + Json_string end_str("end"); + if (json_key_matches(je, end_str.get())) + { + if (read_bucket_endpoint(je, field, &value_buf, err)) + return 1; + last_bucket_end_endp.assign(value_buf.ptr(), value_buf.length()); + *assigned_last_end= true; + continue; + } + save1.restore_to(je); + + // Less common endoints: + Json_string start_hex_str("start_hex"); + if (json_key_matches(je, start_hex_str.get())) + { + if (read_hex_bucket_endpoint(je, field, &value_buf, err)) + return 1; + + have_start= true; + continue; + } + save1.restore_to(je); + + Json_string end_hex_str("end_hex"); + if (json_key_matches(je, end_hex_str.get())) + { + if (read_hex_bucket_endpoint(je, field, &value_buf, err)) + return 1; + last_bucket_end_endp.assign(value_buf.ptr(), value_buf.length()); + *assigned_last_end= true; + continue; + } + save1.restore_to(je); + + + // Some unknown member. Skip it. + if (json_skip_key(je)) + return 1; + } + + if (rc) + return 1; + + if (!have_start) + { + *err= "\"start\" element not present"; + return 1; + } + if (!have_size) + { + *err= "\"size\" element not present"; + return 1; + } + if (!have_ndv) + { + *err= "\"ndv\" element not present"; + return 1; + } + + *total_size += size_d; + + buckets.push_back({std::string(value_buf.ptr(), value_buf.length()), + *total_size, ndv_ll}); + + return 0; // Ok, continue reading +} + + +/* + @brief + Parse the histogram from its on-disk JSON representation + + @detail + See opt_histogram_json.h, class Histogram_json_hb for description of the + data format. + + @return + false OK + True Error +*/ + +bool Histogram_json_hb::parse(MEM_ROOT *mem_root, const char *db_name, + const char *table_name, Field *field, + Histogram_type type_arg, + const char *hist_data, size_t hist_data_len) +{ + json_engine_t je; + int rc; + const char *err= "JSON parse error"; + double total_size; + int end_element; + bool end_assigned; + DBUG_ENTER("Histogram_json_hb::parse"); + DBUG_ASSERT(type_arg == JSON_HB); + + json_scan_start(&je, &my_charset_utf8mb4_bin, + (const uchar*)hist_data, + (const uchar*)hist_data+hist_data_len); + + if (json_scan_next(&je)) + goto err; + + if (je.state != JST_OBJ_START) + { + err= "Root JSON element must be a JSON object"; + goto err; + } + + while (1) + { + if (json_scan_next(&je)) + goto err; + if (je.state == JST_OBJ_END) + break; // End of object + + if (je.state != JST_KEY) + goto err; // Can' really have this: JSON object has keys in it + + Json_string hist_key_name(JSON_NAME); + if (json_key_matches(&je, hist_key_name.get())) + { + total_size= 0.0; + end_element= -1; + if (json_scan_next(&je)) + goto err; + + if (je.state != JST_ARRAY_START) + { + err= "histogram_hb must contain an array"; + goto err; + } + + while (!(rc= parse_bucket(&je, field, &total_size, &end_assigned, &err))) + { + if (end_assigned && end_element != -1) + end_element= (int)buckets.size(); + } + if (rc > 0) // Got error other than EOF + goto err; + } + else + { + // Some unknown member. Skip it. + if (json_skip_key(&je)) + return 1; + } + } + + if (buckets.size() < 1) + { + err= "Histogram must have at least one bucket"; + goto err; + } + + if (end_element == -1) + { + buckets.back().start_value= last_bucket_end_endp; + } + else if (end_element < (int)buckets.size()) + { + err= ".end is only allowed in the last bucket"; + goto err; + } + + DBUG_RETURN(false); // Ok +err: + THD *thd= current_thd; + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_JSON_HISTOGRAM_PARSE_FAILED, + ER_THD(thd, ER_JSON_HISTOGRAM_PARSE_FAILED), + db_name, table_name, + err, (je.s.c_str - (const uchar*)hist_data)); + sql_print_error(ER_THD(thd, ER_JSON_HISTOGRAM_PARSE_FAILED), + db_name, table_name, err, + (je.s.c_str - (const uchar*)hist_data)); + + DBUG_RETURN(true); +} + + +static +void store_key_image_to_rec_no_null(Field *field, const char *ptr, size_t len) +{ + MY_BITMAP *old_map= dbug_tmp_use_all_columns(field->table, + &field->table->write_set); + field->set_key_image((const uchar*)ptr, (uint)len); + dbug_tmp_restore_column_map(&field->table->write_set, old_map); +} + + +static +double position_in_interval(Field *field, const uchar *key, uint key_len, + const std::string& left, const std::string& right) +{ + double res; + if (field->pos_through_val_str()) + { + StringBuffer<64> buf1, buf2, buf3; + + store_key_image_to_rec_no_null(field, left.data(), left.size()); + String *min_str= field->val_str(&buf1); + /* + Make sure we've saved a copy of the data, not a pointer into the + field->ptr. We will overwrite the contents of field->ptr with the next + store_key_image_to_rec_no_null call + */ + if (&buf1 != min_str) + buf1.copy(*min_str); + else + buf1.copy(); + + store_key_image_to_rec_no_null(field, right.data(), right.size()); + String *max_str= field->val_str(&buf2); + /* Same as above */ + if (&buf2 != max_str) + buf2.copy(*max_str); + else + buf2.copy(); + + store_key_image_to_rec_no_null(field, (const char*)key, key_len); + String *midp_str= field->val_str(&buf3); + + res= pos_in_interval_for_string(field->charset(), + (const uchar*)midp_str->ptr(), midp_str->length(), + (const uchar*)buf1.ptr(), buf1.length(), + (const uchar*)buf2.ptr(), buf2.length()); + } + else + { + store_key_image_to_rec_no_null(field, left.data(), field->key_length()); + double min_val_real= field->val_real(); + + store_key_image_to_rec_no_null(field, right.data(), field->key_length()); + double max_val_real= field->val_real(); + + store_key_image_to_rec_no_null(field, (const char*)key, field->key_length()); + double midp_val_real= field->val_real(); + + res= pos_in_interval_for_double(midp_val_real, min_val_real, max_val_real); + } + return res; +} + + +double Histogram_json_hb::point_selectivity(Field *field, key_range *endpoint, + double avg_sel) +{ + const uchar *key = endpoint->key; + if (field->real_maybe_null()) + key++; + + // If the value is outside of the histogram's range, this will "clip" it to + // first or last bucket. + int endp_cmp; + int idx= find_bucket(field, key, &endp_cmp); + + double sel; + + if (buckets[idx].ndv == 1 && (endp_cmp!=0)) + { + /* + The bucket has a single value and it doesn't match! Return a very + small value. + */ + sel= 0.0; + } + else + { + /* + We get here when: + * The bucket has one value and this is the value we are looking for. + * The bucket has multiple values. Then, assume + */ + sel= (buckets[idx].cum_fract - get_left_fract(idx)) / buckets[idx].ndv; + } + return sel; +} + + +double Histogram_json_hb::get_left_fract(int idx) +{ + if (!idx) + return 0.0; + else + return buckets[idx-1].cum_fract; +} + +std::string& Histogram_json_hb::get_end_value(int idx) +{ + if (idx == (int)buckets.size()-1) + return last_bucket_end_endp; + else + return buckets[idx+1].start_value; +} + +/* + @param field The table field histogram is for. We don't care about the + field's current value, we only need its virtual functions to + perform various operations + @param min_endp Left endpoint, or NULL if there is none + @param max_endp Right endpoint, or NULL if there is none + @param avg_sel Average selectivity of "field=const" equality for this field + + @return + Range selectivity: a number between 0.0 and 1.0. + + @note + This may return 0.0. Adjustments to avoid multiply-by-zero meltdown are + made elsewhere. +*/ + +double Histogram_json_hb::range_selectivity(Field *field, key_range *min_endp, + key_range *max_endp, double avg_sel) +{ + double min, max; + + if (min_endp && !(field->real_maybe_null() && min_endp->key[0])) + { + bool exclusive_endp= (min_endp->flag == HA_READ_AFTER_KEY)? true: false; + const uchar *min_key= min_endp->key; + uint min_key_len= min_endp->length; + if (field->real_maybe_null()) + { + min_key++; + min_key_len--; + } + + // Find the leftmost bucket that contains the lookup value. + // (If the lookup value is to the left of all buckets, find bucket #0) + int endp_cmp; + int idx= find_bucket(field, min_key, &endp_cmp); + + double sel; + // Special handling for buckets with ndv=1: + if (buckets[idx].ndv == 1) + { + if (endp_cmp < 0) + sel= 0.0; + else if (endp_cmp > 0) + sel= 1.0; + else // endp_cmp == 0.0 + sel= (exclusive_endp)? 1.0 : 0.0; + } + else + { + sel= position_in_interval(field, min_key, min_key_len, + buckets[idx].start_value, + get_end_value(idx)); + } + double left_fract= get_left_fract(idx); + min= left_fract + sel * (buckets[idx].cum_fract - left_fract); + } + else + min= 0.0; + + if (max_endp) + { + // The right endpoint cannot be NULL + DBUG_ASSERT(!(field->real_maybe_null() && max_endp->key[0])); + bool inclusive_endp= (max_endp->flag == HA_READ_AFTER_KEY)? true: false; + const uchar *max_key= max_endp->key; + uint max_key_len= max_endp->length; + if (field->real_maybe_null()) + { + max_key++; + max_key_len--; + } + int endp_cmp; + int idx= find_bucket(field, max_key, &endp_cmp); + + if ((endp_cmp == 0) && !inclusive_endp) + { + /* + The range is "col < $CONST" and we've found a bucket starting with + $CONST. + */ + if (idx > 0) + { + // Move to the previous bucket + endp_cmp= 1; + idx--; + } + else + endp_cmp= -1; + } + double sel; + + // Special handling for buckets with ndv=1: + if (buckets[idx].ndv == 1) + { + if (endp_cmp < 0) + sel= 0.0; + else if (endp_cmp > 0) + sel= 1.0; + else // endp_cmp == 0.0 + sel= inclusive_endp? 1.0 : 0.0; + } + else + { + sel= position_in_interval(field, max_key, max_key_len, + buckets[idx].start_value, + get_end_value(idx)); + } + double left_fract= get_left_fract(idx); + max= left_fract + sel * (buckets[idx].cum_fract - left_fract); + } + else + max= 1.0; + + if (min > max) + { + /* + This can happen due to rounding errors. + + What is the acceptable error size? Json_writer::add_double() uses + %.11lg format. This gives 9 digits after the dot. A histogram may have + hundreds of buckets, let's multiply the error by 1000. 9-3=6 + */ + DBUG_ASSERT(max < min + 1e-6); + max= min; + } + return max - min; +} + + +void Histogram_json_hb::serialize(Field *field) +{ + field->store(json_text.data(), json_text.size(), &my_charset_bin); +} + + +#ifndef DBUG_OFF +static int SGN(int x) +{ + if (!x) + return 0; + return (x < 0)? -1 : 1; +} +#endif + + +/* + @brief + Find the leftmost histogram bucket such that "lookup_val >= start_value". + + @param field Field object (used to do value comparisons) + @param lookup_val The lookup value in KeyTupleFormat. + @param cmp OUT How the lookup_val compares to found_bucket.left_bound: + 0 - lookup_val == bucket.left_bound + >0 - lookup_val > bucket.left_bound (the most typical) + <0 - lookup_val < bucket.left_bound. This can only happen + for the first bucket, for all other buckets we would just + pick the previous bucket and have cmp>=0. + @return + The bucket index +*/ + +int Histogram_json_hb::find_bucket(const Field *field, const uchar *lookup_val, + int *cmp) +{ + int res; + int low= 0; + int high= (int)buckets.size() - 1; + *cmp= 1; // By default, (bucket[retval].start_value < *lookup_val) + + while (low + 1 < high) + { + int middle= (low + high) / 2; + res= field->key_cmp((uchar*)buckets[middle].start_value.data(), lookup_val); + if (!res) + { + *cmp= res; + low= middle; + goto end; + } + else if (res < 0) + low= middle; + else //res > 0 + high= middle; + } + + /* + If low and high were assigned a value in the above loop and we got here, + then the following holds: + + bucket[low].start_value < lookup_val < bucket[high].start_value + + Besides that, there are two special cases: low=0 and high=last_bucket. + Handle them below. + */ + if (low == 0) + { + res= field->key_cmp(lookup_val, (uchar*)buckets[0].start_value.data()); + if (res <= 0) + *cmp= res; + else // res>0, lookup_val > buckets[0].start_value + { + res= field->key_cmp(lookup_val, (uchar*)buckets[high].start_value.data()); + if (res >= 0) // lookup_val >= buckets[high].start_value + { + // Move to that bucket + low= high; + *cmp= res; + } + else + *cmp= 1; + } + } + else if (high == (int)buckets.size() - 1) + { + res= field->key_cmp(lookup_val, (uchar*)buckets[high].start_value.data()); + if (res >= 0) + { + // Ok the value is in the last bucket. + *cmp= res; + low= high; + } + else + { + // The value is in the 'low' bucket. + res= field->key_cmp(lookup_val, (uchar*)buckets[low].start_value.data()); + *cmp= res; + } + } + +end: + // Verification: *cmp has correct value + DBUG_ASSERT(SGN(*cmp) == + SGN(field->key_cmp(lookup_val, + (uchar*)buckets[low].start_value.data()))); + // buckets[low] <= lookup_val, with one exception of the first bucket. + DBUG_ASSERT(low == 0 || + field->key_cmp((uchar*)buckets[low].start_value.data(), lookup_val)<= 0); + // buckets[low+1] > lookup_val, with one exception of the last bucket + DBUG_ASSERT(low == (int)buckets.size()-1 || + field->key_cmp((uchar*)buckets[low+1].start_value.data(), lookup_val)> 0); + return low; +} diff --git a/sql/opt_histogram_json.h b/sql/opt_histogram_json.h new file mode 100644 index 00000000000..e9b69869f4b --- /dev/null +++ b/sql/opt_histogram_json.h @@ -0,0 +1,148 @@ +/* + Copyright (c) 2021, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "sql_statistics.h" + +/* + An equi-height histogram which stores real values for bucket bounds. + + Handles @@histogram_type=JSON_HB + + Histogram format in JSON: + + { + // The next three are saved but not currently analyzed: + "target_histogram_size": nnn, + "collected_at": "(date and time)", + "collected_by": "(server version)", + + "histogram_hb": [ + { "start": "value", "size":nnn.nn, "ndv": nnn }, + ... + + // Optionally, start and/or end can be replaced with _hex variant + { "start_hex: "value", "size":nnn.nn, "ndv":nnn}, + + ... + { "start": "value", "size":nnn.nn, "ndv": nnn, "end": "value"}, + ] + } + + Histogram is a JSON object. It has some global properties and "histogram_hb" + member whose value is a JSON array of histogram buckets. + + Each bucket is an object with these members: + "start" - the first value in the bucket. + "size" - fraction of table rows that is contained in the bucket. + "ndv" - Number of Distinct Values in the bucket. + "end" - Optionally, the last value in the bucket. + + A bucket is a single-point bucket if it has ndv=1. + + Most buckets have no "end" member: the bucket is assumed to contain all + values up to the "start" of the next bucket. + + The exception is single-point buckets where last value is the same as the + first value. + + start/end can be replaced with start_hex/end_hex. In _hex variant, the + constant is encoded in hex. This encoding is used to handle so called + "unassigned characters": some non-UTF8 charsets have byte combinations that + are not mapped to any UTF8 character. +*/ + +class Histogram_json_hb : public Histogram_base +{ + size_t size; /* Number of elements in the histogram */ + + /* Collection-time only: collected histogram in the JSON form. */ + std::string json_text; + + struct Bucket + { + // The left endpoint in KeyTupleFormat. The endpoint is inclusive, this + // value is in this bucket. + std::string start_value; + + // Cumulative fraction: The fraction of table rows that fall into this + // and preceding buckets. + double cum_fract; + + // Number of distinct values in the bucket. + longlong ndv; + }; + + std::vector<Bucket> buckets; + + std::string last_bucket_end_endp; + +public: + static constexpr const char* JSON_NAME="histogram_hb"; + + bool parse(MEM_ROOT *mem_root, const char *db_name, const char *table_name, + Field *field, Histogram_type type_arg, + const char *hist_data, size_t hist_data_len) override; + + void serialize(Field *field) override; + + Histogram_builder *create_builder(Field *col, uint col_len, + ha_rows rows) override; + + // returns number of buckets in the histogram + uint get_width() override + { + return (uint)size; + } + + Histogram_type get_type() override + { + return JSON_HB; + } + + /* + @brief + This used to be the size of the histogram on disk, which was redundant + (one can check the size directly). Return the number of buckets instead. + */ + uint get_size() override + { + return (uint)size; + } + + void init_for_collection(MEM_ROOT *mem_root, Histogram_type htype_arg, + ulonglong size) override; + + double point_selectivity(Field *field, key_range *endpoint, + double avg_sel) override; + double range_selectivity(Field *field, key_range *min_endp, + key_range *max_endp, double avg_sel) override; + + void set_json_text(ulonglong sz, const char *json_text_arg, + size_t json_text_len) + { + size= (size_t) sz; + json_text.assign(json_text_arg, json_text_len); + } + +private: + int parse_bucket(json_engine_t *je, Field *field, double *cumulative_size, + bool *assigned_last_end, const char **err); + + double get_left_fract(int idx); + std::string& get_end_value(int idx); + int find_bucket(const Field *field, const uchar *lookup_val, int *cmp); +}; + diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 3bbb6144c0b..a32d4cfe24e 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2008, 2020, MariaDB + Copyright (c) 2008, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1308,7 +1308,7 @@ QUICK_RANGE_SELECT::QUICK_RANGE_SELECT(THD *thd, TABLE *table, uint key_nr, *create_error= 1; } else - my_bitmap_init(&column_bitmap, bitmap, head->s->fields, FALSE); + my_bitmap_init(&column_bitmap, bitmap, head->s->fields); DBUG_VOID_RETURN; } @@ -1906,7 +1906,7 @@ inline void SEL_ARG::make_root() weight= 1 + (next_key_part? next_key_part->weight : 0); } -SEL_ARG::SEL_ARG(Field *f,const uchar *min_value_arg, +SEL_ARG::SEL_ARG(Field *f, const uchar *min_value_arg, const uchar *max_value_arg) :min_flag(0), max_flag(0), maybe_flag(0), maybe_null(f->real_maybe_null()), elements(1), use_count(1), field(f), min_value((uchar*) min_value_arg), @@ -1921,7 +1921,8 @@ SEL_ARG::SEL_ARG(Field *field_,uint8 part_, uchar *min_value_, uchar *max_value_, uint8 min_flag_,uint8 max_flag_,uint8 maybe_flag_) :min_flag(min_flag_),max_flag(max_flag_),maybe_flag(maybe_flag_), - part(part_),maybe_null(field_->real_maybe_null()), elements(1),use_count(1), + part(part_),maybe_null(field_->real_maybe_null()), + elements(1),use_count(1), field(field_), min_value(min_value_), max_value(max_value_), next(0),prev(0),next_key_part(0),color(BLACK),type(KEY_RANGE), weight(1) { @@ -1969,7 +1970,8 @@ public: but we don't know if rounding or truncation happened (as some Field::store() do not report minor data changes). */ - SEL_ARG_LT(THD *thd, const uchar *key, Field *field, Item *value) + SEL_ARG_LT(THD *thd, const uchar *key, Field *field, + Item *value) :SEL_ARG_LE(key, field) { if (stored_field_cmp_to_item(thd, field, value) == 0) @@ -2061,7 +2063,8 @@ SEL_ARG *SEL_ARG::clone(RANGE_OPT_PARAM *param, SEL_ARG *new_parent, } else { - if (!(tmp= new (param->mem_root) SEL_ARG(field,part, min_value,max_value, + if (!(tmp= new (param->mem_root) SEL_ARG(field, part, + min_value, max_value, min_flag, max_flag, maybe_flag))) return 0; // OOM tmp->parent=new_parent; @@ -2579,7 +2582,7 @@ static int fill_used_fields_bitmap(PARAM *param) param->fields_bitmap_size= table->s->column_bitmap_size; if (!(tmp= (my_bitmap_map*) alloc_root(param->mem_root, param->fields_bitmap_size)) || - my_bitmap_init(¶m->needed_fields, tmp, table->s->fields, FALSE)) + my_bitmap_init(¶m->needed_fields, tmp, table->s->fields)) return 1; bitmap_copy(¶m->needed_fields, table->read_set); @@ -2808,7 +2811,6 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, trace_idx_details.add("usable", false).add("cause", "fulltext"); continue; // ToDo: ft-keys in non-ft ranges, if possible SerG } - trace_idx_details.add("usable", true); param.key[param.keys]=key_parts; key_part_info= key_info->key_part; @@ -2830,6 +2832,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, key_parts->flag= (uint8) key_part_info->key_part_flag; trace_keypart.add(key_parts->field->field_name); } + trace_keypart.end(); param.real_keynr[param.keys++]=idx; if (cur_key_len > max_key_len) max_key_len= cur_key_len; @@ -3243,6 +3246,7 @@ double records_in_column_ranges(PARAM *param, uint idx, seq.keyno= idx; seq.real_keyno= MAX_KEY; + seq.key_parts= param->key[idx]; seq.param= param; seq.start= tree; seq.is_ror_scan= FALSE; @@ -3281,7 +3285,10 @@ double records_in_column_ranges(PARAM *param, uint idx, break; } total_rows += rows; - } + } + if (total_rows == 0) + total_rows= MY_MIN(1, rows2double(param->table->stat_records())); + return total_rows; } @@ -3370,7 +3377,7 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond) my_bitmap_map* buf; if (!(buf= (my_bitmap_map*)thd->alloc(table->s->column_bitmap_size))) DBUG_RETURN(TRUE); - my_bitmap_init(&handled_columns, buf, table->s->fields, FALSE); + my_bitmap_init(&handled_columns, buf, table->s->fields); /* Calculate the selectivity of the range conditions supported by indexes. @@ -4161,7 +4168,7 @@ static int find_used_partitions_imerge_list(PART_PRUNE_PARAM *ppar, */ return find_used_partitions_imerge(ppar, merges.head()); } - my_bitmap_init(&all_merges, bitmap_buf, n_bits, FALSE); + my_bitmap_init(&all_merges, bitmap_buf, n_bits); bitmap_set_prefix(&all_merges, n_bits); List_iterator<SEL_IMERGE> it(merges); @@ -4435,12 +4442,14 @@ int find_used_partitions(PART_PRUNE_PARAM *ppar, SEL_ARG *key_tree) key_tree->next_key_part->store_min_key(ppar->key, &tmp_min_key, &tmp_min_flag, - ppar->last_part_partno); + ppar->last_part_partno, + true); if (!tmp_max_flag) key_tree->next_key_part->store_max_key(ppar->key, &tmp_max_key, &tmp_max_flag, - ppar->last_part_partno); + ppar->last_part_partno, + false); flag= tmp_min_flag | tmp_max_flag; } else @@ -4817,8 +4826,7 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar) uint32 bufsize= bitmap_buffer_size(ppar->part_info->num_subparts); if (!(buf= (my_bitmap_map*) alloc_root(alloc, bufsize))) return TRUE; - my_bitmap_init(&ppar->subparts_bitmap, buf, ppar->part_info->num_subparts, - FALSE); + my_bitmap_init(&ppar->subparts_bitmap, buf, ppar->part_info->num_subparts); } range_par->key_parts= key_part; Field **field= (ppar->part_fields)? part_info->part_field_array : @@ -5645,7 +5653,7 @@ bool create_fields_bitmap(PARAM *param, MY_BITMAP *fields_bitmap) if (!(bitmap_buf= (my_bitmap_map *) alloc_root(param->mem_root, param->fields_bitmap_size))) return TRUE; - if (my_bitmap_init(fields_bitmap, bitmap_buf, param->table->s->fields, FALSE)) + if (my_bitmap_init(fields_bitmap, bitmap_buf, param->table->s->fields)) return TRUE; return FALSE; @@ -6557,7 +6565,7 @@ ROR_SCAN_INFO *make_ror_scan(const PARAM *param, int idx, SEL_ARG *sel_arg) DBUG_RETURN(NULL); if (my_bitmap_init(&ror_scan->covered_fields, bitmap_buf, - param->table->s->fields, FALSE)) + param->table->s->fields)) DBUG_RETURN(NULL); bitmap_clear_all(&ror_scan->covered_fields); @@ -6674,8 +6682,7 @@ ROR_INTERSECT_INFO* ror_intersect_init(const PARAM *param) if (!(buf= (my_bitmap_map*) alloc_root(param->mem_root, param->fields_bitmap_size))) return NULL; - if (my_bitmap_init(&info->covered_fields, buf, param->table->s->fields, - FALSE)) + if (my_bitmap_init(&info->covered_fields, buf, param->table->s->fields)) return NULL; info->is_covering= FALSE; info->index_scan_costs= 0.0; @@ -7320,7 +7327,7 @@ TRP_ROR_INTERSECT *get_best_covering_ror_intersect(PARAM *param, param->fields_bitmap_size); if (!covered_fields->bitmap || my_bitmap_init(covered_fields, covered_fields->bitmap, - param->table->s->fields, FALSE)) + param->table->s->fields)) DBUG_RETURN(0); bitmap_clear_all(covered_fields); @@ -11538,6 +11545,7 @@ ha_rows check_quick_select(PARAM *param, uint idx, bool index_only, seq.keyno= idx; seq.real_keyno= keynr; + seq.key_parts= param->key[idx]; seq.param= param; seq.start= tree; @@ -11789,6 +11797,46 @@ get_quick_select(PARAM *param,uint idx,SEL_ARG *key_tree, uint mrr_flags, } +void SEL_ARG::store_next_min_max_keys(KEY_PART *key, + uchar **cur_min_key, uint *cur_min_flag, + uchar **cur_max_key, uint *cur_max_flag, + int *min_part, int *max_part) +{ + DBUG_ASSERT(next_key_part); + const bool asc = !(key[next_key_part->part].flag & HA_REVERSE_SORT); + + if (!get_min_flag(key)) + { + if (asc) + { + *min_part += next_key_part->store_min_key(key, cur_min_key, + cur_min_flag, MAX_KEY, true); + } + else + { + uint tmp_flag = invert_min_flag(*cur_min_flag); + *min_part += next_key_part->store_max_key(key, cur_min_key, &tmp_flag, + MAX_KEY, true); + *cur_min_flag = invert_max_flag(tmp_flag); + } + } + if (!get_max_flag(key)) + { + if (asc) + { + *max_part += next_key_part->store_max_key(key, cur_max_key, + cur_max_flag, MAX_KEY, false); + } + else + { + uint tmp_flag = invert_max_flag(*cur_max_flag); + *max_part += next_key_part->store_min_key(key, cur_max_key, &tmp_flag, + MAX_KEY, false); + *cur_max_flag = invert_min_flag(tmp_flag); + } + } +} + /* ** Fix this to get all possible sub_ranges */ @@ -11802,17 +11850,20 @@ get_quick_keys(PARAM *param,QUICK_RANGE_SELECT *quick,KEY_PART *key, int min_part= key_tree->part-1, // # of keypart values in min_key buffer max_part= key_tree->part-1; // # of keypart values in max_key buffer - if (key_tree->left != &null_element) + const bool asc = !(key[key_tree->part].flag & HA_REVERSE_SORT); + SEL_ARG *next_tree = asc ? key_tree->left : key_tree->right; + if (next_tree != &null_element) { - if (get_quick_keys(param,quick,key,key_tree->left, + if (get_quick_keys(param,quick,key,next_tree, min_key,min_key_flag, max_key, max_key_flag)) return 1; } uchar *tmp_min_key=min_key,*tmp_max_key=max_key; - min_part+= key_tree->store_min(key[key_tree->part].store_length, - &tmp_min_key,min_key_flag); - max_part+= key_tree->store_max(key[key_tree->part].store_length, - &tmp_max_key,max_key_flag); + + key_tree->store_min_max(key, key[key_tree->part].store_length, + &tmp_min_key, min_key_flag, + &tmp_max_key, max_key_flag, + &min_part, &max_part); if (key_tree->next_key_part && key_tree->next_key_part->type == SEL_ARG::KEY_RANGE && @@ -11822,31 +11873,40 @@ get_quick_keys(PARAM *param,QUICK_RANGE_SELECT *quick,KEY_PART *key, memcmp(min_key, max_key, (uint)(tmp_max_key - max_key))==0 && key_tree->min_flag==0 && key_tree->max_flag==0) { + // psergey-note: simplified the parameters below as follows: + // min_key_flag | key_tree->min_flag -> min_key_flag + // max_key_flag | key_tree->max_flag -> max_key_flag if (get_quick_keys(param,quick,key,key_tree->next_key_part, - tmp_min_key, min_key_flag | key_tree->min_flag, - tmp_max_key, max_key_flag | key_tree->max_flag)) + tmp_min_key, min_key_flag, + tmp_max_key, max_key_flag)) return 1; goto end; // Ugly, but efficient } { - uint tmp_min_flag=key_tree->min_flag,tmp_max_flag=key_tree->max_flag; - if (!tmp_min_flag) - min_part+= key_tree->next_key_part->store_min_key(key, - &tmp_min_key, - &tmp_min_flag, - MAX_KEY); - if (!tmp_max_flag) - max_part+= key_tree->next_key_part->store_max_key(key, - &tmp_max_key, - &tmp_max_flag, - MAX_KEY); + uint tmp_min_flag= key_tree->get_min_flag(key); + uint tmp_max_flag= key_tree->get_max_flag(key); + + key_tree->store_next_min_max_keys(key, + &tmp_min_key, &tmp_min_flag, + &tmp_max_key, &tmp_max_flag, + &min_part, &max_part); flag=tmp_min_flag | tmp_max_flag; } } else { - flag = (key_tree->min_flag & GEOM_FLAG) ? - key_tree->min_flag : key_tree->min_flag | key_tree->max_flag; + if (asc) + { + flag= (key_tree->min_flag & GEOM_FLAG) ? key_tree->min_flag: + (key_tree->min_flag | + key_tree->max_flag); + } + else + { + // Invert flags for DESC keypart + flag= invert_min_flag(key_tree->min_flag) | + invert_max_flag(key_tree->max_flag); + } } /* @@ -11907,8 +11967,9 @@ get_quick_keys(PARAM *param,QUICK_RANGE_SELECT *quick,KEY_PART *key, return 1; end: - if (key_tree->right != &null_element) - return get_quick_keys(param,quick,key,key_tree->right, + next_tree= asc ? key_tree->right : key_tree->left; + if (next_tree != &null_element) + return get_quick_keys(param,quick,key,next_tree, min_key,min_key_flag, max_key,max_key_flag); return 0; @@ -12630,10 +12691,10 @@ int QUICK_RANGE_SELECT::reset() if (!mrr_buf_desc) empty_buf.buffer= empty_buf.buffer_end= empty_buf.end_of_used_area= NULL; - - error= file->multi_range_read_init(&seq_funcs, (void*)this, ranges.elements, - mrr_flags, mrr_buf_desc? mrr_buf_desc: - &empty_buf); + + error= file->multi_range_read_init(&seq_funcs, (void*)this, + (uint)ranges.elements, mrr_flags, + mrr_buf_desc? mrr_buf_desc: &empty_buf); err: /* Restore bitmaps set on entry */ if (in_ror_merged_scan) @@ -12745,7 +12806,7 @@ int QUICK_RANGE_SELECT::get_next_prefix(uint prefix_length, } } - uint count= ranges.elements - (uint)(cur_range - (QUICK_RANGE**) ranges.buffer); + size_t count= ranges.elements - (size_t)(cur_range - (QUICK_RANGE**) ranges.buffer); if (count == 0) { /* Ranges have already been used up before. None is left for read. */ @@ -12790,7 +12851,7 @@ int QUICK_RANGE_SELECT_GEOM::get_next() DBUG_RETURN(result); } - uint count= ranges.elements - (uint)(cur_range - (QUICK_RANGE**) ranges.buffer); + size_t count= ranges.elements - (size_t)(cur_range - (QUICK_RANGE**) ranges.buffer); if (count == 0) { /* Ranges have already been used up before. None is left for read. */ @@ -12831,9 +12892,9 @@ int QUICK_RANGE_SELECT_GEOM::get_next() bool QUICK_RANGE_SELECT::row_in_ranges() { QUICK_RANGE *res; - uint min= 0; - uint max= ranges.elements - 1; - uint mid= (max + min)/2; + size_t min= 0; + size_t max= ranges.elements - 1; + size_t mid= (max + min)/2; while (min != max) { @@ -13037,24 +13098,25 @@ int QUICK_RANGE_SELECT::cmp_next(QUICK_RANGE *range_arg) key+= store_length, key_part++) { int cmp; + bool reverse= MY_TEST(key_part->flag & HA_REVERSE_SORT); store_length= key_part->store_length; if (key_part->null_bit) { if (*key) { if (!key_part->field->is_null()) - return 1; + return reverse ? 0 : 1; continue; } else if (key_part->field->is_null()) - return 0; + return reverse ? 1 : 0; key++; // Skip null byte store_length--; } if ((cmp=key_part->field->key_cmp(key, key_part->length)) < 0) - return 0; + return reverse ? 1 : 0; if (cmp > 0) - return 1; + return reverse ? 0 : 1; } return (range_arg->flag & NEAR_MAX) ? 1 : 0; // Exact match } @@ -13781,6 +13843,17 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time) cause= "not covering"; goto next_index; } + + { + for (uint i= 0; i < table->actual_n_key_parts(cur_index_info); i++) + { + if (cur_index_info->key_part[i].key_part_flag & HA_REVERSE_SORT) + { + cause="Reverse-ordered (not supported yet)"; + goto next_index; + } + } + } /* This function is called on the precondition that the index is covering. @@ -15830,7 +15903,7 @@ int QUICK_GROUP_MIN_MAX_SELECT::next_max_in_range() DBUG_ASSERT(min_max_ranges.elements > 0); - for (uint range_idx= min_max_ranges.elements; range_idx > 0; range_idx--) + for (size_t range_idx= min_max_ranges.elements; range_idx > 0; range_idx--) { /* Search from the right-most range to the left. */ get_dynamic(&min_max_ranges, (uchar*)&cur_range, range_idx - 1); @@ -16377,7 +16450,7 @@ void QUICK_GROUP_MIN_MAX_SELECT::dbug_dump(int indent, bool verbose) } if (min_max_ranges.elements > 0) { - fprintf(DBUG_FILE, "%*susing %d quick_ranges for MIN/MAX:\n", + fprintf(DBUG_FILE, "%*susing %zu quick_ranges for MIN/MAX:\n", indent, "", min_max_ranges.elements); } } @@ -16518,6 +16591,7 @@ static void trace_ranges(Json_writer_array *range_trace, uint n_key_parts= param->table->actual_n_key_parts(keyinfo); DBUG_ASSERT(range_trace->trace_started()); seq.keyno= idx; + seq.key_parts= param->key[idx]; seq.real_keyno= param->real_keynr[idx]; seq.param= param; seq.start= keypart; @@ -16595,6 +16669,8 @@ void print_keyparts_name(String *out, const KEY_PART_INFO *key_part, else out->append(STRING_WITH_LEN(",")); out->append(key_part->field->field_name); + if (key_part->key_part_flag & HA_REVERSE_SORT) + out->append(STRING_WITH_LEN(" DESC")); } else break; diff --git a/sql/opt_range.h b/sql/opt_range.h index 6e1dab84e47..a505cd09ea4 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -54,6 +54,33 @@ struct KEY_PART { }; +/** + A helper function to invert min flags to max flags for DESC key parts. + It changes NEAR_MIN, NO_MIN_RANGE to NEAR_MAX, NO_MAX_RANGE appropriately +*/ + +inline uint invert_min_flag(uint min_flag) +{ + uint max_flag_out = min_flag & ~(NEAR_MIN | NO_MIN_RANGE); + if (min_flag & NEAR_MIN) max_flag_out |= NEAR_MAX; + if (min_flag & NO_MIN_RANGE) max_flag_out |= NO_MAX_RANGE; + return max_flag_out; +} + + +/** + A helper function to invert max flags to min flags for DESC key parts. + It changes NEAR_MAX, NO_MAX_RANGE to NEAR_MIN, NO_MIN_RANGE appropriately +*/ + +inline uint invert_max_flag(uint max_flag) +{ + uint min_flag_out = max_flag & ~(NEAR_MAX | NO_MAX_RANGE); + if (max_flag & NEAR_MAX) min_flag_out |= NEAR_MIN; + if (max_flag & NO_MAX_RANGE) min_flag_out |= NO_MIN_RANGE; + return min_flag_out; +} + class RANGE_OPT_PARAM; /* A construction block of the SEL_ARG-graph. @@ -267,6 +294,8 @@ class RANGE_OPT_PARAM; - it is a lot easier to compute than computing the number of ranges, - it can be updated incrementally when performing AND/OR operations on parts of the graph. + + 6. For handling DESC keyparts, See HowRangeOptimizerHandlesDescKeyparts */ class SEL_ARG :public Sql_alloc @@ -326,11 +355,15 @@ public: SEL_ARG() = default; SEL_ARG(SEL_ARG &); - SEL_ARG(Field *,const uchar *, const uchar *); - SEL_ARG(Field *field, uint8 part, uchar *min_value, uchar *max_value, + SEL_ARG(Field *, const uchar *, const uchar *); + SEL_ARG(Field *field, uint8 part, + uchar *min_value, uchar *max_value, uint8 min_flag, uint8 max_flag, uint8 maybe_flag); + + /* This is used to construct degenerate SEL_ARGS like ALWAYS, IMPOSSIBLE, etc */ SEL_ARG(enum Type type_arg) - :min_flag(0), max_part_no(0) /* first key part means 1. 0 mean 'no parts'*/, + :min_flag(0), + max_part_no(0) /* first key part means 1. 0 mean 'no parts'*/, elements(1),use_count(1),left(0),right(0), next_key_part(0), color(BLACK), type(type_arg), weight(1) {} @@ -408,13 +441,14 @@ public: { new_max=arg->max_value; flag_max=arg->max_flag; } - return new (thd->mem_root) SEL_ARG(field, part, new_min, new_max, flag_min, + return new (thd->mem_root) SEL_ARG(field, part, + new_min, new_max, flag_min, flag_max, MY_TEST(maybe_flag && arg->maybe_flag)); } SEL_ARG *clone_first(SEL_ARG *arg) { // min <= X < arg->min - return new SEL_ARG(field,part, min_value, arg->min_value, + return new SEL_ARG(field, part, min_value, arg->min_value, min_flag, arg->min_flag & NEAR_MIN ? 0 : NEAR_MAX, maybe_flag | arg->maybe_flag); } @@ -503,6 +537,57 @@ public: return 0; } + /* Save minimum and maximum, taking index order into account */ + void store_min_max(KEY_PART *kp, + uint length, + uchar **min_key, uint min_flag, + uchar **max_key, uint max_flag, + int *min_part, int *max_part) + { + if (kp[part].flag & HA_REVERSE_SORT) { + *max_part += store_min(length, max_key, min_flag); + *min_part += store_max(length, min_key, max_flag); + } else { + *min_part += store_min(length, min_key, min_flag); + *max_part += store_max(length, max_key, max_flag); + } + } + /* + Get the flag for range's starting endpoint, taking index order into + account. + */ + uint get_min_flag(KEY_PART *kp) + { + return (kp[part].flag & HA_REVERSE_SORT)? invert_max_flag(max_flag) : min_flag; + } + /* + Get the flag for range's starting endpoint, taking index order into + account. + */ + uint get_max_flag(KEY_PART *kp) + { + return (kp[part].flag & HA_REVERSE_SORT)? invert_min_flag(min_flag) : max_flag ; + } + /* Get the previous interval, taking index order into account */ + inline SEL_ARG* index_order_prev(KEY_PART *kp) + { + return (kp[part].flag & HA_REVERSE_SORT)? next : prev; + } + /* Get the next interval, taking index order into account */ + inline SEL_ARG* index_order_next(KEY_PART *kp) + { + return (kp[part].flag & HA_REVERSE_SORT)? prev : next; + } + + /* + Produce a single multi-part interval, taking key part ordering into + account. + */ + void store_next_min_max_keys(KEY_PART *key, uchar **cur_min_key, + uint *cur_min_flag, uchar **cur_max_key, + uint *cur_max_flag, int *min_part, + int *max_part); + /* Returns a number of keypart values appended to the key buffer for min key and max key. This function is used by both Range @@ -515,7 +600,8 @@ public: int store_min_key(KEY_PART *key, uchar **range_key, uint *range_key_flag, - uint last_part) + uint last_part, + bool start_key) { SEL_ARG *key_tree= first(); uint res= key_tree->store_min(key[key_tree->part].store_length, @@ -524,15 +610,26 @@ public: if (!res) return 0; *range_key_flag|= key_tree->min_flag; - if (key_tree->next_key_part && - key_tree->next_key_part->type == SEL_ARG::KEY_RANGE && + SEL_ARG *nkp= key_tree->next_key_part; + if (nkp && nkp->type == SEL_ARG::KEY_RANGE && key_tree->part != last_part && - key_tree->next_key_part->part == key_tree->part+1 && + nkp->part == key_tree->part+1 && !(*range_key_flag & (NO_MIN_RANGE | NEAR_MIN))) - res+= key_tree->next_key_part->store_min_key(key, - range_key, - range_key_flag, - last_part); + { + const bool asc = !(key[key_tree->part].flag & HA_REVERSE_SORT); + if (start_key == asc) + { + res+= nkp->store_min_key(key, range_key, range_key_flag, last_part, + start_key); + } + else + { + uint tmp_flag = invert_min_flag(*range_key_flag); + res += nkp->store_max_key(key, range_key, &tmp_flag, last_part, + start_key); + *range_key_flag = invert_max_flag(tmp_flag); + } + } return res; } @@ -540,7 +637,8 @@ public: int store_max_key(KEY_PART *key, uchar **range_key, uint *range_key_flag, - uint last_part) + uint last_part, + bool start_key) { SEL_ARG *key_tree= last(); uint res=key_tree->store_max(key[key_tree->part].store_length, @@ -548,15 +646,26 @@ public: if (!res) return 0; *range_key_flag|= key_tree->max_flag; - if (key_tree->next_key_part && - key_tree->next_key_part->type == SEL_ARG::KEY_RANGE && + SEL_ARG *nkp= key_tree->next_key_part; + if (nkp && nkp->type == SEL_ARG::KEY_RANGE && key_tree->part != last_part && - key_tree->next_key_part->part == key_tree->part+1 && + nkp->part == key_tree->part+1 && !(*range_key_flag & (NO_MAX_RANGE | NEAR_MAX))) - res+= key_tree->next_key_part->store_max_key(key, - range_key, - range_key_flag, - last_part); + { + const bool asc = !(key[key_tree->part].flag & HA_REVERSE_SORT); + if ((!start_key && asc) || (start_key && !asc)) + { + res += nkp->store_max_key(key, range_key, range_key_flag, last_part, + start_key); + } + else + { + uint tmp_flag = invert_max_flag(*range_key_flag); + res += nkp->store_min_key(key, range_key, &tmp_flag, last_part, + start_key); + *range_key_flag = invert_min_flag(tmp_flag); + } + } return res; } @@ -660,6 +769,73 @@ public: SEL_ARG *clone_tree(RANGE_OPT_PARAM *param); }; +/* + HowRangeOptimizerHandlesDescKeyparts + ==================================== + + Starting with MySQL-8.0 and MariaDB 10.8, index key parts may be descending, + for example: + + INDEX idx1(col1, col2 DESC, col3, col4 DESC) + + Range Optimizer handles this as follows: + + Other than that, the SEL_ARG graph is built without any regard to DESC + keyparts. + + For example, for an index + + INDEX idx2(kp1 DESC, kp2) + + and range + + kp1 BETWEEN 10 and 20 (RANGE-1) + + the SEL_ARG will have min_value=10, max_value=20 + + The ordering of key parts is taken into account when SEL_ARG graph is + linearized to ranges, in sel_arg_range_seq_next() and get_quick_keys(). + + The storage engine expects the first bound to be the first in the index and + the last bound to be the last, that is, for (RANGE-1) we will flip min and + max and generate these key_range structures: + + start.key='20' , end.key='10' + + See SEL_ARG::store_min_max(). The flag values are flipped as well, see + SEL_ARG::get_min_flag(), get_max_flag(). + + == Handling multiple key parts == + + For multi-part keys, the order of key parts has an effect on which ranges are + generated. Consider + + kp1 >= 10 AND kp2 >'foo' + + for INDEX(kp1 ASC, kp2 ASC) the range will be + + (kp1, kp2) > (10, 'foo') + + while for INDEX(kp1 ASC, kp2 DESC) it will be just + + kp1 >= 10 + + Another example: + + (kp1 BETWEEN 10 AND 20) AND (kp2 BETWEEN 'foo' AND 'quux') + + with INDEX (kp1 ASC, kp2 ASC) will generate + + (10, 'foo') <= (kp1, kp2) < (20, 'quux') + + while with index INDEX (kp1 ASC, kp2 DESC) it will generate + + (10, 'quux') <= (kp1, kp2) < (20, 'foo') + + This is again achieved by sel_arg_range_seq_next() and get_quick_keys() + flipping SEL_ARG's min,max, their flags and next/prev as needed. +*/ + extern MYSQL_PLUGIN_IMPORT SEL_ARG null_element; class SEL_ARG_IMPOSSIBLE: public SEL_ARG diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc index 20413f5df63..452a6864f06 100644 --- a/sql/opt_range_mrr.cc +++ b/sql/opt_range_mrr.cc @@ -34,7 +34,7 @@ typedef struct st_range_seq_entry uint min_key_flag, max_key_flag; /* Number of key parts */ - uint min_key_parts, max_key_parts; + int min_key_parts, max_key_parts; SEL_ARG *key_tree; } RANGE_SEQ_ENTRY; @@ -47,6 +47,7 @@ typedef struct st_sel_arg_range_seq uint keyno; /* index of used tree in SEL_TREE structure */ uint real_keyno; /* Number of the index in tables */ PARAM *param; + KEY_PART *key_parts; SEL_ARG *start; /* Root node of the traversed SEL_ARG* graph */ RANGE_SEQ_ENTRY stack[MAX_REF_PARTS]; @@ -105,13 +106,14 @@ static void step_down_to(SEL_ARG_RANGE_SEQ *arg, SEL_ARG *key_tree) cur->max_key_parts= prev->max_key_parts; uint16 stor_length= arg->param->key[arg->keyno][key_tree->part].store_length; - cur->min_key_parts += key_tree->store_min(stor_length, &cur->min_key, - prev->min_key_flag); - cur->max_key_parts += key_tree->store_max(stor_length, &cur->max_key, - prev->max_key_flag); - cur->min_key_flag= prev->min_key_flag | key_tree->min_flag; - cur->max_key_flag= prev->max_key_flag | key_tree->max_flag; + key_tree->store_min_max(arg->key_parts, stor_length, + &cur->min_key, prev->min_key_flag, + &cur->max_key, prev->max_key_flag, + &cur->min_key_parts, &cur->max_key_parts); + + cur->min_key_flag= prev->min_key_flag | key_tree->get_min_flag(arg->key_parts); + cur->max_key_flag= prev->max_key_flag | key_tree->get_max_flag(arg->key_parts); if (key_tree->is_null_interval()) cur->min_key_flag |= NULL_RANGE; @@ -165,12 +167,13 @@ bool sel_arg_range_seq_next(range_seq_t rseq, KEY_MULTI_RANGE *range) /* Ok, we're at some "full tuple" position in the tree */ /* Step down if we can */ - if (key_tree->next && key_tree->next != &null_element) + if (key_tree->index_order_next(seq->key_parts) && + key_tree->index_order_next(seq->key_parts) != &null_element) { //step down; (update the tuple, we'll step right and stay there) seq->i--; - step_down_to(seq, key_tree->next); - key_tree= key_tree->next; + step_down_to(seq, key_tree->index_order_next(seq->key_parts)); + key_tree= key_tree->index_order_next(seq->key_parts); seq->is_ror_scan= FALSE; goto walk_right_n_up; } @@ -185,12 +188,13 @@ bool sel_arg_range_seq_next(range_seq_t rseq, KEY_MULTI_RANGE *range) key_tree= seq->stack[seq->i].key_tree; /* Step down if we can */ - if (key_tree->next && key_tree->next != &null_element) + if (key_tree->index_order_next(seq->key_parts) && + key_tree->index_order_next(seq->key_parts) != &null_element) { // Step down; update the tuple seq->i--; - step_down_to(seq, key_tree->next); - key_tree= key_tree->next; + step_down_to(seq, key_tree->index_order_next(seq->key_parts)); + key_tree= key_tree->index_order_next(seq->key_parts); break; } } @@ -214,16 +218,10 @@ walk_right_n_up: !key_tree->min_flag && !key_tree->max_flag)) { seq->is_ror_scan= FALSE; - if (!key_tree->min_flag) - cur->min_key_parts += - key_tree->next_key_part->store_min_key(seq->param->key[seq->keyno], - &cur->min_key, - &cur->min_key_flag, MAX_KEY); - if (!key_tree->max_flag) - cur->max_key_parts += - key_tree->next_key_part->store_max_key(seq->param->key[seq->keyno], - &cur->max_key, - &cur->max_key_flag, MAX_KEY); + key_tree->store_next_min_max_keys(seq->param->key[seq->keyno], + &cur->min_key, &cur->min_key_flag, + &cur->max_key, &cur->max_key_flag, + &cur->min_key_parts, &cur->max_key_parts); break; } } @@ -235,10 +233,11 @@ walk_right_n_up: key_tree= key_tree->next_key_part; walk_up_n_right: - while (key_tree->prev && key_tree->prev != &null_element) + while (key_tree->index_order_prev(seq->key_parts) && + key_tree->index_order_prev(seq->key_parts) != &null_element) { /* Step up */ - key_tree= key_tree->prev; + key_tree= key_tree->index_order_prev(seq->key_parts); } step_down_to(seq, key_tree); } diff --git a/sql/opt_split.cc b/sql/opt_split.cc index 627aebe9dec..e00a26e3328 100644 --- a/sql/opt_split.cc +++ b/sql/opt_split.cc @@ -765,7 +765,7 @@ double spl_postjoin_oper_cost(THD *thd, double join_record_count, uint rec_len) void JOIN::add_keyuses_for_splitting() { uint i; - uint idx; + size_t idx; KEYUSE_EXT *keyuse_ext; KEYUSE_EXT keyuse_ext_end; double oper_cost; diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index 040d3ed529a..46db641987b 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -718,7 +718,7 @@ int check_and_do_in_subquery_rewrites(JOIN *join) { DBUG_PRINT("info", ("Subquery is semi-join conversion candidate")); - (void)subquery_types_allow_materialization(thd, in_subs); + //(void)subquery_types_allow_materialization(thd, in_subs); in_subs->is_flattenable_semijoin= TRUE; @@ -1286,6 +1286,7 @@ bool convert_join_subqueries_to_semijoins(JOIN *join) while ((in_subq= li++)) { bool remove_item= TRUE; + subquery_types_allow_materialization(thd, in_subq); /* Stop processing if we've reached a subquery that's attached to the ON clause */ if (in_subq->do_not_convert_to_sj) @@ -4583,7 +4584,7 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) STEP 1: Get temporary table name */ if (use_temp_pool && !(test_flags & TEST_KEEP_TMP_TABLES)) - temp_pool_slot = bitmap_lock_set_next(&temp_pool); + temp_pool_slot = temp_pool_set_next(); if (temp_pool_slot != MY_BIT_NONE) // we got a slot sprintf(path, "%s-subquery-%lx-%i", tmp_file_prefix, @@ -4620,7 +4621,7 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) NullS)) { if (temp_pool_slot != MY_BIT_NONE) - bitmap_lock_clear_bit(&temp_pool, temp_pool_slot); + temp_pool_clear_bit(temp_pool_slot); DBUG_RETURN(TRUE); } strmov(tmpname,path); @@ -4830,7 +4831,7 @@ err: thd->mem_root= mem_root_save; free_tmp_table(thd,table); /* purecov: inspected */ if (temp_pool_slot != MY_BIT_NONE) - bitmap_lock_clear_bit(&temp_pool, temp_pool_slot); + temp_pool_clear_bit(temp_pool_slot); DBUG_RETURN(TRUE); /* purecov: inspected */ } diff --git a/sql/opt_sum.cc b/sql/opt_sum.cc index 627ddc86abd..794ec40fc66 100644 --- a/sql/opt_sum.cc +++ b/sql/opt_sum.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2008, 2017, MariaDB Corporation. + Copyright (c) 2008, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -953,6 +953,9 @@ static bool find_key_for_maxmin(bool max_fl, TABLE_REF *ref, part->length < part_field->key_length()) break; + if (part->key_part_flag & HA_REVERSE_SORT) + break; // TODO MDEV-27576 + if (field->eq(part->field)) { ref->key= idx; diff --git a/sql/opt_table_elimination.cc b/sql/opt_table_elimination.cc index c3297cb81cd..03ee1a2ad62 100644 --- a/sql/opt_table_elimination.cc +++ b/sql/opt_table_elimination.cc @@ -1070,7 +1070,7 @@ bool Dep_analysis_context::setup_equality_modules_deps(List<Dep_module> void *buf; if (!(buf= thd->alloc(bitmap_buffer_size(offset))) || - my_bitmap_init(&expr_deps, (my_bitmap_map*)buf, offset, FALSE)) + my_bitmap_init(&expr_deps, (my_bitmap_map*)buf, offset)) { DBUG_RETURN(TRUE); /* purecov: inspected */ } diff --git a/sql/partition_element.h b/sql/partition_element.h index c372625682f..1abaa315218 100644 --- a/sql/partition_element.h +++ b/sql/partition_element.h @@ -2,6 +2,7 @@ #define PARTITION_ELEMENT_INCLUDED /* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -111,7 +112,6 @@ public: ha_rows part_min_rows; longlong range_value; const char *partition_name; - const char *tablespace_name; struct st_ddl_log_memory_entry *log_entry; const char* part_comment; const char* data_file_name; @@ -127,9 +127,12 @@ public: bool empty; elem_type_enum type; + engine_option_value *option_list; // create options for partition + ha_table_option_struct *option_struct; // structure with parsed options + partition_element() : part_max_rows(0), part_min_rows(0), range_value(0), - partition_name(NULL), tablespace_name(NULL), + partition_name(NULL), log_entry(NULL), part_comment(NULL), data_file_name(NULL), index_file_name(NULL), engine_type(NULL), connect_string(null_clex_str), part_state(PART_NORMAL), @@ -137,13 +140,13 @@ public: signed_flag(FALSE), max_value(FALSE), id(UINT_MAX32), empty(true), - type(CONVENTIONAL) + type(CONVENTIONAL), + option_list(NULL), option_struct(NULL) {} partition_element(partition_element *part_elem) : part_max_rows(part_elem->part_max_rows), part_min_rows(part_elem->part_min_rows), range_value(0), partition_name(NULL), - tablespace_name(part_elem->tablespace_name), log_entry(NULL), part_comment(part_elem->part_comment), data_file_name(part_elem->data_file_name), @@ -157,7 +160,9 @@ public: max_value(part_elem->max_value), id(part_elem->id), empty(part_elem->empty), - type(CONVENTIONAL) + type(CONVENTIONAL), + option_list(part_elem->option_list), + option_struct(part_elem->option_struct) {} ~partition_element() = default; diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 3af7e97db2b..8bef4165878 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -2434,7 +2434,7 @@ bool partition_info::has_same_partitioning(partition_info *new_part_info) partition_element *new_part_elem= new_part_it++; /* The following must match: - partition_name, tablespace_name, data_file_name, index_file_name, + partition_name, data_file_name, index_file_name, engine_type, part_max_rows, part_min_rows, nodegroup_id. (max_value, signed_flag, has_null_value only on partition level, RANGE/LIST) @@ -2520,9 +2520,7 @@ bool partition_info::has_same_partitioning(partition_info *new_part_info) if (strcmp_null(sub_part_elem->data_file_name, new_sub_part_elem->data_file_name) || strcmp_null(sub_part_elem->index_file_name, - new_sub_part_elem->index_file_name) || - strcmp_null(sub_part_elem->tablespace_name, - new_sub_part_elem->tablespace_name)) + new_sub_part_elem->index_file_name)) DBUG_RETURN(false); } while (++j < num_subparts); @@ -2538,9 +2536,7 @@ bool partition_info::has_same_partitioning(partition_info *new_part_info) if (strcmp_null(part_elem->data_file_name, new_part_elem->data_file_name) || strcmp_null(part_elem->index_file_name, - new_part_elem->index_file_name) || - strcmp_null(part_elem->tablespace_name, - new_part_elem->tablespace_name)) + new_part_elem->index_file_name)) DBUG_RETURN(false); } } while (++i < num_parts); diff --git a/sql/partition_info.h b/sql/partition_info.h index ebd41ce1764..d80676057d3 100644 --- a/sql/partition_info.h +++ b/sql/partition_info.h @@ -83,7 +83,7 @@ struct Vers_part_info : public Sql_alloc See generate_partition_syntax() for details of how the data is used in partition expression. */ -class partition_info : public Sql_alloc +class partition_info : public DDL_LOG_STATE, public Sql_alloc { public: /* @@ -162,10 +162,6 @@ public: Item *item_free_list; - struct st_ddl_log_memory_entry *first_log_entry; - struct st_ddl_log_memory_entry *exec_log_entry; - struct st_ddl_log_memory_entry *frm_log_entry; - /* Bitmaps of partitions used by the current query. * read_partitions - partitions to be used for reading. @@ -305,7 +301,6 @@ public: part_field_buffers(NULL), subpart_field_buffers(NULL), restore_part_field_ptrs(NULL), restore_subpart_field_ptrs(NULL), part_expr(NULL), subpart_expr(NULL), item_free_list(NULL), - first_log_entry(NULL), exec_log_entry(NULL), frm_log_entry(NULL), bitmaps_are_initialized(FALSE), list_array(NULL), vers_info(NULL), err_value(0), part_info_string(NULL), @@ -327,6 +322,7 @@ public: is_auto_partitioned(FALSE), has_null_value(FALSE), column_list(FALSE) { + bzero((DDL_LOG_STATE *) this, sizeof(DDL_LOG_STATE)); all_fields_in_PF.clear_all(); all_fields_in_PPF.clear_all(); all_fields_in_SPF.clear_all(); @@ -432,8 +428,13 @@ public: return NULL; } uint next_part_no(uint new_parts) const; + + int gen_part_type(THD *thd, String *str) const; }; +void part_type_error(THD *thd, partition_info *work_part_info, + const char *part_type, partition_info *tab_part_info); + uint32 get_next_partition_id_range(struct st_partition_iter* part_iter); bool check_partition_dirs(partition_info *part_info); diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index fce5f260639..f7059668b11 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -17,6 +17,7 @@ /* Definitions for MariaDB global transaction ID (GTID). */ +#ifndef MYSQL_CLIENT #include "mariadb.h" #include "sql_priv.h" #include "unireg.h" @@ -24,7 +25,6 @@ #include "sql_base.h" #include "sql_parse.h" #include "key.h" -#include "rpl_gtid.h" #include "rpl_rli.h" #include "slave.h" #include "log_event.h" @@ -249,8 +249,9 @@ rpl_slave_state::rpl_slave_state() { mysql_mutex_init(key_LOCK_slave_state, &LOCK_slave_state, MY_MUTEX_INIT_SLOW); - my_hash_init(PSI_INSTRUMENT_ME, &hash, &my_charset_bin, 32, offsetof(element, domain_id), - sizeof(uint32), NULL, rpl_slave_state_free_element, HASH_UNIQUE); + my_hash_init(PSI_INSTRUMENT_ME, &hash, &my_charset_bin, 32, + offsetof(element, domain_id), sizeof(element::domain_id), + NULL, rpl_slave_state_free_element, HASH_UNIQUE); my_init_dynamic_array(PSI_INSTRUMENT_ME, >id_sort_array, sizeof(rpl_gtid), 8, 8, MYF(0)); } @@ -366,7 +367,8 @@ rpl_slave_state::get_element(uint32 domain_id) { struct element *elem; - elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0); + elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, + sizeof(domain_id)); if (elem) return elem; @@ -402,7 +404,8 @@ rpl_slave_state::put_back_list(list_element *list) list_element *next= list->next; if ((!e || e->domain_id != list->domain_id) && - !(e= (element *)my_hash_search(&hash, (const uchar *)&list->domain_id, 0))) + !(e= (element *)my_hash_search(&hash, (const uchar *)&list->domain_id, + sizeof(list->domain_id)))) { err= 1; goto end; @@ -604,6 +607,8 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, wait_for_commit* suspended_wfc; void *hton= NULL; LEX_CSTRING gtid_pos_table_name; + TABLE *tbl= nullptr; + MDL_savepoint m_start_of_statement_svp(thd->mdl_context.mdl_savepoint()); DBUG_ENTER("record_gtid"); *out_hton= NULL; @@ -622,6 +627,18 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, if (!in_statement) thd->reset_for_next_command(); + if (thd->rgi_slave && (thd->rgi_slave->gtid_ev_flags_extra & + Gtid_log_event::FL_START_ALTER_E1)) + { + /* + store the open table table list in ptr, so that is close_thread_tables + is called start alter tables are not closed + */ + mysql_mutex_lock(&thd->LOCK_thd_data); + tbl= thd->open_tables; + thd->open_tables= nullptr; + mysql_mutex_unlock(&thd->LOCK_thd_data); + } /* Only the SQL thread can call select_gtid_pos_table without a mutex Other threads needs to use a mutex and take into account that the @@ -731,12 +748,23 @@ end: { if (err || (err= ha_commit_trans(thd, FALSE))) ha_rollback_trans(thd, FALSE); - close_thread_tables(thd); - if (in_transaction) - thd->mdl_context.release_statement_locks(); - else - thd->release_transactional_locks(); + if (!thd->rgi_slave || !(thd->rgi_slave->gtid_ev_flags_extra & + Gtid_log_event::FL_START_ALTER_E1)) + { + if (in_transaction) + thd->mdl_context.release_statement_locks(); + else + thd->release_transactional_locks(); + } + } + if (thd->rgi_slave && + thd->rgi_slave->gtid_ev_flags_extra & Gtid_log_event::FL_START_ALTER_E1) + { + mysql_mutex_lock(&thd->LOCK_thd_data); + thd->open_tables= tbl; + mysql_mutex_unlock(&thd->LOCK_thd_data); + thd->mdl_context.rollback_to_savepoint(m_start_of_statement_svp); } thd->lex->restore_backup_query_tables_list(&lex_backup); thd->variables.option_bits= thd_saved_option; @@ -1107,8 +1135,8 @@ rpl_slave_state::iterate(int (*cb)(rpl_gtid *, void *), void *data, bool locked= false; my_hash_init(PSI_INSTRUMENT_ME, >id_hash, &my_charset_bin, 32, - offsetof(rpl_gtid, domain_id), sizeof(uint32), NULL, NULL, - HASH_UNIQUE); + offsetof(rpl_gtid, domain_id), sizeof(rpl_gtid::domain_id), + NULL, NULL, HASH_UNIQUE); for (i= 0; i < num_extra; ++i) if (extra_gtids[i].server_id == global_system_variables.server_id && my_hash_insert(>id_hash, (uchar *)(&extra_gtids[i]))) @@ -1143,7 +1171,8 @@ rpl_slave_state::iterate(int (*cb)(rpl_gtid *, void *), void *data, } /* Check if we have something newer in the extra list. */ - rec= my_hash_search(>id_hash, (const uchar *)&best_gtid.domain_id, 0); + rec= my_hash_search(>id_hash, (const uchar *)&best_gtid.domain_id, + sizeof(best_gtid.domain_id)); if (rec) { gtid= (rpl_gtid *)rec; @@ -1243,7 +1272,8 @@ rpl_slave_state::domain_to_gtid(uint32 domain_id, rpl_gtid *out_gtid) uint64 best_sub_id; mysql_mutex_lock(&LOCK_slave_state); - elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0); + elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, + sizeof(domain_id)); if (!elem || !(list= elem->list)) { mysql_mutex_unlock(&LOCK_slave_state); @@ -1268,6 +1298,7 @@ rpl_slave_state::domain_to_gtid(uint32 domain_id, rpl_gtid *out_gtid) return true; } +#endif /* Parse a GTID at the start of a string, and update the pointer to point @@ -1305,7 +1336,6 @@ gtid_parser_helper(const char **ptr, const char *end, rpl_gtid *out_gtid) return 0; } - rpl_gtid * gtid_parse_string_to_list(const char *str, size_t str_len, uint32 *out_len) { @@ -1344,6 +1374,7 @@ gtid_parse_string_to_list(const char *str, size_t str_len, uint32 *out_len) return list; } +#ifndef MYSQL_CLIENT /* Update the slave replication state with the GTID position obtained from @@ -1477,8 +1508,9 @@ rpl_slave_state::alloc_gtid_pos_table(LEX_CSTRING *table_name, void *hton, void rpl_binlog_state::init() { - my_hash_init(PSI_INSTRUMENT_ME, &hash, &my_charset_bin, 32, offsetof(element, domain_id), - sizeof(uint32), NULL, my_free, HASH_UNIQUE); + my_hash_init(PSI_INSTRUMENT_ME, &hash, &my_charset_bin, 32, + offsetof(element, domain_id), sizeof(element::domain_id), + NULL, my_free, HASH_UNIQUE); my_init_dynamic_array(PSI_INSTRUMENT_ME, >id_sort_array, sizeof(rpl_gtid), 8, 8, MYF(0)); mysql_mutex_init(key_LOCK_binlog_state, &LOCK_binlog_state, MY_MUTEX_INIT_SLOW); @@ -1580,7 +1612,8 @@ rpl_binlog_state::update_nolock(const struct rpl_gtid *gtid, bool strict) element *elem; if ((elem= (element *)my_hash_search(&hash, - (const uchar *)(>id->domain_id), 0))) + (const uchar *)(>id->domain_id), + sizeof(gtid->domain_id)))) { if (strict && elem->last_gtid && elem->last_gtid->seq_no >= gtid->seq_no) { @@ -1628,7 +1661,8 @@ rpl_binlog_state::update_with_next_gtid(uint32 domain_id, uint32 server_id, gtid->server_id= server_id; mysql_mutex_lock(&LOCK_binlog_state); - if ((elem= (element *)my_hash_search(&hash, (const uchar *)(&domain_id), 0))) + if ((elem= (element *)my_hash_search(&hash, (const uchar *)(&domain_id), + sizeof(domain_id)))) { gtid->seq_no= ++elem->seq_no_counter; if (!elem->update_element(gtid)) @@ -1667,7 +1701,8 @@ rpl_binlog_state::element::update_element(const rpl_gtid *gtid) } lookup_gtid= (rpl_gtid *) - my_hash_search(&hash, (const uchar *)>id->server_id, 0); + my_hash_search(&hash, (const uchar *)>id->server_id, + sizeof(gtid->server_id)); if (lookup_gtid) { lookup_gtid->seq_no= gtid->seq_no; @@ -1705,8 +1740,8 @@ rpl_binlog_state::alloc_element_nolock(const rpl_gtid *gtid) { elem->domain_id= gtid->domain_id; my_hash_init(PSI_INSTRUMENT_ME, &elem->hash, &my_charset_bin, 32, - offsetof(rpl_gtid, server_id), sizeof(uint32), NULL, my_free, - HASH_UNIQUE); + offsetof(rpl_gtid, server_id), sizeof(rpl_gtid::domain_id), + NULL, my_free, HASH_UNIQUE); elem->last_gtid= lookup_gtid; elem->seq_no_counter= gtid->seq_no; memcpy(lookup_gtid, gtid, sizeof(*lookup_gtid)); @@ -1741,7 +1776,8 @@ rpl_binlog_state::check_strict_sequence(uint32 domain_id, uint32 server_id, mysql_mutex_lock(&LOCK_binlog_state); if ((elem= (element *)my_hash_search(&hash, - (const uchar *)(&domain_id), 0)) && + (const uchar *)(&domain_id), + sizeof(domain_id))) && elem->last_gtid && elem->last_gtid->seq_no >= seq_no) { if (!no_error) @@ -1769,7 +1805,8 @@ rpl_binlog_state::bump_seq_no_if_needed(uint32 domain_id, uint64 seq_no) int res; mysql_mutex_lock(&LOCK_binlog_state); - if ((elem= (element *)my_hash_search(&hash, (const uchar *)(&domain_id), 0))) + if ((elem= (element *)my_hash_search(&hash, (const uchar *)(&domain_id), + sizeof(domain_id)))) { if (elem->seq_no_counter < seq_no) elem->seq_no_counter= seq_no; @@ -1787,8 +1824,8 @@ rpl_binlog_state::bump_seq_no_if_needed(uint32 domain_id, uint64 seq_no) elem->domain_id= domain_id; my_hash_init(PSI_INSTRUMENT_ME, &elem->hash, &my_charset_bin, 32, - offsetof(rpl_gtid, server_id), sizeof(uint32), NULL, my_free, - HASH_UNIQUE); + offsetof(rpl_gtid, server_id), sizeof(rpl_gtid::server_id), + NULL, my_free, HASH_UNIQUE); elem->last_gtid= NULL; elem->seq_no_counter= seq_no; if (0 == my_hash_insert(&hash, (const uchar *)elem)) @@ -1892,9 +1929,11 @@ rpl_gtid * rpl_binlog_state::find_nolock(uint32 domain_id, uint32 server_id) { element *elem; - if (!(elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0))) + if (!(elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, + sizeof(domain_id)))) return NULL; - return (rpl_gtid *)my_hash_search(&elem->hash, (const uchar *)&server_id, 0); + return (rpl_gtid *)my_hash_search(&elem->hash, (const uchar *)&server_id, + sizeof(server_id)); } rpl_gtid * @@ -1914,7 +1953,8 @@ rpl_binlog_state::find_most_recent(uint32 domain_id) rpl_gtid *gtid= NULL; mysql_mutex_lock(&LOCK_binlog_state); - elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, 0); + elem= (element *)my_hash_search(&hash, (const uchar *)&domain_id, + sizeof(domain_id)); if (elem && elem->last_gtid) gtid= elem->last_gtid; mysql_mutex_unlock(&LOCK_binlog_state); @@ -2183,7 +2223,8 @@ rpl_binlog_state::drop_domain(DYNAMIC_ARRAY *ids, ptr_domain_id= (uint32*) dynamic_array_ptr(ids, i); elem= (rpl_binlog_state::element *) - my_hash_search(&hash, (const uchar *) ptr_domain_id, 0); + my_hash_search(&hash, (const uchar *) ptr_domain_id, + sizeof(ptr_domain_id[0])); if (!elem) { push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, @@ -2244,7 +2285,7 @@ slave_connection_state::slave_connection_state() { my_hash_init(PSI_INSTRUMENT_ME, &hash, &my_charset_bin, 32, offsetof(entry, gtid) + offsetof(rpl_gtid, domain_id), - sizeof(uint32), NULL, my_free, HASH_UNIQUE); + sizeof(rpl_gtid::domain_id), NULL, my_free, HASH_UNIQUE); my_init_dynamic_array(PSI_INSTRUMENT_ME, >id_sort_array, sizeof(rpl_gtid), 8, 8, MYF(0)); } @@ -2299,7 +2340,8 @@ slave_connection_state::load(const char *slave_request, size_t len) return 1; } if ((e= (const entry *) - my_hash_search(&hash, (const uchar *)(>id->domain_id), 0))) + my_hash_search(&hash, (const uchar *)(>id->domain_id), + sizeof(gtid->domain_id)))) { my_error(ER_DUPLICATE_GTID_DOMAIN, MYF(0), gtid->domain_id, gtid->server_id, (ulonglong)gtid->seq_no, e->gtid.domain_id, @@ -2366,7 +2408,8 @@ slave_connection_state::load(rpl_slave_state *state, slave_connection_state::entry * slave_connection_state::find_entry(uint32 domain_id) { - return (entry *) my_hash_search(&hash, (const uchar *)(&domain_id), 0); + return (entry *) my_hash_search(&hash, (const uchar *)(&domain_id), + sizeof(domain_id)); } @@ -2384,7 +2427,8 @@ int slave_connection_state::update(const rpl_gtid *in_gtid) { entry *e; - uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0); + uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), + sizeof(in_gtid->domain_id)); if (rec) { e= (entry *)rec; @@ -2409,7 +2453,8 @@ slave_connection_state::update(const rpl_gtid *in_gtid) void slave_connection_state::remove(const rpl_gtid *in_gtid) { - uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0); + uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), + sizeof(in_gtid->domain_id)); #ifdef DBUG_ASSERT_EXISTS bool err; rpl_gtid *slave_gtid= &((entry *)rec)->gtid; @@ -2426,7 +2471,8 @@ slave_connection_state::remove(const rpl_gtid *in_gtid) void slave_connection_state::remove_if_present(const rpl_gtid *in_gtid) { - uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0); + uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), + sizeof(in_gtid->domain_id)); if (rec) my_hash_delete(&hash, rec); } @@ -2870,7 +2916,8 @@ void gtid_waiting::init() { my_hash_init(PSI_INSTRUMENT_ME, &hash, &my_charset_bin, 32, - offsetof(hash_element, domain_id), sizeof(uint32), NULL, + offsetof(hash_element, domain_id), + sizeof(hash_element::domain_id), NULL, free_hash_element, HASH_UNIQUE); mysql_mutex_init(key_LOCK_gtid_waiting, &LOCK_gtid_waiting, 0); } @@ -2903,7 +2950,8 @@ gtid_waiting::get_entry(uint32 domain_id) { hash_element *e; - if ((e= (hash_element *)my_hash_search(&hash, (const uchar *)&domain_id, 0))) + if ((e= (hash_element *)my_hash_search(&hash, (const uchar *)&domain_id, + sizeof(domain_id)))) return e; if (!(e= (hash_element *)my_malloc(PSI_INSTRUMENT_ME, sizeof(*e), MYF(MY_WME)))) @@ -2953,3 +3001,843 @@ gtid_waiting::remove_from_wait_queue(gtid_waiting::hash_element *he, queue_remove(&he->queue, elem->queue_idx); } + +#endif + +void free_domain_lookup_element(void *p) +{ + struct Binlog_gtid_state_validator::audit_elem *audit_elem= + (struct Binlog_gtid_state_validator::audit_elem *) p; + delete_dynamic(&audit_elem->late_gtids_previous); + delete_dynamic(&audit_elem->late_gtids_real); + my_free(audit_elem); +} + +Binlog_gtid_state_validator::Binlog_gtid_state_validator() +{ + my_hash_init(PSI_INSTRUMENT_ME, &m_audit_elem_domain_lookup, &my_charset_bin, 32, + offsetof(struct audit_elem, domain_id), sizeof(uint32), + NULL, free_domain_lookup_element, HASH_UNIQUE); +} + +Binlog_gtid_state_validator::~Binlog_gtid_state_validator() +{ + my_hash_free(&m_audit_elem_domain_lookup); +} + +void Binlog_gtid_state_validator::initialize_start_gtids(rpl_gtid *start_gtids, + size_t n_gtids) +{ + size_t i; + for(i= 0; i < n_gtids; i++) + { + rpl_gtid *domain_state_gtid= &start_gtids[i]; + + /* + If we are initializing from a GLLE, we can have repeat domain ids from + differing servers, so we want to ensure our start gtid matches the last + known position + */ + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(domain_state_gtid->domain_id), 0); + if (audit_elem) + { + /* + We have this domain already specified, so try to overwrite with the + more recent GTID + */ + if (domain_state_gtid->seq_no > audit_elem->start_gtid.seq_no) + audit_elem->start_gtid = *domain_state_gtid; + continue; + } + + /* Initialize a new domain */ + audit_elem= (struct audit_elem *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(struct audit_elem), MYF(MY_WME)); + if (!audit_elem) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return; + } + + audit_elem->domain_id= start_gtids[i].domain_id; + audit_elem->start_gtid= start_gtids[i]; + audit_elem->last_gtid= {audit_elem->domain_id, 0, 0}; + + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_real, + sizeof(rpl_gtid), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_previous, + sizeof(rpl_gtid), 8, 8, MYF(0)); + + if (my_hash_insert(&m_audit_elem_domain_lookup, (uchar *) audit_elem)) + { + my_free(audit_elem); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return; + } + } +} + +my_bool Binlog_gtid_state_validator::initialize_gtid_state(FILE *out, + rpl_gtid *gtids, + size_t n_gtids) +{ + size_t i; + my_bool err= FALSE; + + /* + We weren't initialized with starting positions explicitly, so assume the + starting positions of the current gtid state + */ + if (!m_audit_elem_domain_lookup.records) + initialize_start_gtids(gtids, n_gtids); + + for(i= 0; i < n_gtids; i++) + { + rpl_gtid *domain_state_gtid= >ids[i]; + + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(domain_state_gtid->domain_id), 0); + + if (!audit_elem) + { + Binlog_gtid_state_validator::error( + out, + "Starting GTID position list does not specify an initial value " + "for domain %u, whose events may be present in the requested binlog " + "file(s). The last known position for this domain was %u-%u-%llu.", + domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid))); + err= TRUE; + continue; + } + + if (audit_elem->start_gtid.seq_no < domain_state_gtid->seq_no) + { + Binlog_gtid_state_validator::error( + out, + "Binary logs are missing data for domain %u. Expected data to " + "start from state %u-%u-%llu; however, the initial GTID state of " + "the logs was %u-%u-%llu.", + domain_state_gtid->domain_id, PARAM_GTID(audit_elem->start_gtid), + PARAM_GTID((*domain_state_gtid))); + err= TRUE; + continue; + } + + if (domain_state_gtid->seq_no > audit_elem->last_gtid.seq_no) + audit_elem->last_gtid= *domain_state_gtid; + } + return err; +} + +my_bool Binlog_gtid_state_validator::verify_stop_state(FILE *out, + rpl_gtid *stop_gtids, + size_t n_stop_gtids) +{ + size_t i; + for(i= 0; i < n_stop_gtids; i++) + { + rpl_gtid *stop_gtid= &stop_gtids[i]; + + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(stop_gtid->domain_id), 0); + + /* + It is okay if stop gtid doesn't exist in current state because it will be treated + as a new domain + */ + if (audit_elem && stop_gtid->seq_no <= audit_elem->start_gtid.seq_no) + { + Binlog_gtid_state_validator::error( + out, + "--stop-position GTID %u-%u-%llu does not exist in the " + "specified binlog files. The current GTID state of domain %u in the " + "specified binary logs is %u-%u-%llu", + PARAM_GTID((*stop_gtid)), stop_gtid->domain_id, + PARAM_GTID(audit_elem->start_gtid)); + return TRUE; + } + } + + /* No issues with any GTIDs */ + return FALSE; +} + +my_bool +Binlog_gtid_state_validator::verify_gtid_state(FILE *out, + rpl_gtid *domain_state_gtid) +{ + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(domain_state_gtid->domain_id), 0); + + if (!audit_elem) + { + Binlog_gtid_state_validator::warn( + out, + "Binary logs are missing data for domain %u. The current binary log " + "specified its " + "current state for this domain as %u-%u-%llu, but neither the " + "starting GTID position list nor any processed events have " + "mentioned " + "this domain.", + domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid))); + return TRUE; + } + + if (audit_elem->last_gtid.seq_no < domain_state_gtid->seq_no) + { + Binlog_gtid_state_validator::warn( + out, + "Binary logs are missing data for domain %u. The current binary log " + "state is %u-%u-%llu, but the last seen event was %u-%u-%llu.", + domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid)), + PARAM_GTID(audit_elem->last_gtid)); + return TRUE; + } + + return FALSE; +} + +my_bool Binlog_gtid_state_validator::record(rpl_gtid *gtid) +{ + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, (const uchar *) &(gtid->domain_id), 0); + + if (!audit_elem) + { + /* + We haven't seen any GTIDs in this domian yet. Perform initial set up for + this domain so we can monitor its events. + */ + audit_elem= (struct audit_elem *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(struct audit_elem), MYF(MY_WME)); + if (!audit_elem) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return TRUE; + } + + audit_elem->domain_id= gtid->domain_id; + audit_elem->last_gtid= *gtid; + audit_elem->start_gtid= {gtid->domain_id, 0, 0}; + + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_real, + sizeof(rpl_gtid), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_previous, + sizeof(rpl_gtid), 8, 8, MYF(0)); + + if (my_hash_insert(&m_audit_elem_domain_lookup, (uchar *) audit_elem)) + { + my_free(audit_elem); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return TRUE; + } + } + else + { + /* Out of order check */ + if (gtid->seq_no <= audit_elem->last_gtid.seq_no && + gtid->seq_no >= audit_elem->start_gtid.seq_no) + { + /* GTID is out of order */ + insert_dynamic(&audit_elem->late_gtids_real, (const void *) gtid); + insert_dynamic(&audit_elem->late_gtids_previous, + (const void *) &(audit_elem->last_gtid)); + + return TRUE; + } + else + { + /* GTID is valid */ + audit_elem->last_gtid= *gtid; + } + } + + return FALSE; +} + +/* + Data structure used to help pass data into report_audit_findings because + my_hash_iterate only passes one parameter +*/ +struct gtid_report_ctx +{ + FILE *out_file; + my_bool is_strict_mode; + my_bool contains_err; +}; + +static my_bool report_audit_findings(void *entry, void *report_ctx_arg) +{ + struct Binlog_gtid_state_validator::audit_elem *audit_el= + (struct Binlog_gtid_state_validator::audit_elem *) entry; + + struct gtid_report_ctx *report_ctx= + (struct gtid_report_ctx *) report_ctx_arg; + FILE *out= report_ctx->out_file; + my_bool is_strict_mode= report_ctx->is_strict_mode; + size_t i; + void (*report_f)(FILE*, const char*, ...); + + if (is_strict_mode) + report_f= Binlog_gtid_state_validator::error; + else + report_f= Binlog_gtid_state_validator::warn; + + if (audit_el) + { + if (audit_el->last_gtid.seq_no < audit_el->start_gtid.seq_no) + { + report_f(out, + "Binary logs never reached expected GTID state of %u-%u-%llu", + PARAM_GTID(audit_el->start_gtid)); + report_ctx->contains_err= TRUE; + } + + /* Report any out of order GTIDs */ + for(i= 0; i < audit_el->late_gtids_real.elements; i++) + { + rpl_gtid *real_gtid= + (rpl_gtid *) dynamic_array_ptr(&(audit_el->late_gtids_real), i); + rpl_gtid *last_gtid= (rpl_gtid *) dynamic_array_ptr( + &(audit_el->late_gtids_previous), i); + DBUG_ASSERT(real_gtid && last_gtid); + + report_f(out, + "Found out of order GTID. Got %u-%u-%llu after %u-%u-%llu", + PARAM_GTID((*real_gtid)), PARAM_GTID((*last_gtid))); + report_ctx->contains_err= TRUE; + } + } + + return FALSE; +} + +my_bool Binlog_gtid_state_validator::report(FILE *out, my_bool is_strict_mode) +{ + struct gtid_report_ctx report_ctx; + report_ctx.out_file= out; + report_ctx.is_strict_mode= is_strict_mode; + report_ctx.contains_err= FALSE; + my_hash_iterate(&m_audit_elem_domain_lookup, report_audit_findings, &report_ctx); + fflush(out); + return is_strict_mode ? report_ctx.contains_err : FALSE; +} + +Window_gtid_event_filter::Window_gtid_event_filter() + : m_has_start(FALSE), m_has_stop(FALSE), m_is_active(FALSE), + m_has_passed(FALSE) +{ + // m_start and m_stop do not need initial values if unused +} + +int Window_gtid_event_filter::set_start_gtid(rpl_gtid *start) +{ + if (m_has_start) + { + sql_print_error( + "Start position cannot have repeated domain " + "ids (found %u-%u-%llu when %u-%u-%llu was previously specified)", + PARAM_GTID((*start)), PARAM_GTID(m_start)); + return 1; + } + + m_has_start= TRUE; + m_start= *start; + return 0; +} + +int Window_gtid_event_filter::set_stop_gtid(rpl_gtid *stop) +{ + if (m_has_stop) + { + sql_print_error( + "Stop position cannot have repeated domain " + "ids (found %u-%u-%llu when %u-%u-%llu was previously specified)", + PARAM_GTID((*stop)), PARAM_GTID(m_stop)); + return 1; + } + + m_has_stop= TRUE; + m_stop= *stop; + return 0; +} + +my_bool Window_gtid_event_filter::is_range_invalid() +{ + if (m_has_start && m_has_stop && m_start.seq_no > m_stop.seq_no) + { + sql_print_error( + "Queried GTID range is invalid in strict mode. Stop position " + "%u-%u-%llu is not greater than or equal to start %u-%u-%llu.", + PARAM_GTID(m_stop), PARAM_GTID(m_start)); + return TRUE; + } + return FALSE; +} + +static inline my_bool is_gtid_at_or_after(rpl_gtid *boundary, + rpl_gtid *test_gtid) +{ + return test_gtid->domain_id == boundary->domain_id && + test_gtid->seq_no >= boundary->seq_no; +} + +static inline my_bool is_gtid_at_or_before(rpl_gtid *boundary, + rpl_gtid *test_gtid) +{ + return test_gtid->domain_id == boundary->domain_id && + test_gtid->seq_no <= boundary->seq_no; +} + +my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + /* Assume result should be excluded to start */ + my_bool should_exclude= TRUE; + + DBUG_ASSERT((m_has_start && gtid->domain_id == m_start.domain_id) || + (m_has_stop && gtid->domain_id == m_stop.domain_id)); + + if (!m_is_active && !m_has_passed) + { + /* + This filter has not yet been activated. Check if the gtid is within the + bounds of this window. + */ + + if (!m_has_start && is_gtid_at_or_before(&m_stop, gtid)) + { + /* + Start GTID was not provided, so we want to include everything from here + up to m_stop + */ + m_is_active= TRUE; + should_exclude= FALSE; + } + else if ((m_has_start && is_gtid_at_or_after(&m_start, gtid)) && + (!m_has_stop || is_gtid_at_or_before(&m_stop, gtid))) + { + m_is_active= TRUE; + + DBUG_PRINT("gtid-event-filter", + ("Window: Begin (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + + /* + As the start of the range is exclusive, if this gtid is the start of + the range, exclude it + */ + if (gtid->seq_no == m_start.seq_no) + should_exclude= TRUE; + else + should_exclude= FALSE; + + if (m_has_stop && gtid->seq_no == m_stop.seq_no) + { + m_has_passed= TRUE; + DBUG_PRINT("gtid-event-filter", + ("Window: End (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + } + } + } /* if (!m_is_active && !m_has_passed) */ + else if (m_is_active && !m_has_passed) + { + /* + This window is currently active so we want the event group to be included + in the results. Additionally check if we are at the end of the window. + If no end of the window is provided, go indefinitely + */ + should_exclude= FALSE; + + if (m_has_stop && is_gtid_at_or_after(&m_stop, gtid)) + { + DBUG_PRINT("gtid-event-filter", + ("Window: End (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + m_is_active= FALSE; + m_has_passed= TRUE; + + if (!is_gtid_at_or_before(&m_stop, gtid)) + { + /* + The GTID is after the finite stop of the window, don't let it pass + through + */ + should_exclude= TRUE; + } + } + } + + return should_exclude; +} + +my_bool Window_gtid_event_filter::has_finished() +{ + return m_has_stop ? m_has_passed : FALSE; +} + +void free_gtid_filter_element(void *p) +{ + gtid_filter_element *gfe = (gtid_filter_element *) p; + if (gfe->filter) + delete gfe->filter; + my_free(gfe); +} + +Id_delegating_gtid_event_filter::Id_delegating_gtid_event_filter() + : m_num_stateful_filters(0), m_num_completed_filters(0) +{ + my_hash_init(PSI_INSTRUMENT_ME, &m_filters_by_id_hash, &my_charset_bin, 32, + offsetof(gtid_filter_element, identifier), + sizeof(gtid_filter_identifier), NULL, free_gtid_filter_element, + HASH_UNIQUE); + + m_default_filter= new Accept_all_gtid_filter(); +} + +Id_delegating_gtid_event_filter::~Id_delegating_gtid_event_filter() +{ + my_hash_free(&m_filters_by_id_hash); + delete m_default_filter; +} + +void Id_delegating_gtid_event_filter::set_default_filter( + Gtid_event_filter *filter) +{ + if (m_default_filter) + delete m_default_filter; + + m_default_filter= filter; +} + +gtid_filter_element * +Id_delegating_gtid_event_filter::find_or_create_filter_element_for_id( + gtid_filter_identifier filter_id) +{ + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); + + if (!fe) + { + gtid_filter_element *new_fe= (gtid_filter_element *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(gtid_filter_element), MYF(MY_WME)); + new_fe->filter= NULL; + new_fe->identifier= filter_id; + if (my_hash_insert(&m_filters_by_id_hash, (uchar*) new_fe)) + { + my_free(new_fe); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + fe= new_fe; + } + + return fe; +} + +my_bool Id_delegating_gtid_event_filter::has_finished() +{ + /* + If all user-defined filters have deactivated, we are effectively + deactivated + */ + return m_num_stateful_filters && + m_num_completed_filters == m_num_stateful_filters; +} + +my_bool Id_delegating_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + gtid_filter_identifier filter_id= get_id_from_gtid(gtid); + gtid_filter_element *filter_element= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); + Gtid_event_filter *filter= + (filter_element ? filter_element->filter : m_default_filter); + my_bool ret= TRUE; + + if(!filter_element || !filter->has_finished()) + { + ret= filter->exclude(gtid); + + /* + If this is an explicitly defined filter, e.g. Window-based filter, check + if it has completed, and update the counter accordingly if so. + */ + if (filter_element && filter->has_finished()) + m_num_completed_filters++; + } + + return ret; +} + +Window_gtid_event_filter * +Domain_gtid_event_filter::find_or_create_window_filter_for_id( + uint32 domain_id) +{ + gtid_filter_element *filter_element= + find_or_create_filter_element_for_id(domain_id); + Window_gtid_event_filter *wgef= NULL; + + if (filter_element->filter == NULL) + { + /* New filter */ + wgef= new Window_gtid_event_filter(); + filter_element->filter= wgef; + } + else if (filter_element->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE) + { + /* We have an existing window filter here */ + wgef= (Window_gtid_event_filter *) filter_element->filter; + } + else + { + /* + We have an existing filter but it is not of window type so propogate NULL + filter + */ + sql_print_error("cannot subset domain id %d by position, another rule " + "exists on that domain", + domain_id); + } + + return wgef; +} + +static my_bool check_filter_entry_validity(void *entry, void *are_filters_invalid_arg) +{ + gtid_filter_element *fe= (gtid_filter_element*) entry; + + if (fe) + { + Gtid_event_filter *gef= fe->filter; + if (gef->get_filter_type() == Gtid_event_filter::WINDOW_GTID_FILTER_TYPE) + { + Window_gtid_event_filter *wgef= (Window_gtid_event_filter *) gef; + if (wgef->is_range_invalid()) + { + *((int *) are_filters_invalid_arg)= 1; + return TRUE; + } + } + } + return FALSE; +} + +int Domain_gtid_event_filter::validate_window_filters() +{ + int are_filters_invalid= 0; + my_hash_iterate(&m_filters_by_id_hash, check_filter_entry_validity, &are_filters_invalid); + return are_filters_invalid; +} + +int Domain_gtid_event_filter::add_start_gtid(rpl_gtid *gtid) +{ + int err= 0; + Window_gtid_event_filter *filter_to_update= + find_or_create_window_filter_for_id(gtid->domain_id); + + if (filter_to_update == NULL) + { + err= 1; + } + else if (!(err= filter_to_update->set_start_gtid(gtid))) + { + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + insert_dynamic(&m_start_filters, (const void *) &fe); + } + + return err; +} + +int Domain_gtid_event_filter::add_stop_gtid(rpl_gtid *gtid) +{ + int err= 0; + Window_gtid_event_filter *filter_to_update= + find_or_create_window_filter_for_id(gtid->domain_id); + + if (filter_to_update == NULL) + { + err= 1; + } + else if (!(err= filter_to_update->set_stop_gtid(gtid))) + { + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + insert_dynamic(&m_stop_filters, (const void *) &fe); + + /* + A window with a stop position can be disabled, and is therefore stateful. + */ + m_num_stateful_filters++; + + /* + Default filtering behavior changes with GTID stop positions, where we + exclude all domains not present in the stop list + */ + if (m_default_filter->get_filter_type() == ACCEPT_ALL_GTID_FILTER_TYPE) + { + delete m_default_filter; + m_default_filter= new Reject_all_gtid_filter(); + } + } + + return err; +} + +rpl_gtid *Domain_gtid_event_filter::get_start_gtids() +{ + rpl_gtid *gtid_list; + uint32 i; + size_t n_start_gtids= get_num_start_gtids(); + + gtid_list= (rpl_gtid *) my_malloc( + PSI_INSTRUMENT_ME, n_start_gtids * sizeof(rpl_gtid), MYF(MY_WME)); + + for (i = 0; i < n_start_gtids; i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + rpl_gtid win_start_gtid= wgef->get_start_gtid(); + gtid_list[i]= win_start_gtid; + } + + return gtid_list; +} + +rpl_gtid *Domain_gtid_event_filter::get_stop_gtids() +{ + rpl_gtid *gtid_list; + uint32 i; + size_t n_stop_gtids= get_num_stop_gtids(); + + gtid_list= (rpl_gtid *) my_malloc( + PSI_INSTRUMENT_ME, n_stop_gtids * sizeof(rpl_gtid), MYF(MY_WME)); + + for (i = 0; i < n_stop_gtids; i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + rpl_gtid win_stop_gtid= wgef->get_stop_gtid(); + gtid_list[i]= win_stop_gtid; + } + + return gtid_list; +} + +void Domain_gtid_event_filter::clear_start_gtids() +{ + uint32 i; + for (i = 0; i < get_num_start_gtids(); i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + if (wgef->has_stop()) + { + /* + Don't delete the whole filter if it already has a stop position attached + */ + wgef->clear_start_pos(); + } + else + { + /* + This domain only has a stop, so delete the whole filter + */ + my_hash_delete(&m_filters_by_id_hash, (uchar *) fe); + } + } + + reset_dynamic(&m_start_filters); +} + +void Domain_gtid_event_filter::clear_stop_gtids() +{ + uint32 i; + + for (i = 0; i < get_num_stop_gtids(); i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + if (wgef->has_start()) + { + /* + Don't delete the whole filter if it already has a start position + attached + */ + wgef->clear_stop_pos(); + } + else + { + /* + This domain only has a start, so delete the whole filter + */ + my_hash_delete(&m_filters_by_id_hash, (uchar *) fe); + } + m_num_stateful_filters--; + } + + /* + Stop positions were cleared and we want to be inclusive again of other + domains again + */ + if (m_default_filter->get_filter_type() == REJECT_ALL_GTID_FILTER_TYPE) + { + delete m_default_filter; + m_default_filter= new Accept_all_gtid_filter(); + } + + reset_dynamic(&m_stop_filters); +} + +my_bool Domain_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + my_bool include_domain= TRUE; + /* + If GTID stop positions are provided, we limit the domains which are output + to only be those specified with stop positions + */ + if (get_num_stop_gtids()) + { + gtid_filter_identifier filter_id= get_id_from_gtid(gtid); + gtid_filter_element *filter_element= + (gtid_filter_element *) my_hash_search(&m_filters_by_id_hash, + (const uchar *) &filter_id, 0); + if (filter_element) + { + Gtid_event_filter *filter= filter_element->filter; + if (filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE) + { + Window_gtid_event_filter *wgef= (Window_gtid_event_filter *) filter; + include_domain= wgef->has_stop(); + } + } + } + + return include_domain ? Id_delegating_gtid_event_filter::exclude(gtid) + : TRUE; +} diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index c8decff8fe8..ffe804a8f01 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -29,6 +29,7 @@ class String; #define PARAM_GTID(G) G.domain_id, G.server_id, G.seq_no #define GTID_MAX_STR_LENGTH (10+1+10+1+20) +#define PARAM_GTID(G) G.domain_id, G.server_id, G.seq_no struct rpl_gtid { @@ -37,6 +38,9 @@ struct rpl_gtid uint64 seq_no; }; +/* Data structure to help with quick lookup for filters. */ +typedef decltype(rpl_gtid::domain_id) gtid_filter_identifier; + inline bool operator==(const rpl_gtid& lhs, const rpl_gtid& rhs) { return @@ -45,6 +49,18 @@ inline bool operator==(const rpl_gtid& lhs, const rpl_gtid& rhs) lhs.seq_no == rhs.seq_no; }; +inline bool operator<(const rpl_gtid& lhs, const rpl_gtid& rhs) +{ + return (lhs.domain_id == rhs.domain_id) ? lhs.seq_no < rhs.seq_no + : lhs.domain_id < rhs.domain_id; +}; + +inline bool operator>(const rpl_gtid& lhs, const rpl_gtid& rhs) +{ + return (lhs.domain_id == rhs.domain_id) ? lhs.seq_no > rhs.seq_no + : lhs.domain_id > rhs.domain_id; +}; + enum enum_gtid_skip_type { GTID_SKIP_NOT, GTID_SKIP_STANDALONE, GTID_SKIP_TRANSACTION }; @@ -382,5 +398,449 @@ extern bool rpl_slave_state_tostring_helper(String *dest, const rpl_gtid *gtid, extern int gtid_check_rpl_slave_state_table(TABLE *table); extern rpl_gtid *gtid_parse_string_to_list(const char *p, size_t len, uint32 *out_len); +extern rpl_gtid *gtid_unpack_string_to_list(const char *p, size_t len, + uint32 *out_len); + + + +/* + This class ensures that the GTID state of an event stream is consistent with + the set of provided binary log files. In particular, it has two concerns: + + 1) Ensuring that GTID events are monotonically increasing within each + domain + 2) Ensuring that the GTID state of the specified binary logs is consistent + both with the initial state that a user provides, and between + binary logs (if multiple are specified) +*/ +class Binlog_gtid_state_validator +{ +public: + + struct audit_elem + { + uint32 domain_id; + + + /* + Holds the largest GTID received, and is indexed by domain_id + */ + rpl_gtid last_gtid; + + /* + Holds the largest GTID received, and is indexed by domain_id + */ + rpl_gtid start_gtid; + + /* + List of the problematic GTIDs received which were out of order + */ + DYNAMIC_ARRAY late_gtids_real; + + /* + For each problematic GTID in late_gtids_real, this list contains the last + GTID of the domain at the time of receiving the out of order GTID. + */ + DYNAMIC_ARRAY late_gtids_previous; + }; + + Binlog_gtid_state_validator(); + ~Binlog_gtid_state_validator(); + + /* + Initialize where we should start monitoring for invalid GTID entries + in the event stream. Note that these start positions must occur at or after + a given binary logs GTID state (from Gtid_list_log_event) + */ + void initialize_start_gtids(rpl_gtid *start_gtids, size_t n_gtids); + + /* + Initialize our current state so we know where to expect GTIDs to start + increasing from. Error if the state exists after our expected start_gtid + positions, because we know we will be missing event data (possibly from + a purged log). + */ + my_bool initialize_gtid_state(FILE *out, rpl_gtid *gtids, size_t n_gtids); + + /* + Ensures that the expected stop GTID positions exist within the specified + binary logs. + */ + my_bool verify_stop_state(FILE *out, rpl_gtid *stop_gtids, size_t n_stop_gtids); + + /* + Ensure a GTID state (e.g., from a Gtid_list_log_event) is consistent with + the current state of our auditing. For example, if we see a GTID from a + Gtid_list_log_event that is ahead of our current state for that domain, we + have missed events (perhaps from a missing log). + */ + my_bool verify_gtid_state(FILE *out, rpl_gtid *gtid_state_cur); + + /* + Take note of a new GTID being processed. + + returns TRUE if the GTID is invalid, FALSE on success + */ + my_bool record(rpl_gtid *gtid); + + /* + Writes warnings/errors (if any) during GTID processing + + Returns TRUE if any findings were reported, FALSE otherwise + */ + my_bool report(FILE *out, my_bool is_strict_mode); + + static void report_details(FILE *out, const char *format, va_list args) + { + vfprintf(out, format, args); + fprintf(out, "\n"); + } + + static void warn(FILE *out, const char *format,...) + { + va_list args; + va_start(args, format); + fprintf(out, "WARNING: "); + report_details(out, format, args); + } + + static void error(FILE *out, const char *format,...) + { + va_list args; + va_start(args, format); + fprintf(out, "ERROR: "); + report_details(out, format, args); + } + +private: + + /* + Holds the records for each domain id we are monitoring. Elements are of type + `struct audit_elem` and indexed by domian_id. + */ + HASH m_audit_elem_domain_lookup; +}; + +/* + Interface to support different methods of filtering log events by GTID +*/ +class Gtid_event_filter +{ +public: + Gtid_event_filter() {}; + virtual ~Gtid_event_filter() {}; + + enum gtid_event_filter_type + { + DELEGATING_GTID_FILTER_TYPE = 1, + WINDOW_GTID_FILTER_TYPE = 2, + ACCEPT_ALL_GTID_FILTER_TYPE = 3, + REJECT_ALL_GTID_FILTER_TYPE = 4 + }; + + /* + Run the filter on an input gtid to test if the corresponding log events + should be excluded from a result + + Returns TRUE when the event group corresponding to the input GTID should be + excluded. + Returns FALSE when the event group should be included. + */ + virtual my_bool exclude(rpl_gtid *) = 0; + + /* + The gtid_event_filter_type that corresponds to the underlying filter + implementation + */ + virtual uint32 get_filter_type() = 0; + + /* + For filters that can maintain their own state, this tests if the filter + implementation has completed. + + Returns TRUE when completed, and FALSE when the filter has not finished. + */ + virtual my_bool has_finished() = 0; +}; + +/* + Filter implementation which will include any and all input GTIDs. This is + used to set default behavior for GTIDs that do not have explicit filters + set on their domain_id, e.g. when a Window_gtid_event_filter is used for + a specific domain, then all other domain_ids will be accepted using this + filter implementation. +*/ +class Accept_all_gtid_filter : public Gtid_event_filter +{ +public: + Accept_all_gtid_filter() {} + ~Accept_all_gtid_filter() {} + my_bool exclude(rpl_gtid *gtid) { return FALSE; } + uint32 get_filter_type() { return ACCEPT_ALL_GTID_FILTER_TYPE; } + my_bool has_finished() { return FALSE; } +}; + +/* + Filter implementation to exclude all tested GTIDs. +*/ +class Reject_all_gtid_filter : public Gtid_event_filter +{ +public: + Reject_all_gtid_filter() {} + ~Reject_all_gtid_filter() {} + my_bool exclude(rpl_gtid *gtid) { return TRUE; } + uint32 get_filter_type() { return REJECT_ALL_GTID_FILTER_TYPE; } + my_bool has_finished() { return FALSE; } +}; + +/* + A filter implementation that includes events that exist between two GTID + positions, m_start (exclusive) and m_stop (inclusive), within a domain. + + This filter is stateful, such that it expects GTIDs to be an increasing + stream, and internally, the window will activate and deactivate when the start + and stop positions of the event stream have passed through, respectively. +*/ +class Window_gtid_event_filter : public Gtid_event_filter +{ +public: + Window_gtid_event_filter(); + ~Window_gtid_event_filter() {} + + my_bool exclude(rpl_gtid*); + my_bool has_finished(); + + /* + Set the GTID that begins this window (exclusive) + + Returns 0 on ok, non-zero on error + */ + int set_start_gtid(rpl_gtid *start); + + /* + Set the GTID that ends this window (inclusive) + + Returns 0 on ok, non-zero on error + */ + int set_stop_gtid(rpl_gtid *stop); + + uint32 get_filter_type() { return WINDOW_GTID_FILTER_TYPE; } + + /* + Validates the underlying range is correct, and writes an error if not, i.e. + m_start >= m_stop. + + Returns FALSE on ok, TRUE if range is invalid + */ + my_bool is_range_invalid(); + + /* + Getter/setter methods + */ + my_bool has_start() { return m_has_start; } + my_bool has_stop() { return m_has_stop; } + rpl_gtid get_start_gtid() { return m_start; } + rpl_gtid get_stop_gtid() { return m_stop; } + + void clear_start_pos() + { + m_has_start= FALSE; + m_start= {0, 0, 0}; + } + + void clear_stop_pos() + { + m_has_stop= FALSE; + m_stop= {0, 0, 0}; + } + +protected: + + /* + When processing GTID streams, the order in which they are processed should + be sequential with no gaps between events. If a gap is found within a + window, warn the user. + */ + void verify_gtid_is_expected(rpl_gtid *gtid); + +private: + + enum warning_flags + { + WARN_GTID_SEQUENCE_NUMBER_OUT_OF_ORDER= 0x1 + }; + + /* + m_has_start : Indicates if a start to this window has been explicitly + provided. A window starts immediately if not provided. + */ + my_bool m_has_start; + + /* + m_has_stop : Indicates if a stop to this window has been explicitly + provided. A window continues indefinitely if not provided. + */ + my_bool m_has_stop; + + /* + m_is_active : Indicates whether or not the program is currently reading + events from within this window. When TRUE, events with + different server ids than those specified by m_start or + m_stop will be passed through. + */ + my_bool m_is_active; + + /* + m_has_passed : Indicates whether or not the program is currently reading + events from within this window. + */ + my_bool m_has_passed; + + /* m_start : marks the GTID that begins the window (exclusive). */ + rpl_gtid m_start; + + /* m_stop : marks the GTID that ends the range (inclusive). */ + rpl_gtid m_stop; +}; + +typedef struct _gtid_filter_element +{ + Gtid_event_filter *filter; + gtid_filter_identifier identifier; /* Used for HASH lookup */ +} gtid_filter_element; + +/* + Gtid_event_filter subclass which has no specific implementation, but rather + delegates the filtering to specific identifiable/mapped implementations. + + A default filter is used for GTIDs that are passed through which no explicit + filter can be identified. + + This class should be subclassed, where the get_id_from_gtid function + specifies how to extract the filter identifier from a GTID. +*/ +class Id_delegating_gtid_event_filter : public Gtid_event_filter +{ +public: + Id_delegating_gtid_event_filter(); + ~Id_delegating_gtid_event_filter(); + + my_bool exclude(rpl_gtid *gtid); + my_bool has_finished(); + void set_default_filter(Gtid_event_filter *default_filter); + + uint32 get_filter_type() { return DELEGATING_GTID_FILTER_TYPE; } + + virtual gtid_filter_identifier get_id_from_gtid(rpl_gtid *) = 0; + +protected: + + uint32 m_num_stateful_filters; + uint32 m_num_completed_filters; + Gtid_event_filter *m_default_filter; + + HASH m_filters_by_id_hash; + + gtid_filter_element *find_or_create_filter_element_for_id(gtid_filter_identifier); +}; + +/* + A subclass of Id_delegating_gtid_event_filter which identifies filters using the + domain id of a GTID. + + Additional helper functions include: + add_start_gtid(GTID) : adds a start GTID position to this filter, to be + identified by its domain id + add_stop_gtid(GTID) : adds a stop GTID position to this filter, to be + identified by its domain id + clear_start_gtids() : removes existing GTID start positions + clear_stop_gtids() : removes existing GTID stop positions + get_start_gtids() : gets all added GTID start positions + get_stop_gtids() : gets all added GTID stop positions + get_num_start_gtids() : gets the count of added GTID start positions + get_num_stop_gtids() : gets the count of added GTID stop positions +*/ +class Domain_gtid_event_filter : public Id_delegating_gtid_event_filter +{ +public: + Domain_gtid_event_filter() + { + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_start_filters, + sizeof(gtid_filter_element*), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_stop_filters, + sizeof(gtid_filter_element*), 8, 8, MYF(0)); + } + ~Domain_gtid_event_filter() + { + delete_dynamic(&m_start_filters); + delete_dynamic(&m_stop_filters); + } + + /* + Returns the domain id of from the input GTID + */ + gtid_filter_identifier get_id_from_gtid(rpl_gtid *gtid) + { + return gtid->domain_id; + } + + /* + Override Id_delegating_gtid_event_filter to extend with domain specific + filtering logic + */ + my_bool exclude(rpl_gtid*); + + /* + Validates that window filters with both a start and stop GTID satisfy + stop_gtid > start_gtid + + Returns 0 on ok, non-zero if any windows are invalid. + */ + int validate_window_filters(); + + /* + Helper function to start a GTID window filter at the given GTID + + Returns 0 on ok, non-zero on error + */ + int add_start_gtid(rpl_gtid *gtid); + + /* + Helper function to end a GTID window filter at the given GTID + + Returns 0 on ok, non-zero on error + */ + int add_stop_gtid(rpl_gtid *gtid); + + /* + If start or stop position is respecified, we remove all existing values + and start over with the new specification. + */ + void clear_start_gtids(); + void clear_stop_gtids(); + + /* + Return list of all GTIDs used as start position. + + Note that this list is allocated and it is up to the user to free it + */ + rpl_gtid *get_start_gtids(); + + /* + Return list of all GTIDs used as stop position. + + Note that this list is allocated and it is up to the user to free it + */ + rpl_gtid *get_stop_gtids(); + + size_t get_num_start_gtids() { return m_start_filters.elements; } + size_t get_num_stop_gtids() { return m_stop_filters.elements; } + +private: + DYNAMIC_ARRAY m_start_filters; + DYNAMIC_ARRAY m_stop_filters; + + Window_gtid_event_filter *find_or_create_window_filter_for_id(gtid_filter_identifier); +}; #endif /* RPL_GTID_H */ diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 8322bcd3042..b9aea39e547 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2006, 2017, Oracle and/or its affiliates. - Copyright (c) 2010, 2017, MariaDB Corporation + Copyright (c) 2010, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -43,8 +43,7 @@ Master_info::Master_info(LEX_CSTRING *connection_name_arg, gtid_reconnect_event_skip_count(0), gtid_event_seen(false), in_start_all_slaves(0), in_stop_all_slaves(0), in_flush_all_relay_logs(0), users(0), killed(0), - total_ddl_groups(0), total_non_trans_groups(0), total_trans_groups(0), - do_accept_own_server_id(false) + total_ddl_groups(0), total_non_trans_groups(0), total_trans_groups(0) { char *tmp; host[0] = 0; user[0] = 0; password[0] = 0; @@ -86,6 +85,14 @@ Master_info::Master_info(LEX_CSTRING *connection_name_arg, mysql_mutex_init(key_master_info_data_lock, &data_lock, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_master_info_start_stop_lock, &start_stop_lock, MY_MUTEX_INIT_SLOW); + /* + start_alter_lock will protect individual start_alter_info while + start_alter_list_lock is for list insertion and deletion operations + */ + mysql_mutex_init(key_master_info_start_alter_lock, &start_alter_lock, + MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_master_info_start_alter_list_lock, &start_alter_list_lock, + MY_MUTEX_INIT_FAST); mysql_mutex_setflags(&run_lock, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_setflags(&data_lock, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_init(key_master_info_sleep_lock, &sleep_lock, MY_MUTEX_INIT_FAST); @@ -93,6 +100,7 @@ Master_info::Master_info(LEX_CSTRING *connection_name_arg, mysql_cond_init(key_master_info_start_cond, &start_cond, NULL); mysql_cond_init(key_master_info_stop_cond, &stop_cond, NULL); mysql_cond_init(key_master_info_sleep_cond, &sleep_cond, NULL); + init_sql_alloc(PSI_INSTRUMENT_ME, &mem_root, MEM_ROOT_BLOCK_SIZE, 0, MYF(0)); } @@ -122,10 +130,13 @@ Master_info::~Master_info() mysql_mutex_destroy(&data_lock); mysql_mutex_destroy(&sleep_lock); mysql_mutex_destroy(&start_stop_lock); + mysql_mutex_destroy(&start_alter_lock); + mysql_mutex_destroy(&start_alter_list_lock); mysql_cond_destroy(&data_cond); mysql_cond_destroy(&start_cond); mysql_cond_destroy(&stop_cond); mysql_cond_destroy(&sleep_cond); + free_root(&mem_root, MYF(0)); } /** @@ -750,7 +761,7 @@ int flush_master_info(Master_info* mi, (1 + mi->ignore_server_ids.elements), MYF(MY_WME)); if (!ignore_server_ids_buf) DBUG_RETURN(1); /* error */ - ulong cur_len= sprintf(ignore_server_ids_buf, "%u", + ulong cur_len= sprintf(ignore_server_ids_buf, "%zu", mi->ignore_server_ids.elements); for (ulong i= 0; i < mi->ignore_server_ids.elements; i++) { @@ -1456,11 +1467,32 @@ bool Master_info_index::add_master_info(Master_info *mi, bool write_to_file) atomic */ -bool Master_info_index::remove_master_info(Master_info *mi) +bool Master_info_index::remove_master_info(Master_info *mi, bool clear_log_files) { + char tmp_name[FN_REFLEN]; DBUG_ENTER("remove_master_info"); mysql_mutex_assert_owner(&LOCK_active_mi); + if (clear_log_files) + { + /* This code is only executed when change_master() failes to create a new master info */ + + // Delete any temporary relay log files that could have been created by change_master() + mi->rli.relay_log.reset_logs(current_thd, 0, (rpl_gtid*) 0, 0, 0); + /* Delete master-'connection'.info */ + create_logfile_name_with_suffix(tmp_name, + sizeof(tmp_name), + master_info_file, 0, + &mi->cmp_connection_name); + my_delete(tmp_name, MYF(0)); + /* Delete relay-log-'connection'.info */ + create_logfile_name_with_suffix(tmp_name, + sizeof(tmp_name), + relay_log_info_file, 0, + &mi->cmp_connection_name); + my_delete(tmp_name, MYF(0)); + } + // Delete Master_info and rewrite others to file if (!my_hash_delete(&master_info_hash, (uchar*) mi)) { @@ -1917,7 +1949,7 @@ char *Domain_id_filter::as_string(enum_list_type type) return NULL; // Store the total number of elements followed by the individual elements. - size_t cur_len= sprintf(buf, "%u", ids->elements); + size_t cur_len= sprintf(buf, "%zu", ids->elements); sz-= cur_len; for (uint i= 0; i < ids->elements; i++) diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index 1377a816d48..ecfecabd6c9 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -227,7 +227,7 @@ class Master_info : public Slave_reporting_capability File fd; // we keep the file open, so we need to remember the file pointer IO_CACHE file; - mysql_mutex_t data_lock, run_lock, sleep_lock, start_stop_lock; + mysql_mutex_t data_lock, run_lock, sleep_lock, start_stop_lock, start_alter_lock, start_alter_list_lock; mysql_cond_t data_cond, start_cond, stop_cond, sleep_cond; THD *io_thd; MYSQL* mysql; @@ -365,7 +365,26 @@ class Master_info : public Slave_reporting_capability to slave) gtid exists in the server's binlog. Then, in gtid strict mode, it must be ignored similarly to the replicate-same-server-id rule. */ - bool do_accept_own_server_id; + bool do_accept_own_server_id= false; + List <start_alter_info> start_alter_list; + MEM_ROOT mem_root; + /* + Flag is raised at the parallel worker slave stop. Its purpose + is to mark the whole start_alter_list when slave stops. + The flag is read by Start Alter event to self-mark its state accordingly + at time its alter info struct is about to be appened to the list. + */ + bool is_shutdown= false; +}; + +struct start_alter_thd_args +{ + rpl_group_info *rgi; + LEX_CSTRING query; + LEX_CSTRING *db; + char *catalog; + bool shutdown; + CHARSET_INFO *cs; }; int init_master_info(Master_info* mi, const char* master_info_fname, @@ -404,7 +423,7 @@ public: bool check_duplicate_master_info(LEX_CSTRING *connection_name, const char *host, uint port); bool add_master_info(Master_info *mi, bool write_to_file); - bool remove_master_info(Master_info *mi); + bool remove_master_info(Master_info *mi, bool clear_log_files); Master_info *get_master_info(const LEX_CSTRING *connection_name, Sql_condition::enum_warning_level warning); bool start_all_slaves(THD *thd); diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 78d81d973e4..1e07ca582da 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -150,6 +150,9 @@ finish_event_group(rpl_parallel_thread *rpt, uint64 sub_id, wait_for_commit *wfc= &rgi->commit_orderer; int err; + if (rgi->get_finish_event_group_called()) + return; + thd->get_stmt_da()->set_overwrite_status(true); /* Remove any left-over registration to wait for a prior commit to @@ -279,6 +282,8 @@ finish_event_group(rpl_parallel_thread *rpt, uint64 sub_id, */ thd->get_stmt_da()->reset_diagnostics_area(); wfc->wakeup_subsequent_commits(rgi->worker_error); + rgi->did_mark_start_commit= false; + rgi->set_finish_event_group_called(true); } @@ -583,6 +588,7 @@ rpl_pause_for_ftwrl(THD *thd) uint32 i; rpl_parallel_thread_pool *pool= &global_rpl_thread_pool; int err; + Dynamic_array<Master_info*> mi_arr(4, 4); // array of replication source mi:s DBUG_ENTER("rpl_pause_for_ftwrl"); /* @@ -634,9 +640,36 @@ rpl_pause_for_ftwrl(THD *thd) mysql_cond_wait(&e->COND_parallel_entry, &e->LOCK_parallel_entry); }; --e->need_sub_id_signal; + thd->EXIT_COND(&old_stage); if (err) break; + /* + Notify any source any domain waiting-for-master Start-Alter to give way. + */ + Master_info *mi= e->rli->mi; + bool found= false; + for (uint i= 0; i < mi_arr.elements() && !found; i++) + found= mi_arr.at(i) == mi; + if (!found) + { + mi_arr.append(mi); + start_alter_info *info=NULL; + mysql_mutex_lock(&mi->start_alter_list_lock); + List_iterator<start_alter_info> info_iterator(mi->start_alter_list); + while ((info= info_iterator++)) + { + mysql_mutex_lock(&mi->start_alter_lock); + + DBUG_ASSERT(info->state == start_alter_state::REGISTERED); + + info->state= start_alter_state::ROLLBACK_ALTER; + info->direct_commit_alter= true; + mysql_cond_broadcast(&info->start_alter_cond); + mysql_mutex_unlock(&mi->start_alter_lock); + } + mysql_mutex_unlock(&mi->start_alter_list_lock); + } } if (err) @@ -828,11 +861,9 @@ do_retry: { mysql_mutex_lock(&entry->LOCK_parallel_entry); register_wait_for_prior_event_group_commit(rgi, entry); - if (!(entry->stop_on_error_sub_id == (uint64) ULONGLONG_MAX || -#ifndef DBUG_OFF - (DBUG_EVALUATE_IF("simulate_mdev_12746", 1, 0)) || -#endif - rgi->gtid_sub_id < entry->stop_on_error_sub_id)) + if (entry->stop_on_error_sub_id != (uint64) ULONGLONG_MAX && + !DBUG_IF("simulate_mdev_12746") && + rgi->gtid_sub_id >= entry->stop_on_error_sub_id) { /* A failure of a preceding "parent" transaction may not be @@ -1712,6 +1743,9 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, { mysql_mutex_lock(&pool->threads[i]->LOCK_rpl_thread); pool->threads[i]->delay_start= false; + pool->threads[i]->current_start_alter_id= 0; + pool->threads[i]->current_start_alter_domain_id= 0; + pool->threads[i]->reserved_start_alter_thread= false; mysql_cond_signal(&pool->threads[i]->COND_rpl_thread); while (!pool->threads[i]->running) mysql_cond_wait(&pool->threads[i]->COND_rpl_thread, @@ -1992,7 +2026,19 @@ rpl_parallel_thread::get_rgi(Relay_log_info *rli, Gtid_log_event *gtid_ev, rgi->retry_start_offset= rli->future_event_relay_log_pos-event_size; rgi->retry_event_count= 0; rgi->killed_for_retry= rpl_group_info::RETRY_KILL_NONE; + /* rgi is transaction specific so we need to move this value to rgi */ + rgi->reserved_start_alter_thread= reserved_start_alter_thread; + rgi->rpt= this; + rgi->direct_commit_alter= false; + rgi->finish_event_group_called= false; + DBUG_ASSERT(!rgi->sa_info); + /* + We can remove the reserved_start_alter_thread flag. + If we get more concurrent alter handle_split_alter will + automatically set this flag again. + */ + reserved_start_alter_thread= false; return rgi; } @@ -2063,6 +2109,10 @@ rpl_parallel_thread::loc_free_gco(group_commit_orderer *gco) loc_gco_list= gco; } +void rpl_group_info::finish_start_alter_event_group() +{ + finish_event_group(rpt, this->gtid_sub_id, this->parallel_entry, this); +} rpl_parallel_thread::rpl_parallel_thread() : channel_name_length(0), last_error_number(0), last_error_timestamp(0), @@ -2072,7 +2122,7 @@ rpl_parallel_thread::rpl_parallel_thread() rpl_parallel_thread_pool::rpl_parallel_thread_pool() - : threads(0), free_list(0), count(0), inited(false), busy(false), + : threads(0), free_list(0), count(0), inited(false),current_start_alters(0), busy(false), pfs_bkp{0, false, false, NULL} { } @@ -2206,6 +2256,129 @@ rpl_parallel_thread_pool::copy_pool_for_pfs(Relay_log_info *rli) } } +/* + START ALTER , COMMIT ALTER / ROLLBACK ALTER scheduling + + Steps:- + 1. (For Gtid_log_event SA). Get the worker thread which is either + e->rpl_threads[i] is NULL means worker from poll has not been assigned yet + e->rpl_threads[i]->current_owner != &e->rpl_threads[i] + Thread has been released, or about to //same as choose_thread logic + !e->rpl_threads[i]->current_start_alter_id is 0 , safe to schedule. + We dont want to schedule on worker which already have been scheduled SA + but CA/RA has not been scheduled yet. current_start_alter_id will indicate + this. If we dont do this we will get deadlock. + 2. (For Gtid_log_event SA) + call choose_thread_internal so that e->rpl_threads[idx] is not null + update the current_start_alter_id + 3. (For Gtid_log_event SA) + update local e->pending_start_alters(local) variable and + pool->current_start_alters(global) + We need 2 status variable (global and local) because we can have + slave_domain_parallel_threads != pool->threads. + 4. (For CA/RA Gtid_log_event) + Update e->pending_start_alters and pool->current_start_alters + while holding mutex lock on pool (if SA is not assigned to + reserved thread) + + + @returns + true Worker allocated (choose_thread_internal called) + false Worker not allocated (choose_thread_internal not called) +*/ +static bool handle_split_alter(rpl_parallel_entry *e, + Gtid_log_event *gtid_ev, uint32 *idx, + //choose_thread_internal specific + bool *did_enter_cond, rpl_group_info* rgi, + PSI_stage_info *old_stage) +{ + uint16 flags_extra= gtid_ev->flags_extra; + bool thread_allocated= false; + //Step 1 + if (flags_extra & Gtid_log_event::FL_START_ALTER_E1 || + //This will arrange finding threads for CA/RA as well + //as concurrent DDL + e->pending_start_alters) + { + /* + j is needed for round robin scheduling, we will start with rpl_thread_idx + go till rpl_thread_max and then start with 0 to rpl_thread_idx + */ + int j= e->rpl_thread_idx; + for(uint i= 0; i < e->rpl_thread_max; i++) + { + if (!e->rpl_threads[j] || e->rpl_threads[j]->current_owner + != &e->rpl_threads[j] || !e->rpl_threads[j]->current_start_alter_id) + { + //This condition will hit atleast one time no matter what happens + *idx= j; + DBUG_PRINT("info", ("Start alter id %d", j)); + goto idx_found; + } + j++; + j= j % e->rpl_thread_max; + } + //We did not find and idx + DBUG_ASSERT(0); + return false; +idx_found: + e->rpl_thread_idx= *idx; + e->choose_thread_internal(*idx, did_enter_cond, rgi, old_stage); + thread_allocated= true; + if (flags_extra & Gtid_log_event::FL_START_ALTER_E1) + { + mysql_mutex_assert_owner(&e->rpl_threads[*idx]->LOCK_rpl_thread); + e->rpl_threads[e->rpl_thread_idx]->current_start_alter_id= gtid_ev->seq_no; + e->rpl_threads[e->rpl_thread_idx]->current_start_alter_domain_id= + gtid_ev->domain_id; + /* + We are locking LOCK_rpl_thread_pool becuase we are going to update + current_start_alters + */ + mysql_mutex_lock(&global_rpl_thread_pool.LOCK_rpl_thread_pool); + if (e->pending_start_alters < e->rpl_thread_max - 1 && + global_rpl_thread_pool.current_start_alters + < global_rpl_thread_pool.count - 1) + { + e->pending_start_alters++; + global_rpl_thread_pool.current_start_alters++; + } + else + { + e->rpl_threads[*idx]->reserved_start_alter_thread= true; + e->rpl_threads[*idx]->current_start_alter_id= 0; + e->rpl_threads[*idx]->current_start_alter_domain_id= 0; + } + mysql_mutex_unlock(&global_rpl_thread_pool.LOCK_rpl_thread_pool); + } + } + if(flags_extra & (Gtid_log_event::FL_COMMIT_ALTER_E1 | + Gtid_log_event::FL_ROLLBACK_ALTER_E1 )) + { + //Free the corrosponding rpt current_start_alter_id + for(uint i= 0; i < e->rpl_thread_max; i++) + { + if(e->rpl_threads[i] && + e->rpl_threads[i]->current_start_alter_id == gtid_ev->sa_seq_no && + e->rpl_threads[i]->current_start_alter_domain_id == gtid_ev->domain_id) + { + mysql_mutex_lock(&global_rpl_thread_pool.LOCK_rpl_thread_pool); + e->rpl_threads[i]->current_start_alter_id= 0; + e->rpl_threads[i]->current_start_alter_domain_id= 0; + global_rpl_thread_pool.current_start_alters--; + e->pending_start_alters--; + DBUG_PRINT("info", ("Commit/Rollback alter id %d", i)); + mysql_mutex_unlock(&global_rpl_thread_pool.LOCK_rpl_thread_pool); + break; + } + } + } + + return thread_allocated; + +} + + /* Obtain a worker thread that we can queue an event to. @@ -2239,25 +2412,32 @@ rpl_parallel_entry::choose_thread(rpl_group_info *rgi, bool *did_enter_cond, Gtid_log_event *gtid_ev) { uint32 idx; - Relay_log_info *rli= rgi->rli; - rpl_parallel_thread *thr; idx= rpl_thread_idx; if (gtid_ev) { + if (++idx >= rpl_thread_max) + idx= 0; + //rpl_thread_idx will be updated handle_split_alter + if (handle_split_alter(this, gtid_ev, &idx, did_enter_cond, rgi, old_stage)) + return rpl_threads[idx]; if (gtid_ev->flags2 & (Gtid_log_event::FL_COMPLETED_XA | Gtid_log_event::FL_PREPARED_XA)) + { idx= my_hash_sort(&my_charset_bin, gtid_ev->xid.key(), gtid_ev->xid.key_length()) % rpl_thread_max; - else - { - ++idx; - if (idx >= rpl_thread_max) - idx= 0; } rpl_thread_idx= idx; } - thr= rpl_threads[idx]; + return choose_thread_internal(idx, did_enter_cond, rgi, old_stage); +} + +rpl_parallel_thread * rpl_parallel_entry::choose_thread_internal(uint idx, + bool *did_enter_cond, rpl_group_info *rgi, + PSI_stage_info *old_stage) +{ + rpl_parallel_thread* thr= rpl_threads[idx]; + Relay_log_info *rli= rgi->rli; if (thr) { *did_enter_cond= false; @@ -2375,12 +2555,13 @@ rpl_parallel::~rpl_parallel() rpl_parallel_entry * -rpl_parallel::find(uint32 domain_id) +rpl_parallel::find(uint32 domain_id, Relay_log_info *rli) { struct rpl_parallel_entry *e; if (!(e= (rpl_parallel_entry *)my_hash_search(&domain_hash, - (const uchar *)&domain_id, 0))) + (const uchar *)&domain_id, + sizeof(domain_id)))) { /* Allocate a new, empty one. */ ulong count= opt_slave_domain_parallel_threads; @@ -2400,6 +2581,8 @@ rpl_parallel::find(uint32 domain_id) e->domain_id= domain_id; e->stop_on_error_sub_id= (uint64)ULONGLONG_MAX; e->pause_sub_id= (uint64)ULONGLONG_MAX; + e->pending_start_alters= 0; + e->rli= rli; mysql_mutex_init(key_LOCK_parallel_entry, &e->LOCK_parallel_entry, MY_MUTEX_INIT_FAST); mysql_cond_init(key_COND_parallel_entry, &e->COND_parallel_entry, NULL); @@ -2412,7 +2595,11 @@ rpl_parallel::find(uint32 domain_id) } } else + { + DBUG_ASSERT(rli == e->rli); + e->force_abort= false; + } return e; } @@ -2429,7 +2616,7 @@ rpl_parallel::wait_for_done(THD *thd, Relay_log_info *rli) struct rpl_parallel_entry *e; rpl_parallel_thread *rpt; uint32 i, j; - + Master_info *mi= rli->mi; /* First signal all workers that they must force quit; no more events will be queued to complete any partial event groups executed. @@ -2482,6 +2669,45 @@ rpl_parallel::wait_for_done(THD *thd, Relay_log_info *rli) #endif global_rpl_thread_pool.copy_pool_for_pfs(rli); + /* + Shutdown SA alter threads through marking their execution states + to force their early post-SA execution exit. Upon that the affected SA threads + change their state to COMPLETED, notify any waiting CA|RA and this thread. + */ + start_alter_info *info=NULL; + mysql_mutex_lock(&mi->start_alter_list_lock); + List_iterator<start_alter_info> info_iterator(mi->start_alter_list); + mi->is_shutdown= true; // a sign to stop in concurrently coming in new SA:s + while ((info= info_iterator++)) + { + mysql_mutex_lock(&mi->start_alter_lock); + if (info->state == start_alter_state::COMPLETED) + { + mysql_mutex_unlock(&mi->start_alter_lock); + continue; + } + info->state= start_alter_state::ROLLBACK_ALTER; + // Any possible CA that is (will be) waiting will complete this ALTER instance + info->direct_commit_alter= true; + mysql_cond_broadcast(&info->start_alter_cond); // notify SA:s + mysql_mutex_unlock(&mi->start_alter_lock); + + // await SA in the COMPLETED state + mysql_mutex_lock(&mi->start_alter_lock); + while(info->state == start_alter_state::ROLLBACK_ALTER) + mysql_cond_wait(&info->start_alter_cond, &mi->start_alter_lock); + + DBUG_ASSERT(info->state == start_alter_state::COMPLETED); + + mysql_mutex_unlock(&mi->start_alter_lock); + } + mysql_mutex_unlock(&mi->start_alter_list_lock); + + DBUG_EXECUTE_IF("rpl_slave_stop_CA_before_binlog", + { + debug_sync_set_action(thd, STRING_WITH_LEN("now signal proceed_CA_1")); + }); + for (i= 0; i < domain_hash.records; ++i) { e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i); @@ -2496,6 +2722,17 @@ rpl_parallel::wait_for_done(THD *thd, Relay_log_info *rli) } } } + // Now that all threads are docked, remained alter states are safe to destroy + mysql_mutex_lock(&mi->start_alter_list_lock); + info_iterator.rewind(); + while ((info= info_iterator++)) + { + info_iterator.remove(); + mysql_cond_destroy(&info->start_alter_cond); + my_free(info); + } + mi->is_shutdown= false; + mysql_mutex_unlock(&mi->start_alter_list_lock); } @@ -2840,7 +3077,7 @@ rpl_parallel::do_event(rpl_group_info *serial_rgi, Log_event *ev, uint32 domain_id= (rli->mi->using_gtid == Master_info::USE_GTID_NO || rli->mi->parallel_mode <= SLAVE_PARALLEL_MINIMAL ? 0 : gtid_ev->domain_id); - if (!(e= find(domain_id))) + if (!(e= find(domain_id, rli))) { my_error(ER_OUT_OF_RESOURCES, MYF(MY_WME)); delete ev; @@ -2852,6 +3089,7 @@ rpl_parallel::do_event(rpl_group_info *serial_rgi, Log_event *ev, gtid.server_id= gtid_ev->server_id; gtid.seq_no= gtid_ev->seq_no; rli->update_relay_log_state(>id, 1); + serial_rgi->gtid_ev_flags_extra= gtid_ev->flags_extra; if (process_gtid_for_restart_pos(rli, >id)) { /* diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h index b8f924b1714..f2bf36aa4a1 100644 --- a/sql/rpl_parallel.h +++ b/sql/rpl_parallel.h @@ -102,6 +102,18 @@ struct rpl_parallel_thread { bool running; bool stop; bool pause_for_ftwrl; + /* + 0 = No start alter assigned + >0 = Start alter assigned + */ + uint64 current_start_alter_id; + uint32 current_start_alter_domain_id; + /* + This flag is true when Start Alter just needs to be binlogged only. + This scenario will happens when there is congestion , and we can not + allocate independent worker to start alter. + */ + bool reserved_start_alter_thread; mysql_mutex_t LOCK_rpl_thread; mysql_cond_t COND_rpl_thread; mysql_cond_t COND_rpl_thread_queue; @@ -301,6 +313,12 @@ struct rpl_parallel_thread_pool { mysql_cond_t COND_rpl_thread_pool; uint32 count; bool inited; + + /* + Lock first LOCK_rpl_thread_pool and then LOCK_rpl_thread to + update this variable. + */ + uint32 current_start_alters; /* While FTWRL runs, this counter is incremented to make SQL thread or STOP/START slave not try to start new activity while that operation @@ -332,6 +350,7 @@ struct rpl_parallel_entry { */ uint32 need_sub_id_signal; uint64 last_commit_id; + uint32 pending_start_alters; bool active; /* Set when SQL thread is shutting down, and no more events can be processed, @@ -414,10 +433,15 @@ struct rpl_parallel_entry { uint64 count_committing_event_groups; /* The group_commit_orderer object for the events currently being queued. */ group_commit_orderer *current_gco; + /* Relay log info of replication source for this entry. */ + Relay_log_info *rli; rpl_parallel_thread * choose_thread(rpl_group_info *rgi, bool *did_enter_cond, PSI_stage_info *old_stage, Gtid_log_event *gtid_ev); + rpl_parallel_thread * + choose_thread_internal(uint idx, bool *did_enter_cond, rpl_group_info *rgi, + PSI_stage_info *old_stage); int queue_master_restart(rpl_group_info *rgi, Format_description_log_event *fdev); }; @@ -429,7 +453,7 @@ struct rpl_parallel { rpl_parallel(); ~rpl_parallel(); void reset(); - rpl_parallel_entry *find(uint32 domain_id); + rpl_parallel_entry *find(uint32 domain_id, Relay_log_info *rli); void wait_for_done(THD *thd, Relay_log_info *rli); void stop_during_until(); bool workers_idle(); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index f038a605906..23961686560 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1632,7 +1632,8 @@ scan_one_gtid_slave_pos_table(THD *thd, HASH *hash, DYNAMIC_ARRAY *array, goto end; } - if ((rec= my_hash_search(hash, (const uchar *)&domain_id, 0))) + if ((rec= my_hash_search(hash, (const uchar *)&domain_id, + sizeof(domain_id)))) { entry= (struct gtid_pos_element *)rec; if (entry->sub_id >= sub_id) @@ -2151,16 +2152,24 @@ rpl_group_info::reinit(Relay_log_info *rli) long_find_row_note_printed= false; did_mark_start_commit= false; gtid_ev_flags2= 0; + gtid_ev_flags_extra= 0; + gtid_ev_sa_seq_no= 0; last_master_timestamp = 0; gtid_ignore_duplicate_state= GTID_DUPLICATE_NULL; speculation= SPECULATE_NO; + rpt= NULL; + start_alter_ev= NULL; + direct_commit_alter= false; commit_orderer.reinit(); } rpl_group_info::rpl_group_info(Relay_log_info *rli) : thd(0), wait_commit_sub_id(0), wait_commit_group_info(0), parallel_entry(0), - deferred_events(NULL), m_annotate_event(0), is_parallel_exec(false) + deferred_events(NULL), m_annotate_event(0), is_parallel_exec(false), + gtid_ev_flags2(0), gtid_ev_flags_extra(0), gtid_ev_sa_seq_no(0), + reserved_start_alter_thread(0), finish_event_group_called(0), rpt(NULL), + start_alter_ev(NULL), direct_commit_alter(false), sa_info(NULL) { reinit(rli); bzero(¤t_gtid, sizeof(current_gtid)); @@ -2169,7 +2178,6 @@ rpl_group_info::rpl_group_info(Relay_log_info *rli) mysql_cond_init(key_rpl_group_info_sleep_cond, &sleep_cond, NULL); } - rpl_group_info::~rpl_group_info() { free_annotate_event(); @@ -2194,6 +2202,7 @@ event_group_new_gtid(rpl_group_info *rgi, Gtid_log_event *gev) rgi->current_gtid.seq_no= gev->seq_no; rgi->commit_id= gev->commit_id; rgi->gtid_pending= true; + rgi->sa_info= NULL; return 0; } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index cc807852bf2..80ee143a8e8 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -641,6 +641,33 @@ struct inuse_relaylog { } }; +enum start_alter_state +{ + INVALID= 0, + REGISTERED, // Start Alter exist, Default state + COMMIT_ALTER, // COMMIT the alter + ROLLBACK_ALTER, // Rollback the alter + COMPLETED // COMMIT/ROLLBACK Alter written in binlog +}; + +struct start_alter_info +{ + /* + ALTER id is defined as a pair of GTID's seq_no and domain_id. + */ + decltype(rpl_gtid::seq_no) sa_seq_no; // key for searching (SA's id) + uint32 domain_id; + bool direct_commit_alter; // when true CA thread executes the whole query + /* + 0 prepared and not error from commit and rollback + >0 error expected in commit/rollback + Rollback can be logged with 0 error if master is killed + */ + uint error; + enum start_alter_state state; + /* We are not using mysql_cond_t because we do not need PSI */ + mysql_cond_t start_alter_cond; +}; /* This is data for various state needed to be kept for the processing of @@ -760,6 +787,9 @@ struct rpl_group_info bool did_mark_start_commit; /* Copy of flags2 from GTID event. */ uchar gtid_ev_flags2; + /* Copy of flags3 from GTID event. */ + uint16 gtid_ev_flags_extra; + uint64 gtid_ev_sa_seq_no; enum { GTID_DUPLICATE_NULL=0, GTID_DUPLICATE_IGNORE=1, @@ -834,6 +864,15 @@ struct rpl_group_info RETRY_KILL_KILLED }; uchar killed_for_retry; + bool reserved_start_alter_thread; + bool finish_event_group_called; + /* + Used for two phase alter table + */ + rpl_parallel_thread *rpt; + Query_log_event *start_alter_ev; + bool direct_commit_alter; + start_alter_info *sa_info; rpl_group_info(Relay_log_info *rli_); ~rpl_group_info(); @@ -961,6 +1000,19 @@ struct rpl_group_info if (!is_parallel_exec) rli->event_relay_log_pos= future_event_relay_log_pos; } + + void finish_start_alter_event_group(); + + bool get_finish_event_group_called() + { + return finish_event_group_called; + } + + void set_finish_event_group_called(bool value) + { + finish_event_group_called= value; + } + }; diff --git a/sql/rpl_utility.cc b/sql/rpl_utility.cc index 9ea8bb3b822..04a2efb3750 100644 --- a/sql/rpl_utility.cc +++ b/sql/rpl_utility.cc @@ -338,7 +338,7 @@ bool event_checksum_test(uchar *event_buf, ulong event_len, DBUG_ASSERT(event_buf[EVENT_TYPE_OFFSET] == FORMAT_DESCRIPTION_EVENT); event_buf[FLAGS_OFFSET]= (uchar) flags; } - res= DBUG_EVALUATE_IF("simulate_checksum_test_failure", TRUE, computed != incoming); + res= (DBUG_IF("simulate_checksum_test_failure") || computed != incoming); } return res; } diff --git a/sql/semisync_master_ack_receiver.cc b/sql/semisync_master_ack_receiver.cc index b65b7824a0e..b54ad58d153 100644 --- a/sql/semisync_master_ack_receiver.cc +++ b/sql/semisync_master_ack_receiver.cc @@ -72,7 +72,7 @@ bool Ack_receiver::start() m_status= ST_UP; - if (DBUG_EVALUATE_IF("rpl_semisync_simulate_create_thread_failure", 1, 0) || + if (DBUG_IF("rpl_semisync_simulate_create_thread_failure") || pthread_attr_init(&attr) != 0 || pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE) != 0 || #ifndef _WIN32 @@ -247,7 +247,7 @@ void Ack_receiver::run() { mysql_mutex_unlock(&m_mutex); - ret= DBUG_EVALUATE_IF("rpl_semisync_simulate_select_error", -1, ret); + ret= DBUG_IF("rpl_semisync_simulate_select_error") ? -1 : ret; if (ret == -1 && errno != EINTR) sql_print_information("Failed to wait on semi-sync sockets, " diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index 3e7578b6a53..788aab78911 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -63,7 +63,7 @@ int Repl_semi_sync_slave::slave_read_sync_header(const uchar *header, if (rpl_semi_sync_slave_status) { - if (DBUG_EVALUATE_IF("semislave_corrupt_log", 0, 1) + if (!DBUG_IF("semislave_corrupt_log") && header[0] == k_packet_magic_num) { semi_sync_need_reply = (header[1] & k_packet_flag_sync); @@ -144,7 +144,7 @@ void Repl_semi_sync_slave::kill_connection(MYSQL *mysql) bool ret= (!mysql_real_connect(kill_mysql, mysql->host, mysql->user, mysql->passwd,0, mysql->port, mysql->unix_socket, 0)); - if (DBUG_EVALUATE_IF("semisync_slave_failed_kill", 1, 0) || ret) + if (DBUG_IF("semisync_slave_failed_kill") || ret) { sql_print_information("cannot connect to master to kill slave io_thread's " "connection"); @@ -198,8 +198,7 @@ int Repl_semi_sync_slave::request_transmit(Master_info *mi) } row= mysql_fetch_row(res); - if (DBUG_EVALUATE_IF("master_not_support_semisync", 1, 0) - || !row) + if (DBUG_IF("master_not_support_semisync") || !row) { /* Master does not support semi-sync */ sql_print_warning("Master server does not support semi-sync, " @@ -260,7 +259,7 @@ int Repl_semi_sync_slave::slave_reply(Master_info *mi) name_len + REPLY_BINLOG_NAME_OFFSET); if (!reply_res) { - reply_res = DBUG_EVALUATE_IF("semislave_failed_net_flush", 1, net_flush(net)); + reply_res = (DBUG_IF("semislave_failed_net_flush") || net_flush(net)); if (reply_res) sql_print_error("Semi-sync slave net_flush() reply failed"); rpl_semi_sync_slave_send_ack++; diff --git a/sql/set_var.cc b/sql/set_var.cc index 3dd97527433..aa9ec5ab5ca 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -85,7 +85,7 @@ uint sys_var_elements() int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags) { - uint saved_elements= long_options->elements; + size_t saved_elements= long_options->elements; DBUG_ENTER("sys_var_add_options"); diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 9fcbc324805..5956c159278 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -1788,7 +1788,7 @@ ER_TOO_LONG_KEY 42000 S1009 spa "Declaración de clave demasiado larga. La máxima longitud de clave es de %d" swe "För lång nyckel. Högsta tillåtna nyckellängd är %d" ukr "Зазначений ключ задовгий. Найбільша довжина ключа %d байтів" -ER_KEY_COLUMN_DOES_NOT_EXITS 42000 S1009 +ER_KEY_COLUMN_DOES_NOT_EXIST 42000 S1009 chi "索引列'%-.192s'不在表里" cze "Klíčový sloupec '%-.192s' v tabulce neexistuje" dan "Nøglefeltet '%-.192s' eksisterer ikke i tabellen" @@ -6638,12 +6638,12 @@ ER_PARTITION_MGMT_ON_NONPARTITIONED ER_FEATURE_NOT_SUPPORTED_WITH_PARTITIONING eng "Partitioned tables do not support %s" spa "Las tablas particionadas no soportan %s" -ER_DROP_PARTITION_NON_EXISTENT - chi "分区列表错误%-.64s" - eng "Error in list of partitions to %-.64s" - ger "Fehler in der Partitionsliste bei %-.64s" - spa "Error en lista de particiones para %-.64s" - swe "Fel i listan av partitioner att %-.64s" +ER_PARTITION_DOES_NOT_EXIST + chi "分区名称或分区列表错误" + eng "Wrong partition name or partition list" + ger "Falscher Name einer Partition oder Fehler in der Partitionsliste" + spa "Error en lista de particiones" + swe "Fel namn av en partition eller fel i listan av partitioner" ER_DROP_LAST_PARTITION chi "无法删除所有分区，请使用删除表" eng "Cannot remove all partitions, use DROP TABLE instead" @@ -9773,9 +9773,9 @@ ER_UNUSED_23 spa "Nunca debería vd de ver esto" ER_PARTITION_WRONG_TYPE - chi "错误的分区类型，预期类型：%`s" - eng "Wrong partitioning type, expected type: %`s" - spa "Tipo de partición equivocada, tipo esperado: %`s" + chi "错误分区类型%`s，应当是%`s" + eng "Wrong partition type %`s for partitioning by %`s" + spa "Tipo de partición equivocada %`s para particionado mediante %`s" WARN_VERS_PART_FULL chi "版本化表%`s.%`s：partition%`s已满，添加更多历史分区(out of %s)" @@ -10060,3 +10060,15 @@ ER_REMOVED_ORPHAN_TRIGGER ER_STORAGE_ENGINE_DISABLED eng "Storage engine %s is disabled" spa "El motor de almacenaje %s está desactivado" +WARN_SFORMAT_ERROR + eng "SFORMAT error: %s" +ER_PARTITION_CONVERT_SUBPARTITIONED + eng "Convert partition is not supported for subpartitioned table." +ER_PROVIDER_NOT_LOADED + eng "MariaDB tried to use the %s, but its provider plugin is not loaded" +ER_JSON_HISTOGRAM_PARSE_FAILED + eng "Failed to parse histogram for table %s.%s: %s at offset %d." +ER_SF_OUT_INOUT_ARG_NOT_ALLOWED + eng "OUT or INOUT argument %d for function %s is not allowed here" +ER_INCONSISTENT_SLAVE_TEMP_TABLE + eng "Replicated query '%s' table `%s.%s` can not be temporary" diff --git a/sql/slave.cc b/sql/slave.cc index cb093560456..83f138b78f0 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -788,7 +788,7 @@ bool init_slave_skip_errors(const char* arg) if (!arg || !*arg) // No errors defined goto end; - if (unlikely(my_bitmap_init(&slave_error_mask,0,MAX_SLAVE_ERROR,0))) + if (my_bitmap_init(&slave_error_mask,0,MAX_SLAVE_ERROR)) DBUG_RETURN(1); use_slave_mask= 1; @@ -2329,11 +2329,11 @@ past_checksum: /* Announce MariaDB slave capabilities. */ DBUG_EXECUTE_IF("simulate_slave_capability_none", goto after_set_capability;); { - int rc= DBUG_EVALUATE_IF("simulate_slave_capability_old_53", + int rc= DBUG_IF("simulate_slave_capability_old_53") ? mysql_real_query(mysql, STRING_WITH_LEN("SET @mariadb_slave_capability=" - STRINGIFY_ARG(MARIA_SLAVE_CAPABILITY_ANNOTATE))), + STRINGIFY_ARG(MARIA_SLAVE_CAPABILITY_ANNOTATE))) : mysql_real_query(mysql, STRING_WITH_LEN("SET @mariadb_slave_capability=" - STRINGIFY_ARG(MARIA_SLAVE_CAPABILITY_MINE)))); + STRINGIFY_ARG(MARIA_SLAVE_CAPABILITY_MINE))); if (unlikely(rc)) { err_code= mysql_errno(mysql); @@ -3932,6 +3932,10 @@ apply_event_and_update_pos_apply(Log_event* ev, THD* thd, rpl_group_info *rgi, DBUG_PRINT("info", ("apply_event error = %d", exec_res)); if (exec_res == 0) { + if (thd->rgi_slave && (thd->rgi_slave->gtid_ev_flags_extra & + Gtid_log_event::FL_START_ALTER_E1) && + thd->rgi_slave->get_finish_event_group_called()) + DBUG_RETURN(exec_res ? 1 : 0); int error= ev->update_pos(rgi); #ifdef DBUG_TRACE DBUG_PRINT("info", ("update_pos error = %d", error)); @@ -4055,6 +4059,11 @@ int apply_event_and_update_pos_for_parallel(Log_event* ev, THD* thd, rpl_group_info *rgi) { + int rc= 0; + ulong retries= 0; + bool is_sa= rgi->gtid_ev_flags_extra == Gtid_log_event::FL_START_ALTER_E1; + bool is_sa_temp_err= false; + mysql_mutex_assert_not_owner(&rgi->rli->data_lock); int reason= apply_event_and_update_pos_setup(ev, thd, rgi); /* @@ -4066,7 +4075,51 @@ apply_event_and_update_pos_for_parallel(Log_event* ev, THD* thd, Calling sql_delay_event() was handled in the SQL driver thread when doing parallel replication. */ - return apply_event_and_update_pos_apply(ev, thd, rgi, reason); + do + { + rc= apply_event_and_update_pos_apply(ev, thd, rgi, reason); + if (rc && is_sa) + { + is_sa_temp_err= + is_parallel_retry_error(rgi, thd->get_stmt_da()->sql_errno()); + } + } + while(is_sa_temp_err && retries++ < slave_trans_retries); + + if (is_sa_temp_err) + { + Master_info *mi= rgi->rli->mi; + mysql_mutex_lock(&mi->start_alter_lock); + + DBUG_ASSERT(!rgi->sa_info->direct_commit_alter); + /* + Give up retrying to hand the whole ALTER execution over to + the "Complete" ALTER. + */ + rgi->sa_info->direct_commit_alter= true; + rgi->sa_info->state= start_alter_state::COMPLETED; + mysql_cond_broadcast(&rgi->sa_info->start_alter_cond); + mysql_mutex_unlock(&mi->start_alter_lock); + if (global_system_variables.log_warnings > 2) + { + rpl_gtid *gtid= &rgi->current_gtid; + sql_print_information("Start Alter Query '%s' " + "GTID %u-%u-%llu having a temporary error %d code " + "has been unsuccessfully retried %lu times; its " + "parallel optimistic execution now proceeds in " + "legacy mode", + static_cast<Query_log_event*>(ev)->query, + gtid->domain_id, gtid->server_id, gtid->seq_no, + thd->get_stmt_da()->sql_errno(), retries - 1); + } + thd->clear_error(); + thd->reset_killed(); + rgi->killed_for_retry = rpl_group_info::RETRY_KILL_NONE; + + rc= false; + } + + return rc; } @@ -4705,7 +4758,7 @@ pthread_handler_t handle_slave_io(void *arg) } thd->variables.wsrep_on= 0; - if (DBUG_EVALUATE_IF("failed_slave_start", 1, 0) + if (DBUG_IF("failed_slave_start") || repl_semisync_slave.slave_start(mi)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, @@ -4982,7 +5035,7 @@ Stopping slave I/O thread due to out-of-memory error from master"); (!repl_semisync_slave.get_slave_enabled() || (!(mi->semi_ack & SEMI_SYNC_SLAVE_DELAY_SYNC) || (mi->semi_ack & (SEMI_SYNC_NEED_ACK)))) && - (DBUG_EVALUATE_IF("failed_flush_master_info", 1, 0) || + (DBUG_IF("failed_flush_master_info") || flush_master_info(mi, TRUE, TRUE))) { sql_print_error("Failed to flush master info file"); @@ -5611,8 +5664,10 @@ pthread_handler_t handle_slave_sql(void *arg) err: if (mi->using_parallel()) + { rli->parallel.wait_for_done(thd, rli); - /* Gtid_list_log_event::do_apply_event has already reported the GTID until */ + }; + /* Gtid_list_log_event::do_apply_event has already reported the GTID until */ if (rli->stop_for_until && rli->until_condition != Relay_log_info::UNTIL_GTID) { if (global_system_variables.log_warnings > 2) @@ -6327,12 +6382,19 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) Rotate_log_event rev(buf, checksum_alg != BINLOG_CHECKSUM_ALG_OFF ? event_len - BINLOG_CHECKSUM_LEN : event_len, mi->rli.relay_log.description_event_for_queue); - - if (unlikely(mi->gtid_reconnect_event_skip_count) && - unlikely(!mi->gtid_event_seen) && - rev.is_artificial_event() && - (mi->prev_master_id != mi->master_id || - strcmp(rev.new_log_ident, mi->master_log_name) != 0)) + bool master_changed= false; + bool maybe_crashed= false; + // Exclude server start scenario + if ((mi->prev_master_id && mi->master_id) && + (mi->prev_master_id != mi->master_id)) + master_changed= true; + if ((mi->master_log_name[0]!='\0') && + (strcmp(rev.new_log_ident, mi->master_log_name) != 0)) + maybe_crashed= true; + + if (unlikely((mi->gtid_reconnect_event_skip_count && master_changed) || + maybe_crashed) && + unlikely(!mi->gtid_event_seen) && rev.is_artificial_event()) { /* Artificial Rotate_log_event is the first event we receive at the start @@ -6368,26 +6430,37 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) case likewise rollback the partially received event group. */ Format_description_log_event fdle(4); + fdle.checksum_alg= checksum_alg; + + /* + Possible crash is flagged in being created FD' common header + to conduct any necessary cleanup by the slave applier. + */ + if (maybe_crashed) + fdle.flags |= LOG_EVENT_BINLOG_IN_USE_F; - if (mi->prev_master_id != mi->master_id) - sql_print_warning("The server_id of master server changed in the " - "middle of GTID %u-%u-%llu. Assuming a change of " - "master server, so rolling back the previously " - "received partial transaction. Expected: %lu, " - "received: %lu", mi->last_queued_gtid.domain_id, - mi->last_queued_gtid.server_id, - mi->last_queued_gtid.seq_no, - mi->prev_master_id, mi->master_id); - else if (strcmp(rev.new_log_ident, mi->master_log_name) != 0) - sql_print_warning("Unexpected change of master binlog file name in the " - "middle of GTID %u-%u-%llu, assuming that master has " - "crashed and rolling back the transaction. Expected: " - "'%s', received: '%s'", - mi->last_queued_gtid.domain_id, - mi->last_queued_gtid.server_id, - mi->last_queued_gtid.seq_no, - mi->master_log_name, rev.new_log_ident); + if (mi->gtid_reconnect_event_skip_count) + { + if (master_changed) + sql_print_warning("The server_id of master server changed in the " + "middle of GTID %u-%u-%llu. Assuming a change of " + "master server, so rolling back the previously " + "received partial transaction. Expected: %lu, " + "received: %lu", mi->last_queued_gtid.domain_id, + mi->last_queued_gtid.server_id, + mi->last_queued_gtid.seq_no, + mi->prev_master_id, mi->master_id); + else + sql_print_warning("Unexpected change of master binlog file name in " + "the middle of GTID %u-%u-%llu, assuming that " + "master has crashed and rolling back the " + "transaction. Expected: '%s', received: '%s'", + mi->last_queued_gtid.domain_id, + mi->last_queued_gtid.server_id, + mi->last_queued_gtid.seq_no, mi->master_log_name, + rev.new_log_ident); + } mysql_mutex_lock(log_lock); if (likely(!rli->relay_log.write_event(&fdle) && !rli->relay_log.flush_and_sync(NULL))) diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 6d982aac937..d23b5e88baa 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -412,6 +412,26 @@ Item *THD::sp_fix_func_item(Item **it_addr) /** + Prepare an Item for evaluation as an assignment source, + for assignment to the given target. + + @param to - the assignment target + @param it_addr - a pointer on item refernce + + @retval - NULL on error + @retval - a prepared item pointer on success +*/ +Item *THD::sp_fix_func_item_for_assignment(const Field *to, Item **it_addr) +{ + DBUG_ENTER("THD::sp_fix_func_item_for_assignment"); + Item *res= sp_fix_func_item(it_addr); + if (res && (!res->check_assignability_to(to, false))) + DBUG_RETURN(res); + DBUG_RETURN(NULL); +} + + +/** Evaluate an expression and store the result in the field. @param result_field the field to store the result @@ -2080,7 +2100,8 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, /* Arguments must be fixed in Item_func_sp::fix_fields */ DBUG_ASSERT(argp[arg_no]->fixed()); - if ((err_status= (*func_ctx)->set_parameter(thd, arg_no, &(argp[arg_no])))) + err_status= bind_input_param(thd, argp[arg_no], arg_no, *func_ctx, TRUE); + if (err_status) goto err_with_cleanup; } @@ -2203,6 +2224,19 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, my_error(ER_SP_NORETURNEND, MYF(0), m_name.str); err_status= TRUE; } + else + { + /* + Copy back all OUT or INOUT values to the previous frame, or + set global user variables + */ + for (arg_no= 0; arg_no < argcount; arg_no++) + { + err_status= bind_output_param(thd, argp[arg_no], arg_no, octx, *func_ctx); + if (err_status) + break; + } + } } #ifndef NO_EMBEDDED_ACCESS_CHECKS @@ -2325,50 +2359,9 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) if (!arg_item) break; - sp_variable *spvar= m_pcont->find_variable(i); - - if (!spvar) - continue; - - if (spvar->mode != sp_variable::MODE_IN) - { - Settable_routine_parameter *srp= - arg_item->get_settable_routine_parameter(); - - if (!srp) - { - my_error(ER_SP_NOT_VAR_ARG, MYF(0), i+1, ErrConvDQName(this).ptr()); - err_status= TRUE; - break; - } - - srp->set_required_privilege(spvar->mode == sp_variable::MODE_INOUT); - } - - if (spvar->mode == sp_variable::MODE_OUT) - { - Item_null *null_item= new (thd->mem_root) Item_null(thd); - Item *tmp_item= null_item; - - if (!null_item || - nctx->set_parameter(thd, i, &tmp_item)) - { - DBUG_PRINT("error", ("set variable failed")); - err_status= TRUE; - break; - } - } - else - { - if (nctx->set_parameter(thd, i, it_args.ref())) - { - DBUG_PRINT("error", ("set variable 2 failed")); - err_status= TRUE; - break; - } - } - - TRANSACT_TRACKER(add_trx_state_from_thd(thd)); + err_status= bind_input_param(thd, arg_item, i, nctx, FALSE); + if (err_status) + break; } /* @@ -2478,31 +2471,9 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) if (!arg_item) break; - sp_variable *spvar= m_pcont->find_variable(i); - - if (spvar->mode == sp_variable::MODE_IN) - continue; - - Settable_routine_parameter *srp= - arg_item->get_settable_routine_parameter(); - - DBUG_ASSERT(srp); - - if (srp->set_value(thd, octx, nctx->get_variable_addr(i))) - { - DBUG_PRINT("error", ("set value failed")); - err_status= TRUE; + err_status= bind_output_param(thd, arg_item, i, octx, nctx); + if (err_status) break; - } - - Send_field *out_param_info= new (thd->mem_root) Send_field(thd, nctx->get_parameter(i)); - out_param_info->db_name= m_db; - out_param_info->table_name= m_name; - out_param_info->org_table_name= m_name; - out_param_info->col_name= spvar->name; - out_param_info->org_col_name= spvar->name; - - srp->set_out_param_info(out_param_info); } } @@ -2533,6 +2504,112 @@ sp_head::execute_procedure(THD *thd, List<Item> *args) DBUG_RETURN(err_status); } +bool +sp_head::bind_input_param(THD *thd, + Item *arg_item, + uint arg_no, + sp_rcontext *nctx, + bool is_function) +{ + DBUG_ENTER("sp_head::bind_input_param"); + + sp_variable *spvar= m_pcont->find_variable(arg_no); + if (!spvar) + DBUG_RETURN(FALSE); + + if (spvar->mode != sp_variable::MODE_IN) + { + Settable_routine_parameter *srp= + arg_item->get_settable_routine_parameter(); + + if (!srp) + { + my_error(ER_SP_NOT_VAR_ARG, MYF(0), arg_no+1, ErrConvDQName(this).ptr()); + DBUG_RETURN(TRUE); + } + + if (is_function) + { + /* + Check if the function is called from SELECT/INSERT/UPDATE/DELETE query + and parameter is OUT or INOUT. + If yes, it is an invalid call - throw error. + */ + if (thd->lex->sql_command == SQLCOM_SELECT || + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_DELETE) + { + my_error(ER_SF_OUT_INOUT_ARG_NOT_ALLOWED, MYF(0), arg_no+1, m_name.str); + DBUG_RETURN(TRUE); + } + } + + srp->set_required_privilege(spvar->mode == sp_variable::MODE_INOUT); + } + + if (spvar->mode == sp_variable::MODE_OUT) + { + Item_null *null_item= new (thd->mem_root) Item_null(thd); + Item *tmp_item= null_item; + + if (!null_item || + nctx->set_parameter(thd, arg_no, &tmp_item)) + { + DBUG_PRINT("error", ("set variable failed")); + DBUG_RETURN(TRUE); + } + } + else + { + if (nctx->set_parameter(thd, arg_no, &arg_item)) + { + DBUG_PRINT("error", ("set variable 2 failed")); + DBUG_RETURN(TRUE); + } + } + + TRANSACT_TRACKER(add_trx_state_from_thd(thd)); + + DBUG_RETURN(FALSE); +} + +bool +sp_head::bind_output_param(THD *thd, + Item *arg_item, + uint arg_no, + sp_rcontext *octx, + sp_rcontext *nctx) +{ + DBUG_ENTER("sp_head::bind_output_param"); + + sp_variable *spvar= m_pcont->find_variable(arg_no); + if (spvar->mode == sp_variable::MODE_IN) + DBUG_RETURN(FALSE); + + Settable_routine_parameter *srp= + arg_item->get_settable_routine_parameter(); + + DBUG_ASSERT(srp); + + if (srp->set_value(thd, octx, nctx->get_variable_addr(arg_no))) + { + DBUG_PRINT("error", ("set value failed")); + DBUG_RETURN(TRUE); + } + + Send_field *out_param_info= new (thd->mem_root) Send_field(thd, nctx->get_parameter(arg_no)); + out_param_info->db_name= m_db; + out_param_info->table_name= m_name; + out_param_info->org_table_name= m_name; + out_param_info->col_name= spvar->name; + out_param_info->org_col_name= spvar->name; + + srp->set_out_param_info(out_param_info); + + DBUG_RETURN(FALSE); +} /** Reset lex during parsing, before we parse a sub statement. @@ -4072,7 +4149,7 @@ sp_instr_jump_if_not::exec_core(THD *thd, uint *nextp) Item *it; int res; - it= thd->sp_prepare_func_item(&m_expr); + it= thd->sp_prepare_func_item(&m_expr, 1); if (! it) { res= -1; diff --git a/sql/sp_head.h b/sql/sp_head.h index 57def5baa83..5b4aa36e518 100644 --- a/sql/sp_head.h +++ b/sql/sp_head.h @@ -504,6 +504,18 @@ private: sp_assignment_lex *param_lex, Item_args *parameters); + bool bind_input_param(THD *thd, + Item *arg_item, + uint arg_no, + sp_rcontext *nctx, + bool is_function); + + bool bind_output_param(THD *thd, + Item *arg_item, + uint arg_no, + sp_rcontext *octx, + sp_rcontext *nctx); + public: /** Generate a code for an "OPEN cursor" statement. @@ -560,7 +572,7 @@ public: { return m_flags & MODIFIES_DATA; } inline uint instructions() - { return m_instr.elements; } + { return (uint)m_instr.elements; } inline sp_instr * last_instruction() diff --git a/sql/sp_rcontext.cc b/sql/sp_rcontext.cc index c4c19dd39f6..d2fe53a2431 100644 --- a/sql/sp_rcontext.cc +++ b/sql/sp_rcontext.cc @@ -518,7 +518,8 @@ bool sp_rcontext::handle_sql_condition(THD *thd, found_condition= new (callers_arena->mem_root) Sql_condition(callers_arena->mem_root, da->get_error_condition_identity(), - da->message()); + da->message(), + da->current_row_for_warning()); } } else if (da->current_statement_warn_count()) @@ -714,7 +715,7 @@ Item_cache *sp_rcontext::create_case_expr_holder(THD *thd, bool sp_rcontext::set_case_expr(THD *thd, int case_expr_id, Item **case_expr_item_ptr) { - Item *case_expr_item= thd->sp_prepare_func_item(case_expr_item_ptr); + Item *case_expr_item= thd->sp_prepare_func_item(case_expr_item_ptr, 1); if (!case_expr_item) return true; diff --git a/sql/sp_rcontext.h b/sql/sp_rcontext.h index 0e0e8921f86..ea669b2d1d8 100644 --- a/sql/sp_rcontext.h +++ b/sql/sp_rcontext.h @@ -111,15 +111,18 @@ public: /// Text message. char *message; + /** Row number where the condition has happened */ + ulong m_row_number; + /// The constructor. /// /// @param _sql_condition The SQL condition. /// @param arena Query arena for SP - Sql_condition_info(const Sql_condition *_sql_condition, - Query_arena *arena) + Sql_condition_info(const Sql_condition *_sql_condition, Query_arena *arena) :Sql_condition_identity(*_sql_condition) { message= strdup_root(arena->mem_root, _sql_condition->get_message_text()); + m_row_number= _sql_condition->m_row_number; } }; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 7e542c55a53..283c629f42a 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -2224,14 +2224,14 @@ static bool has_validation_plugins() MariaDB_PASSWORD_VALIDATION_PLUGIN, NULL); } -struct validation_data { const LEX_CSTRING *user, *password; }; +struct validation_data { const LEX_CSTRING *user, *password, *host; }; static my_bool do_validate(THD *, plugin_ref plugin, void *arg) { struct validation_data *data= (struct validation_data *)arg; struct st_mariadb_password_validation *handler= (st_mariadb_password_validation *)plugin_decl(plugin)->info; - if (handler->validate_password(data->user, data->password)) + if (handler->validate_password(data->user, data->password, data->host)) { my_error(ER_NOT_VALID_PASSWORD, MYF(0), plugin_ref_to_int(plugin)->name.str); return true; @@ -2241,12 +2241,14 @@ static my_bool do_validate(THD *, plugin_ref plugin, void *arg) static bool validate_password(THD *thd, const LEX_CSTRING &user, + const LEX_CSTRING &host, const LEX_CSTRING &pwtext, bool has_hash) { if (pwtext.length || !has_hash) { struct validation_data data= { &user, - pwtext.str ? &pwtext : &empty_clex_str }; + pwtext.str ? &pwtext : &empty_clex_str, + &host }; if (plugin_foreach(NULL, do_validate, MariaDB_PASSWORD_VALIDATION_PLUGIN, &data)) { @@ -2299,6 +2301,7 @@ static int set_user_salt(ACL_USER::AUTH *auth, plugin_ref plugin) not loaded, if the auth_string is invalid, if the password is not applicable */ static int set_user_auth(THD *thd, const LEX_CSTRING &user, + const LEX_CSTRING &host, ACL_USER::AUTH *auth, const LEX_CSTRING &pwtext) { const char *plugin_name= auth->plugin.str; @@ -2330,7 +2333,7 @@ static int set_user_auth(THD *thd, const LEX_CSTRING &user, } if (info->hash_password && - validate_password(thd, user, pwtext, auth->auth_string.length)) + validate_password(thd, user, host, pwtext, auth->auth_string.length)) { res= ER_NOT_VALID_PASSWORD; goto end; @@ -3442,7 +3445,9 @@ static int acl_user_update(THD *thd, ACL_USER *acl_user, uint nauth, auth->auth_str); if (fix_user_plugin_ptr(work_copy + i)) work_copy[i].plugin= safe_lexcstrdup_root(&acl_memroot, auth->plugin); - if (set_user_auth(thd, acl_user->user, work_copy + i, auth->pwtext)) + if (set_user_auth(thd, acl_user->user, + {acl_user->host.hostname, acl_user->hostname_length}, + work_copy + i, auth->pwtext)) return 1; } } @@ -3710,14 +3715,14 @@ static void init_check_host(void) (my_hash_get_key) check_get_key, 0, 0); if (!allow_all_hosts) { - for (uint i=0 ; i < acl_users.elements ; i++) + for (size_t i=0 ; i < acl_users.elements ; i++) { ACL_USER *acl_user=dynamic_element(&acl_users,i,ACL_USER*); if (strchr(acl_user->host.hostname,wild_many) || strchr(acl_user->host.hostname,wild_one) || acl_user->host.ip_mask) { // Has wildcard - uint j; + size_t j; for (j=0 ; j < acl_wild_hosts.elements ; j++) { // Check if host already exists acl_host_and_ip *acl=dynamic_element(&acl_wild_hosts,j, @@ -3836,7 +3841,7 @@ static bool add_role_user_mapping(const char *uname, const char *hname, static void remove_ptr_from_dynarray(DYNAMIC_ARRAY *array, void *ptr) { bool found __attribute__((unused))= false; - for (uint i= 0; i < array->elements; i++) + for (size_t i= 0; i < array->elements; i++) { if (ptr == *dynamic_element(array, i, void**)) { @@ -3885,7 +3890,7 @@ static void rebuild_role_grants(void) /* Reset every user's and role's role_grants array */ - for (uint i=0; i < acl_users.elements; i++) { + for (size_t i=0; i < acl_users.elements; i++) { ACL_USER *user= dynamic_element(&acl_users, i, ACL_USER *); reset_dynamic(&user->role_grants); } @@ -3911,7 +3916,7 @@ bool acl_check_host(const char *host, const char *ip) mysql_mutex_unlock(&acl_cache->lock); return 0; // Found host } - for (uint i=0 ; i < acl_wild_hosts.elements ; i++) + for (size_t i=0 ; i < acl_wild_hosts.elements ; i++) { acl_host_and_ip *acl=dynamic_element(&acl_wild_hosts,i,acl_host_and_ip*); if (compare_hostname(acl, host, ip)) @@ -4068,7 +4073,8 @@ bool change_password(THD *thd, LEX_USER *user) { auth= acl_user->auth[i]; auth.auth_string= safe_lexcstrdup_root(&acl_memroot, user->auth->auth_str); - int r= set_user_auth(thd, user->user, &auth, user->auth->pwtext); + int r= set_user_auth(thd, user->user, user->host, + &auth, user->auth->pwtext); if (r == ER_SET_PASSWORD_AUTH_PLUGIN) password_plugin= auth.plugin.str; else if (r) @@ -4361,14 +4367,18 @@ static ACL_USER * find_user_wild(const char *host, const char *user, const char */ static ACL_ROLE *find_acl_role(const char *role) { + size_t length= strlen(role); DBUG_ENTER("find_acl_role"); DBUG_PRINT("enter",("role: '%s'", role)); DBUG_PRINT("info", ("Hash elements: %ld", acl_roles.records)); mysql_mutex_assert_owner(&acl_cache->lock); + if (!length) + DBUG_RETURN(NULL); + ACL_ROLE *r= (ACL_ROLE *)my_hash_search(&acl_roles, (uchar *)role, - strlen(role)); + length); DBUG_RETURN(r); } @@ -5120,7 +5130,7 @@ acl_update_proxy_user(ACL_PROXY_USER *new_value, bool is_revoke) mysql_mutex_assert_owner(&acl_cache->lock); DBUG_ENTER("acl_update_proxy_user"); - for (uint i= 0; i < acl_proxy_users.elements; i++) + for (size_t i= 0; i < acl_proxy_users.elements; i++) { ACL_PROXY_USER *acl_user= dynamic_element(&acl_proxy_users, i, ACL_PROXY_USER *); @@ -6401,7 +6411,7 @@ static int traverse_role_graph_impl(ACL_USER_BASE *user, void *context, end: /* Cleanup */ - for (uint i= 0; i < to_clear.elements(); i++) + for (size_t i= 0; i < to_clear.elements(); i++) { ACL_USER_BASE *current= to_clear.at(i); DBUG_ASSERT(current->flags & (ROLE_EXPLORED | ROLE_ON_STACK | ROLE_OPENED)); @@ -6469,7 +6479,7 @@ static bool merge_role_global_privileges(ACL_ROLE *grantee) DBUG_EXECUTE_IF("role_merge_stats", role_global_merges++;); - for (uint i= 0; i < grantee->role_grants.elements; i++) + for (size_t i= 0; i < grantee->role_grants.elements; i++) { ACL_ROLE *r= *dynamic_element(&grantee->role_grants, i, ACL_ROLE**); grantee->access|= r->access; @@ -6608,8 +6618,8 @@ static bool merge_role_db_privileges(ACL_ROLE *grantee, const char *dbname, if (update_flags & 4) { // Remove elements marked for deletion. - uint count= 0; - for(uint i= 0; i < acl_dbs.elements(); i++) + size_t count= 0; + for(size_t i= 0; i < acl_dbs.elements(); i++) { ACL_DB *acl_db= &acl_dbs.at(i); if (acl_db->sort) @@ -6973,7 +6983,7 @@ static int merge_role_privileges(ACL_ROLE *role __attribute__((unused)), if (data->what != PRIVS_TO_MERGE::GLOBAL) { role_hash.insert(grantee); - for (uint i= 0; i < grantee->role_grants.elements; i++) + for (size_t i= 0; i < grantee->role_grants.elements; i++) role_hash.insert(*dynamic_element(&grantee->role_grants, i, ACL_ROLE**)); } @@ -9558,7 +9568,7 @@ static bool show_role_grants(THD *thd, const char *hostname, ACL_USER_BASE *acl_entry, char *buff, size_t buffsize) { - uint counter; + size_t counter; Protocol *protocol= thd->protocol; LEX_CSTRING host= {const_cast<char*>(hostname), strlen(hostname)}; @@ -9671,7 +9681,7 @@ static bool show_database_privileges(THD *thd, const char *username, privilege_t want_access(NO_ACL); Protocol *protocol= thd->protocol; - for (uint i=0 ; i < acl_dbs.elements() ; i++) + for (size_t i=0 ; i < acl_dbs.elements() ; i++) { const char *user, *host; @@ -10359,14 +10369,14 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, if (drop) { // delete the role from cross-reference arrays - for (uint i=0; i < acl_role->role_grants.elements; i++) + for (size_t i=0; i < acl_role->role_grants.elements; i++) { ACL_ROLE *grant= *dynamic_element(&acl_role->role_grants, i, ACL_ROLE**); remove_ptr_from_dynarray(&grant->parent_grantee, acl_role); } - for (uint i=0; i < acl_role->parent_grantee.elements; i++) + for (size_t i=0; i < acl_role->parent_grantee.elements; i++) { ACL_USER_BASE *grantee= *dynamic_element(&acl_role->parent_grantee, i, ACL_USER_BASE**); @@ -10393,7 +10403,7 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, /* Get the number of elements in the in-memory structure. */ switch (struct_no) { case USER_ACL: - elements= acl_users.elements; + elements= int(acl_users.elements); break; case DB_ACL: elements= int(acl_dbs.elements()); @@ -10419,7 +10429,7 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, elements= grant_name_hash->records; break; case PROXY_USERS_ACL: - elements= acl_proxy_users.elements; + elements= int(acl_proxy_users.elements); break; case ROLES_MAPPINGS_HASH: roles_mappings_hash= &acl_roles_mappings; @@ -12277,11 +12287,11 @@ SHOW_VAR acl_statistics[] = { {"procedure_grants", (char*)&proc_priv_hash.records, SHOW_ULONG}, {"package_spec_grants", (char*)&package_spec_priv_hash.records, SHOW_ULONG}, {"package_body_grants", (char*)&package_body_priv_hash.records, SHOW_ULONG}, - {"proxy_users", (char*)&acl_proxy_users.elements, SHOW_UINT}, + {"proxy_users", (char*)&acl_proxy_users.elements, SHOW_SIZE_T}, {"role_grants", (char*)&acl_roles_mappings.records, SHOW_ULONG}, {"roles", (char*)&acl_roles.records, SHOW_ULONG}, {"table_grants", (char*)&column_priv_hash.records, SHOW_ULONG}, - {"users", (char*)&acl_users.elements, SHOW_UINT}, + {"users", (char*)&acl_users.elements, SHOW_SIZE_T}, #endif {NullS, NullS, SHOW_LONG}, }; diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 00d7e5efecd..465145cf25f 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -659,8 +659,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, protocol->store(operator_name, system_charset_info); protocol->store(&error_clex_str, system_charset_info); length= my_snprintf(buff, sizeof(buff), - ER_THD(thd, ER_DROP_PARTITION_NON_EXISTENT), - table_name.str); + ER_THD(thd, ER_PARTITION_DOES_NOT_EXIST)); protocol->store(buff, length, system_charset_info); if(protocol->write()) goto err; @@ -993,6 +992,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, else compl_result_code= HA_ADMIN_FAILED; + if (table->table) + free_statistics_for_table(thd, table->table); if (compl_result_code) result_code= HA_ADMIN_FAILED; else diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index bcbf86f1e8f..2a7f885734e 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -19,6 +19,9 @@ #include "sql_table.h" // mysql_alter_table, // mysql_exchange_partition #include "sql_alter.h" +#include "rpl_mi.h" +#include "slave.h" +#include "debug_sync.h" #include "wsrep_mysqld.h" Alter_info::Alter_info(const Alter_info &rhs, MEM_ROOT *mem_root) @@ -427,6 +430,8 @@ bool Sql_cmd_alter_table::execute(THD *thd) as for RENAME TO, as being done by SQLCOM_RENAME_TABLE */ if ((alter_info.partition_flags & ALTER_PARTITION_DROP) || + (alter_info.partition_flags & ALTER_PARTITION_CONVERT_IN) || + (alter_info.partition_flags & ALTER_PARTITION_CONVERT_OUT) || (alter_info.flags & ALTER_RENAME)) priv_needed|= DROP_ACL; diff --git a/sql/sql_alter.h b/sql/sql_alter.h index d0ac4ab27f5..99e717d50b2 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -250,13 +250,15 @@ public: const LEX_CSTRING *new_db_arg, const LEX_CSTRING *new_name_arg); /** - @return true if the table is moved to another database, false otherwise. + @return true if the table is moved to another database or a new table + created by ALTER_PARTITION_CONVERT_OUT, false otherwise. */ bool is_database_changed() const { return (new_db.str != db.str); }; /** - @return true if the table is renamed, false otherwise. + @return true if the table is renamed or a new table created by + ALTER_PARTITION_CONVERT_OUT, false otherwise. */ bool is_table_renamed() const { return (is_database_changed() || new_name.str != table_name.str); }; diff --git a/sql/sql_array.h b/sql/sql_array.h index 8610e971016..85a53ae1a6f 100644 --- a/sql/sql_array.h +++ b/sql/sql_array.h @@ -114,19 +114,19 @@ template <class Elem> class Dynamic_array { DYNAMIC_ARRAY array; public: - Dynamic_array(PSI_memory_key psi_key, uint prealloc=16, uint increment=16) + Dynamic_array(PSI_memory_key psi_key, size_t prealloc=16, size_t increment=16) { init(psi_key, prealloc, increment); } - Dynamic_array(MEM_ROOT *root, uint prealloc=16, uint increment=16) + Dynamic_array(MEM_ROOT *root, size_t prealloc=16, size_t increment=16) { void *init_buffer= alloc_root(root, sizeof(Elem) * prealloc); - init_dynamic_array2(root->m_psi_key, &array, sizeof(Elem), init_buffer, - prealloc, increment, MYF(0)); + init_dynamic_array2(root->psi_key, &array, sizeof(Elem), init_buffer, + prealloc, increment, MYF(0)); } - void init(PSI_memory_key psi_key, uint prealloc=16, uint increment=16) + void init(PSI_memory_key psi_key, size_t prealloc=16, size_t increment=16) { init_dynamic_array2(psi_key, &array, sizeof(Elem), 0, prealloc, increment, MYF(0)); } @@ -217,7 +217,7 @@ public: void del(size_t idx) { DBUG_ASSERT(idx <= array.max_element); - delete_dynamic_element(&array, (uint)idx); + delete_dynamic_element(&array, idx); } size_t elements() const @@ -228,7 +228,7 @@ public: void elements(size_t num_elements) { DBUG_ASSERT(num_elements <= array.max_element); - array.elements= (uint)num_elements; + array.elements= num_elements; } void clear() @@ -236,7 +236,7 @@ public: elements(0); } - void set(uint idx, const Elem &el) + void set(size_t idx, const Elem &el) { set_dynamic(&array, &el, idx); } @@ -248,7 +248,7 @@ public: bool reserve(size_t new_size) { - return allocate_dynamic(&array, (uint)new_size); + return allocate_dynamic(&array, new_size); } @@ -260,7 +260,7 @@ public: if (new_size > old_size) { - set_dynamic(&array, (uchar*)&default_val, (uint)(new_size - 1)); + set_dynamic(&array, (uchar*)&default_val, new_size - 1); /*for (size_t i= old_size; i != new_size; i++) { at(i)= default_val; diff --git a/sql/sql_audit.cc b/sql/sql_audit.cc index 6ee6ede31b8..c9c59c1b849 100644 --- a/sql/sql_audit.cc +++ b/sql/sql_audit.cc @@ -348,14 +348,11 @@ static my_bool calc_class_mask(THD *thd, plugin_ref plugin, void *arg) */ int finalize_audit_plugin(st_plugin_int *plugin) { + int deinit_status= 0; unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE]; - if (plugin->plugin->deinit && plugin->plugin->deinit(NULL)) - { - DBUG_PRINT("warning", ("Plugin '%s' deinit function returned error.", - plugin->name.str)); - DBUG_EXECUTE("finalize_audit_plugin", return 1; ); - } + if (plugin->plugin->deinit) + deinit_status= plugin->plugin->deinit(NULL); plugin->data= NULL; bzero(&event_class_mask, sizeof(event_class_mask)); @@ -374,7 +371,7 @@ int finalize_audit_plugin(st_plugin_int *plugin) bmove(mysql_global_audit_mask, event_class_mask, sizeof(event_class_mask)); mysql_mutex_unlock(&LOCK_audit_mask); - return 0; + return deinit_status; } diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 30a8d2c2d6f..3a5a90966b8 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -6040,7 +6040,7 @@ find_field_in_table(THD *thd, TABLE *table, const char *name, size_t length, if (field) { if (field->invisible == INVISIBLE_FULL && - DBUG_EVALUATE_IF("test_completely_invisible", 0, 1)) + !DBUG_IF("test_completely_invisible")) DBUG_RETURN((Field*)0); if (field->invisible == INVISIBLE_SYSTEM && @@ -7455,7 +7455,7 @@ store_top_level_join_columns(THD *thd, TABLE_LIST *table_ref, /* Add a TRUE condition to outer joins that have no common columns. */ if (table_ref_2->outer_join && !table_ref_1->on_expr && !table_ref_2->on_expr) - table_ref_2->on_expr= (Item*) &Item_true; + table_ref_2->on_expr= (Item*) Item_true; /* Change this table reference to become a leaf for name resolution. */ if (left_neighbor) @@ -9030,7 +9030,7 @@ fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, Field **ptr, my_bool mysql_rm_tmp_tables(void) { - uint i, idx; + size_t i, idx; char path[FN_REFLEN], *tmpdir, path_copy[FN_REFLEN]; MY_DIR *dirp; FILEINFO *file; @@ -9052,7 +9052,7 @@ my_bool mysql_rm_tmp_tables(void) /* Remove all SQLxxx tables from directory */ - for (idx=0 ; idx < (uint) dirp->number_of_files ; idx++) + for (idx=0 ; idx < dirp->number_of_files ; idx++) { file=dirp->dir_entry+idx; diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 9f61135232f..e71c7015238 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -1,5 +1,6 @@ /* Copyright (c) 2005, 2013, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -20,6 +21,7 @@ #include "sql_parse.h" #include "sql_acl.h" #include "rpl_rli.h" +#include "rpl_mi.h" #include "slave.h" #include "log_event.h" @@ -69,7 +71,8 @@ static int check_event_type(int type, Relay_log_info *rli) /* It is always allowed to execute FD events. */ return 0; - + + case QUERY_EVENT: case TABLE_MAP_EVENT: case WRITE_ROWS_EVENT_V1: case UPDATE_ROWS_EVENT_V1: @@ -166,6 +169,57 @@ int binlog_defragment(THD *thd) return 0; } +/** + Wraps Log_event::apply_event to save and restore + session context in case of Query_log_event. + + @param ev replication event + @param rgi execution context for the event + + @return + 0 on success, + non-zero otherwise. +*/ +#if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) +int save_restore_context_apply_event(Log_event *ev, rpl_group_info *rgi) +{ + if (ev->get_type_code() != QUERY_EVENT) + return ev->apply_event(rgi); + + THD *thd= rgi->thd; + Relay_log_info *rli= thd->rli_fake; + DBUG_ASSERT(!rli->mi); + LEX_CSTRING connection_name= { STRING_WITH_LEN("BINLOG_BASE64_EVENT") }; + + if (!(rli->mi= new Master_info(&connection_name, false))) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return -1; + } + + sql_digest_state *m_digest= thd->m_digest; + PSI_statement_locker *m_statement_psi= thd->m_statement_psi;; + LEX_CSTRING save_db= thd->db; + my_thread_id m_thread_id= thd->variables.pseudo_thread_id; + + thd->system_thread_info.rpl_sql_info= NULL; + thd->reset_db(&null_clex_str); + + thd->m_digest= NULL; + thd->m_statement_psi= NULL; + + int err= ev->apply_event(rgi); + + thd->m_digest= m_digest; + thd->m_statement_psi= m_statement_psi; + thd->variables.pseudo_thread_id= m_thread_id; + thd->reset_db(&save_db); + delete rli->mi; + rli->mi= NULL; + + return err; +} +#endif /** Execute a BINLOG statement. @@ -215,11 +269,9 @@ void mysql_client_binlog_statement(THD* thd) if (!(rgi= thd->rgi_fake)) rgi= thd->rgi_fake= new rpl_group_info(rli); rgi->thd= thd; - const char *error= 0; Log_event *ev = 0; my_bool is_fragmented= FALSE; - /* Out of memory check */ @@ -372,7 +424,7 @@ void mysql_client_binlog_statement(THD* thd) LEX *backup_lex; thd->backup_and_reset_current_lex(&backup_lex); - err= ev->apply_event(rgi); + err= save_restore_context_apply_event(ev, rgi); thd->restore_current_lex(backup_lex); } thd->variables.option_bits= @@ -388,7 +440,7 @@ void mysql_client_binlog_statement(THD* thd) i.e. when this thread terminates. */ if (ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) - delete ev; + delete ev; ev= 0; if (err) { @@ -396,7 +448,8 @@ void mysql_client_binlog_statement(THD* thd) TODO: Maybe a better error message since the BINLOG statement now contains several events. */ - my_error(ER_UNKNOWN_ERROR, MYF(0)); + if (!thd->is_error()) + my_error(ER_UNKNOWN_ERROR, MYF(0)); goto end; } } @@ -412,5 +465,7 @@ end: thd->variables.option_bits= thd_options; rgi->slave_close_thread_tables(thd); my_free(buf); + delete rgi; + rgi= thd->rgi_fake= NULL; DBUG_VOID_RETURN; } diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 56d0d1682cb..3858cbfca45 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -294,7 +294,7 @@ bool Foreign_key::validate(List<Create_field> &table_fields) &sql_field->field_name)) {} if (!sql_field) { - my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str); + my_error(ER_KEY_COLUMN_DOES_NOT_EXIST, MYF(0), column->field_name.str); DBUG_RETURN(TRUE); } if (type == Key::FOREIGN_KEY && sql_field->vcol_info) @@ -752,8 +752,7 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) will be re-initialized in init_for_queries(). */ init_sql_alloc(key_memory_thd_main_mem_root, - &main_mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0, - MYF(MY_THREAD_SPECIFIC)); + &main_mem_root, 64, 0, MYF(MY_THREAD_SPECIFIC)); /* Allocation of user variables for binary logging is always done with main @@ -965,10 +964,8 @@ Internal_error_handler *THD::pop_internal_handler() void THD::raise_error(uint sql_errno) { const char* msg= ER_THD(this, sql_errno); - (void) raise_condition(sql_errno, - NULL, - Sql_condition::WARN_LEVEL_ERROR, - msg); + (void) raise_condition(sql_errno, "\0\0\0\0\0", + Sql_condition::WARN_LEVEL_ERROR, msg); } void THD::raise_error_printf(uint sql_errno, ...) @@ -981,20 +978,16 @@ void THD::raise_error_printf(uint sql_errno, ...) va_start(args, sql_errno); my_vsnprintf(ebuff, sizeof(ebuff), format, args); va_end(args); - (void) raise_condition(sql_errno, - NULL, - Sql_condition::WARN_LEVEL_ERROR, - ebuff); + (void) raise_condition(sql_errno, "\0\0\0\0\0", + Sql_condition::WARN_LEVEL_ERROR, ebuff); DBUG_VOID_RETURN; } void THD::raise_warning(uint sql_errno) { const char* msg= ER_THD(this, sql_errno); - (void) raise_condition(sql_errno, - NULL, - Sql_condition::WARN_LEVEL_WARN, - msg); + (void) raise_condition(sql_errno, "\0\0\0\0\0", + Sql_condition::WARN_LEVEL_WARN, msg); } void THD::raise_warning_printf(uint sql_errno, ...) @@ -1007,10 +1000,8 @@ void THD::raise_warning_printf(uint sql_errno, ...) va_start(args, sql_errno); my_vsnprintf(ebuff, sizeof(ebuff), format, args); va_end(args); - (void) raise_condition(sql_errno, - NULL, - Sql_condition::WARN_LEVEL_WARN, - ebuff); + (void) raise_condition(sql_errno, "\0\0\0\0\0", + Sql_condition::WARN_LEVEL_WARN, ebuff); DBUG_VOID_RETURN; } @@ -1021,10 +1012,8 @@ void THD::raise_note(uint sql_errno) if (!(variables.option_bits & OPTION_SQL_NOTES)) DBUG_VOID_RETURN; const char* msg= ER_THD(this, sql_errno); - (void) raise_condition(sql_errno, - NULL, - Sql_condition::WARN_LEVEL_NOTE, - msg); + (void) raise_condition(sql_errno, "\0\0\0\0\0", + Sql_condition::WARN_LEVEL_NOTE, msg); DBUG_VOID_RETURN; } @@ -1040,21 +1029,20 @@ void THD::raise_note_printf(uint sql_errno, ...) va_start(args, sql_errno); my_vsnprintf(ebuff, sizeof(ebuff), format, args); va_end(args); - (void) raise_condition(sql_errno, - NULL, - Sql_condition::WARN_LEVEL_NOTE, - ebuff); + (void) raise_condition(sql_errno, "\0\0\0\0\0", + Sql_condition::WARN_LEVEL_NOTE, ebuff); DBUG_VOID_RETURN; } -Sql_condition* THD::raise_condition(uint sql_errno, - const char* sqlstate, - Sql_condition::enum_warning_level level, - const Sql_user_condition_identity &ucid, - const char* msg) +Sql_condition* THD::raise_condition(const Sql_condition *cond) { + uint sql_errno= cond->get_sql_errno(); + const char *sqlstate= cond->get_sqlstate(); + Sql_condition::enum_warning_level level= cond->get_level(); + const char *msg= cond->get_message_text(); + Diagnostics_area *da= get_stmt_da(); - Sql_condition *cond= NULL; + Sql_condition *raised= NULL; DBUG_ENTER("THD::raise_condition"); DBUG_ASSERT(level < Sql_condition::WARN_LEVEL_END); @@ -1082,22 +1070,18 @@ Sql_condition* THD::raise_condition(uint sql_errno, sql_errno= ER_UNKNOWN_ERROR; if (msg == NULL) msg= ER_THD(this, sql_errno); - if (sqlstate == NULL) + if (!*sqlstate) sqlstate= mysql_errno_to_sqlstate(sql_errno); - if ((level == Sql_condition::WARN_LEVEL_WARN) && - really_abort_on_warning()) + if ((level == Sql_condition::WARN_LEVEL_WARN) && really_abort_on_warning()) { - /* - FIXME: - push_warning and strict SQL_MODE case. - */ + /* FIXME: push_warning and strict SQL_MODE case. */ level= Sql_condition::WARN_LEVEL_ERROR; } if (!is_fatal_error && - handle_condition(sql_errno, sqlstate, &level, msg, &cond)) - DBUG_RETURN(cond); + handle_condition(sql_errno, sqlstate, &level, msg, &raised)) + goto ret; switch (level) { case Sql_condition::WARN_LEVEL_NOTE: @@ -1122,8 +1106,7 @@ Sql_condition* THD::raise_condition(uint sql_errno, With wsrep we allow converting BF abort error to warning if errors are ignored. */ - if (!is_fatal_error && - no_errors && + if (!is_fatal_error && no_errors && (wsrep_trx().bf_aborted() || wsrep_retry_counter)) { WSREP_DEBUG("BF abort error converted to warning"); @@ -1134,7 +1117,7 @@ Sql_condition* THD::raise_condition(uint sql_errno, if (!da->is_error()) { set_row_count_func(-1); - da->set_error_status(sql_errno, msg, sqlstate, ucid, cond); + da->set_error_status(sql_errno, msg, sqlstate, *cond, raised); } } } @@ -1149,9 +1132,13 @@ Sql_condition* THD::raise_condition(uint sql_errno, if (likely(!(is_fatal_error && (sql_errno == EE_OUTOFMEMORY || sql_errno == ER_OUTOFMEMORY)))) { - cond= da->push_warning(this, sql_errno, sqlstate, level, ucid, msg); + raised= da->push_warning(this, sql_errno, sqlstate, level, *cond, msg, + cond->m_row_number); } - DBUG_RETURN(cond); +ret: + if (raised) + raised->copy_opt_attributes(cond); + DBUG_RETURN(raised); } extern "C" @@ -1313,10 +1300,7 @@ void THD::init() wsrep_desynced_backup_stage= false; #endif /* WITH_WSREP */ - if (variables.sql_log_bin) - variables.option_bits|= OPTION_BIN_LOG; - else - variables.option_bits&= ~OPTION_BIN_LOG; + set_binlog_bit(); select_commands= update_commands= other_commands= 0; /* Set to handle counting of aborted connections */ @@ -5394,6 +5378,16 @@ thd_rpl_deadlock_check(MYSQL_THD thd, MYSQL_THD other_thd) return 0; if (rgi->gtid_sub_id > other_rgi->gtid_sub_id) return 0; + if (rgi->finish_event_group_called || other_rgi->finish_event_group_called) + { + /* + If either of two transactions has already performed commit + (e.g split ALTER, asserted below) there won't be any deadlock. + */ + DBUG_ASSERT(rgi->sa_info || other_rgi->sa_info); + + return 0; + } /* This transaction is about to wait for another transaction that is required by replication binlog order to commit after. This would cause a deadlock. diff --git a/sql/sql_class.h b/sql/sql_class.h index 6265d8060ce..c5085590511 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -87,6 +87,7 @@ enum wsrep_consistency_check_mode { class Reprepare_observer; class Relay_log_info; struct rpl_group_info; +struct rpl_parallel_thread; class Rpl_filter; class Query_log_event; class Load_log_event; @@ -319,9 +320,9 @@ class Key_part_spec :public Sql_alloc { public: LEX_CSTRING field_name; uint length; - bool generated; + bool generated, asc; Key_part_spec(const LEX_CSTRING *name, uint len, bool gen= false) - : field_name(*name), length(len), generated(gen) + : field_name(*name), length(len), generated(gen), asc(1) {} bool operator==(const Key_part_spec& other) const; /** @@ -589,7 +590,8 @@ typedef enum enum_diag_condition_item_name DIAG_CURSOR_NAME= 9, DIAG_MESSAGE_TEXT= 10, DIAG_MYSQL_ERRNO= 11, - LAST_DIAG_SET_PROPERTY= DIAG_MYSQL_ERRNO + DIAG_ROW_NUMBER= 12, + LAST_DIAG_SET_PROPERTY= DIAG_ROW_NUMBER } Diag_condition_item_name; /** @@ -881,6 +883,7 @@ typedef struct system_variables vers_asof_timestamp_t vers_asof_timestamp; ulong vers_alter_history; + my_bool binlog_alter_two_phase; } SV; /** @@ -1160,6 +1163,7 @@ struct THD_count { static Atomic_counter<uint32_t> count; static uint value() { return static_cast<uint>(count); } + static uint connection_thd_count(); THD_count() { count++; } ~THD_count() { count--; } }; @@ -3070,6 +3074,11 @@ public: } bool binlog_table_should_be_logged(const LEX_CSTRING *db); + // Accessors and setters of two-phase loggable ALTER binlog properties + uchar get_binlog_flags_for_alter(); + void set_binlog_flags_for_alter(uchar); + uint64 get_binlog_start_alter_seq_no(); + void set_binlog_start_alter_seq_no(uint64); #endif /* MYSQL_CLIENT */ public: @@ -3125,8 +3134,8 @@ public: { bzero((char*)this, sizeof(*this)); implicit_xid.null(); - init_sql_alloc(key_memory_thd_transactions, &mem_root, - ALLOC_ROOT_MIN_BLOCK_SIZE, 0, MYF(MY_THREAD_SPECIFIC)); + init_sql_alloc(key_memory_thd_transactions, &mem_root, 256, + 0, MYF(MY_THREAD_SPECIFIC)); } } default_transaction, *transaction; Global_read_lock global_read_lock; @@ -3296,6 +3305,13 @@ public: auto_inc_intervals_forced.empty(); // in case of multiple SET INSERT_ID auto_inc_intervals_forced.append(next_id, ULONGLONG_MAX, 0); } + inline void set_binlog_bit() + { + if (variables.sql_log_bin) + variables.option_bits |= OPTION_BIN_LOG; + else + variables.option_bits &= ~OPTION_BIN_LOG; + } ulonglong limit_found_rows; @@ -3992,6 +4008,11 @@ public: user_time= t; set_time(); } + inline void force_set_time(my_time_t t, ulong sec_part) + { + start_time= system_time.sec= t; + start_time_sec_part= system_time.sec_part= sec_part; + } /* this is only used by replication and BINLOG command. usecs > TIME_MAX_SECOND_PART means "was not in binlog" @@ -4003,15 +4024,9 @@ public: else { if (sec_part <= TIME_MAX_SECOND_PART) - { - start_time= system_time.sec= t; - start_time_sec_part= system_time.sec_part= sec_part; - } + force_set_time(t, sec_part); else if (t != system_time.sec) - { - start_time= system_time.sec= t; - start_time_sec_part= system_time.sec_part= 0; - } + force_set_time(t, 0); else { start_time= t; @@ -4879,45 +4894,17 @@ private: @param msg the condition message text @return The condition raised, or NULL */ - Sql_condition* - raise_condition(uint sql_errno, - const char* sqlstate, - Sql_condition::enum_warning_level level, - const char* msg) + Sql_condition* raise_condition(uint sql_errno, const char* sqlstate, + Sql_condition::enum_warning_level level, const char* msg) { - return raise_condition(sql_errno, sqlstate, level, - Sql_user_condition_identity(), msg); + Sql_condition cond(NULL, // don't strdup the msg + Sql_condition_identity(sql_errno, sqlstate, level, + Sql_user_condition_identity()), + msg, get_stmt_da()->current_row_for_warning()); + return raise_condition(&cond); } - /** - Raise a generic or a user defined SQL condition. - @param ucid - the user condition identity - (or an empty identity if not a user condition) - @param sql_errno - the condition error number - @param sqlstate - the condition SQLSTATE - @param level - the condition level - @param msg - the condition message text - @return The condition raised, or NULL - */ - Sql_condition* - raise_condition(uint sql_errno, - const char* sqlstate, - Sql_condition::enum_warning_level level, - const Sql_user_condition_identity &ucid, - const char* msg); - - Sql_condition* - raise_condition(const Sql_condition *cond) - { - Sql_condition *raised= raise_condition(cond->get_sql_errno(), - cond->get_sqlstate(), - cond->get_level(), - *cond/*Sql_user_condition_identity*/, - cond->get_message_text()); - if (raised) - raised->copy_opt_attributes(cond); - return raised; - } + Sql_condition* raise_condition(const Sql_condition *cond); private: void push_warning_truncated_priv(Sql_condition::enum_warning_level level, @@ -5559,7 +5546,8 @@ public: bool restore_from_local_lex_to_old_lex(LEX *oldlex); Item *sp_fix_func_item(Item **it_addr); - Item *sp_prepare_func_item(Item **it_addr, uint cols= 1); + Item *sp_fix_func_item_for_assignment(const Field *to, Item **it_addr); + Item *sp_prepare_func_item(Item **it_addr, uint cols); bool sp_eval_expr(Field *result_field, Item **expr_item_ptr); bool sql_parser(LEX *old_lex, LEX *lex, @@ -7945,6 +7933,41 @@ extern THD_list server_threads; void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps, uint field_count); +#ifdef WITH_WSREP +extern void wsrep_to_isolation_end(THD*); +#endif +/* + RAII utility class to ease binlogging with temporary setting + THD etc context and restoring the original one upon logger execution. +*/ +class Write_log_with_flags +{ + THD* m_thd; +#ifdef WITH_WSREP + bool wsrep_to_isolation; +#endif + +public: +~Write_log_with_flags() + { + m_thd->set_binlog_flags_for_alter(0); + m_thd->set_binlog_start_alter_seq_no(0); +#ifdef WITH_WSREP + if (wsrep_to_isolation) + wsrep_to_isolation_end(m_thd); +#endif + } + + Write_log_with_flags(THD *thd, uchar flags, + bool do_wsrep_iso __attribute__((unused))= false) : + m_thd(thd) + { + m_thd->set_binlog_flags_for_alter(flags); +#ifdef WITH_WSREP + wsrep_to_isolation= do_wsrep_iso && WSREP(m_thd); +#endif + } +}; #endif /* MYSQL_SERVER */ #endif /* SQL_CLASS_INCLUDED */ diff --git a/sql/sql_cmd.h b/sql/sql_cmd.h index 53dd6e750f8..6554fc78f27 100644 --- a/sql/sql_cmd.h +++ b/sql/sql_cmd.h @@ -75,7 +75,6 @@ enum enum_sql_command { SQLCOM_XA_START, SQLCOM_XA_END, SQLCOM_XA_PREPARE, SQLCOM_XA_COMMIT, SQLCOM_XA_ROLLBACK, SQLCOM_XA_RECOVER, SQLCOM_SHOW_PROC_CODE, SQLCOM_SHOW_FUNC_CODE, - SQLCOM_ALTER_TABLESPACE, SQLCOM_INSTALL_PLUGIN, SQLCOM_UNINSTALL_PLUGIN, SQLCOM_SHOW_AUTHORS, SQLCOM_BINLOG_BASE64_EVENT, SQLCOM_SHOW_PLUGINS, SQLCOM_SHOW_CONTRIBUTORS, diff --git a/sql/sql_db.cc b/sql/sql_db.cc index 37e136927f2..c5defc1959c 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -1363,9 +1363,7 @@ static bool find_db_tables_and_rm_known_files(THD *thd, MY_DIR *dirp, *tables= tot_list; /* and at last delete all non-table files */ - for (uint idx=0 ; - idx < (uint) dirp->number_of_files && !thd->killed ; - idx++) + for (size_t idx=0; idx < dirp->number_of_files && !thd->killed; idx++) { FILEINFO *file=dirp->dir_entry+idx; char *extension; @@ -1488,9 +1486,7 @@ long mysql_rm_arc_files(THD *thd, MY_DIR *dirp, const char *org_path) DBUG_ENTER("mysql_rm_arc_files"); DBUG_PRINT("enter", ("path: %s", org_path)); - for (uint idx=0 ; - idx < (uint) dirp->number_of_files && !thd->killed ; - idx++) + for (size_t idx=0; idx < dirp->number_of_files && !thd->killed; idx++) { FILEINFO *file=dirp->dir_entry+idx; char *extension, *revision; @@ -1970,8 +1966,8 @@ bool mysql_upgrade_db(THD *thd, const LEX_CSTRING *old_db) /* Step2: Move tables to the new database */ if ((dirp = my_dir(path,MYF(MY_DONT_SORT)))) { - uint nfiles= (uint) dirp->number_of_files; - for (uint idx=0 ; idx < nfiles && !thd->killed ; idx++) + size_t nfiles= dirp->number_of_files; + for (size_t idx=0 ; idx < nfiles && !thd->killed ; idx++) { FILEINFO *file= dirp->dir_entry + idx; char *extension, tname[FN_REFLEN + 1]; @@ -2060,8 +2056,8 @@ bool mysql_upgrade_db(THD *thd, const LEX_CSTRING *old_db) if ((dirp = my_dir(path,MYF(MY_DONT_SORT)))) { - uint nfiles= (uint) dirp->number_of_files; - for (uint idx=0 ; idx < nfiles ; idx++) + size_t nfiles= dirp->number_of_files; + for (size_t idx=0 ; idx < nfiles ; idx++) { FILEINFO *file= dirp->dir_entry + idx; char oldname[FN_REFLEN + 1], newname[FN_REFLEN + 1]; diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index b2e5d1ba397..41e2892be5f 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -733,6 +733,8 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, explain= (Explain_delete*)thd->lex->explain->get_upd_del_plan(); explain->tracker.on_scan_init(); + thd->get_stmt_da()->reset_current_row_for_warning(1); + if (!delete_while_scanning) { /* @@ -798,9 +800,11 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, THD_STAGE_INFO(thd, stage_updating); fix_rownum_pointers(thd, thd->lex->current_select, &deleted); + thd->get_stmt_da()->reset_current_row_for_warning(0); while (likely(!(error=info.read_record())) && likely(!thd->killed) && likely(!thd->is_error())) { + thd->get_stmt_da()->inc_current_row_for_warning(); if (delete_while_scanning) delete_record= record_should_be_deleted(thd, table, select, explain, delete_history); @@ -876,6 +880,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, else break; } + thd->get_stmt_da()->reset_current_row_for_warning(1); terminate_delete: killed_status= thd->killed; diff --git a/sql/sql_error.cc b/sql/sql_error.cc index cef9e6cec00..85be61c34ef 100644 --- a/sql/sql_error.cc +++ b/sql/sql_error.cc @@ -204,6 +204,7 @@ Sql_condition::copy_opt_attributes(const Sql_condition *cond) copy_string(m_mem_root, & m_table_name, & cond->m_table_name); copy_string(m_mem_root, & m_column_name, & cond->m_column_name); copy_string(m_mem_root, & m_cursor_name, & cond->m_cursor_name); + m_row_number= cond->m_row_number; } @@ -216,7 +217,7 @@ Sql_condition::set_builtin_message_text(const char* str) */ const char* copy; - copy= strdup_root(m_mem_root, str); + copy= m_mem_root ? strdup_root(m_mem_root, str) : str; m_message_text.set(copy, strlen(copy), error_message_charset_info); DBUG_ASSERT(! m_message_text.is_alloced()); } @@ -500,7 +501,7 @@ Diagnostics_area::disable_status() Warning_info::Warning_info(ulonglong warn_id_arg, bool allow_unlimited_warnings, bool initialize) :m_current_statement_warn_count(0), - m_current_row_for_warning(1), + m_current_row_for_warning(0), m_warn_id(warn_id_arg), m_error_condition(NULL), m_allow_unlimited_warnings(allow_unlimited_warnings), @@ -557,7 +558,7 @@ void Warning_info::clear(ulonglong new_id) free_memory(); memset(m_warn_count, 0, sizeof(m_warn_count)); m_current_statement_warn_count= 0; - m_current_row_for_warning= 1; /* Start counting from the first row */ + m_current_row_for_warning= 0; clear_error_condition(); } @@ -663,7 +664,8 @@ void Warning_info::reserve_space(THD *thd, uint count) Sql_condition *Warning_info::push_warning(THD *thd, const Sql_condition_identity *value, - const char *msg) + const char *msg, + ulong current_row_number) { Sql_condition *cond= NULL; @@ -672,7 +674,8 @@ Sql_condition *Warning_info::push_warning(THD *thd, if (m_allow_unlimited_warnings || m_warn_list.elements() < thd->variables.max_error_count) { - cond= new (& m_warn_root) Sql_condition(& m_warn_root, *value, msg); + cond= new (& m_warn_root) Sql_condition(& m_warn_root, *value, msg, + current_row_number); if (cond) m_warn_list.push_back(cond); } @@ -688,7 +691,8 @@ Sql_condition *Warning_info::push_warning(THD *thd, const Sql_condition *sql_condition) { Sql_condition *new_condition= push_warning(thd, sql_condition, - sql_condition->get_message_text()); + sql_condition->get_message_text(), + sql_condition->m_row_number); if (new_condition) new_condition->copy_opt_attributes(sql_condition); @@ -723,7 +727,7 @@ void push_warning(THD *thd, Sql_condition::enum_warning_level level, if (level == Sql_condition::WARN_LEVEL_ERROR) level= Sql_condition::WARN_LEVEL_WARN; - (void) thd->raise_condition(code, NULL, level, msg); + (void) thd->raise_condition(code, "\0\0\0\0\0", level, msg); /* Make sure we also count warnings pushed after calling set_ok_status(). */ thd->get_stmt_da()->increment_warning(); diff --git a/sql/sql_error.h b/sql/sql_error.h index 679672e5994..541b92b4531 100644 --- a/sql/sql_error.h +++ b/sql/sql_error.h @@ -305,6 +305,9 @@ protected: /** SQL CURSOR_NAME condition item. */ String m_cursor_name; + /** SQL ROW_NUMBER condition item. */ + ulong m_row_number; + Sql_condition_items() :m_class_origin((const char*) NULL, 0, & my_charset_utf8mb3_bin), m_subclass_origin((const char*) NULL, 0, & my_charset_utf8mb3_bin), @@ -315,7 +318,8 @@ protected: m_schema_name((const char*) NULL, 0, & my_charset_utf8mb3_bin), m_table_name((const char*) NULL, 0, & my_charset_utf8mb3_bin), m_column_name((const char*) NULL, 0, & my_charset_utf8mb3_bin), - m_cursor_name((const char*) NULL, 0, & my_charset_utf8mb3_bin) + m_cursor_name((const char*) NULL, 0, & my_charset_utf8mb3_bin), + m_row_number(0) { } void clear() @@ -330,6 +334,7 @@ protected: m_table_name.length(0); m_column_name.length(0); m_cursor_name.length(0); + m_row_number= 0; } }; @@ -433,16 +438,14 @@ private: @param level - the error level for this condition @param msg - the message text for this condition */ - Sql_condition(MEM_ROOT *mem_root, - const Sql_condition_identity &value, - const char *msg) - :Sql_condition_identity(value), - m_mem_root(mem_root) + Sql_condition(MEM_ROOT *mem_root, const Sql_condition_identity &value, + const char *msg, ulong current_row_for_warning) + : Sql_condition_identity(value), m_mem_root(mem_root) { - DBUG_ASSERT(mem_root != NULL); DBUG_ASSERT(value.get_sql_errno() != 0); DBUG_ASSERT(msg != NULL); set_builtin_message_text(msg); + m_row_number= current_row_for_warning; } /** Destructor. */ @@ -718,7 +721,7 @@ private: void inc_current_row_for_warning() { m_current_row_for_warning++; } /** Reset the current row counter. Start counting from the first row. */ - void reset_current_row_for_warning() { m_current_row_for_warning= 1; } + void reset_current_row_for_warning(int n) { m_current_row_for_warning= n; } ulong set_current_row_for_warning(ulong row) { @@ -747,9 +750,8 @@ private: @return a pointer to the added SQL-condition. */ - Sql_condition *push_warning(THD *thd, - const Sql_condition_identity *identity, - const char* msg); + Sql_condition *push_warning(THD *thd, const Sql_condition_identity *identity, + const char* msg, ulong current_row_number); /** Add a new SQL-condition to the current list and increment the respective @@ -1151,8 +1153,8 @@ public: void inc_current_row_for_warning() { get_warning_info()->inc_current_row_for_warning(); } - void reset_current_row_for_warning() - { get_warning_info()->reset_current_row_for_warning(); } + void reset_current_row_for_warning(int n) + { get_warning_info()->reset_current_row_for_warning(n); } bool is_warning_info_read_only() const { return get_warning_info()->is_read_only(); } @@ -1183,10 +1185,12 @@ public: const char* sqlstate, Sql_condition::enum_warning_level level, const Sql_user_condition_identity &ucid, - const char* msg) + const char* msg, + ulong current_row_number) { Sql_condition_identity tmp(sql_errno_arg, sqlstate, level, ucid); - return get_warning_info()->push_warning(thd, &tmp, msg); + return get_warning_info()->push_warning(thd, &tmp, msg, + current_row_number); } Sql_condition *push_warning(THD *thd, @@ -1196,7 +1200,7 @@ public: const char* msg) { return push_warning(thd, sqlerrno, sqlstate, level, - Sql_user_condition_identity(), msg); + Sql_user_condition_identity(), msg, 0); } void mark_sql_conditions_for_removal() { get_warning_info()->mark_sql_conditions_for_removal(); } diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index e9bdc4be66f..e49ba0da11c 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -25,6 +25,8 @@ #include "opt_range.h" #include "sql_expression_cache.h" +#include <stack> + const char * STR_DELETING_ALL_ROWS= "Deleting all rows"; const char * STR_IMPOSSIBLE_WHERE= "Impossible WHERE"; const char * STR_NO_ROWS_AFTER_PRUNING= "No matching rows after partition pruning"; @@ -41,7 +43,7 @@ static void write_item(Json_writer *writer, Item *item); static void append_item_to_str(String *out, Item *item); Explain_query::Explain_query(THD *thd_arg, MEM_ROOT *root) : - mem_root(root), upd_del_plan(NULL), insert_plan(NULL), + mem_root(root), upd_del_plan(nullptr), insert_plan(nullptr), unions(root), selects(root), thd(thd_arg), apc_enabled(false), operations(0) { @@ -1090,14 +1092,13 @@ void Explain_aggr_window_funcs::print_json_members(Json_writer *writer, { Explain_aggr_filesort *srt; List_iterator<Explain_aggr_filesort> it(sorts); - writer->add_member("sorts").start_object(); + Json_writer_array sorts(writer, "sorts"); while ((srt= it++)) { - writer->add_member("filesort").start_object(); + Json_writer_object sort(writer); + Json_writer_object filesort(writer, "filesort"); srt->print_json_members(writer, is_analyze); - writer->end_object(); // filesort } - writer->end_object(); // sorts } @@ -1119,17 +1120,26 @@ print_explain_json_interns(Explain_query *query, Json_writer *writer, bool is_analyze) { - Json_writer_nesting_guard guard(writer); - for (uint i=0; i< n_join_tabs; i++) { - if (join_tabs[i]->start_dups_weedout) - writer->add_member("duplicates_removal").start_object(); + Json_writer_array loop(writer, "nested_loop"); + for (uint i=0; i< n_join_tabs; i++) + { + if (join_tabs[i]->start_dups_weedout) + { + writer->start_object(); + writer->add_member("duplicates_removal"); + writer->start_array(); + } - join_tabs[i]->print_explain_json(query, writer, is_analyze); + join_tabs[i]->print_explain_json(query, writer, is_analyze); - if (join_tabs[i]->end_dups_weedout) - writer->end_object(); - } + if (join_tabs[i]->end_dups_weedout) + { + writer->end_array(); + writer->end_object(); + } + } + } // "nested_loop" print_explain_json_for_children(query, writer, is_analyze); } @@ -1714,7 +1724,7 @@ void Explain_table_access::print_explain_json(Explain_query *query, Json_writer *writer, bool is_analyze) { - Json_writer_nesting_guard guard(writer); + Json_writer_object jsobj(writer); if (pre_join_sort) { @@ -2133,14 +2143,15 @@ void Explain_quick_select::print_json(Json_writer *writer) } else { - writer->add_member(get_name_by_type()).start_object(); + Json_writer_array ranges(writer, get_name_by_type()); List_iterator_fast<Explain_quick_select> it (children); Explain_quick_select* child; while ((child = it++)) + { + Json_writer_object obj(writer); child->print_json(writer); - - writer->end_object(); + } } } diff --git a/sql/sql_explain.h b/sql/sql_explain.h index af110a25296..7639df4604a 100644 --- a/sql/sql_explain.h +++ b/sql/sql_explain.h @@ -74,7 +74,6 @@ class Json_writer; *************************************************************************************/ -const uint FAKE_SELECT_LEX_ID= UINT_MAX; class Explain_query; diff --git a/sql/sql_get_diagnostics.cc b/sql/sql_get_diagnostics.cc index 197bf5e7a00..240975d2974 100644 --- a/sql/sql_get_diagnostics.cc +++ b/sql/sql_get_diagnostics.cc @@ -338,6 +338,8 @@ Condition_information_item::get_value(THD *thd, const Sql_condition *cond) str.set_ascii(cond->get_sqlstate(), strlen(cond->get_sqlstate())); value= make_utf8_string_item(thd, &str); break; + case ROW_NUMBER: + value= new (thd->mem_root) Item_uint(thd, cond->m_row_number); } DBUG_RETURN(value); diff --git a/sql/sql_get_diagnostics.h b/sql/sql_get_diagnostics.h index f283aa5b2c6..efe526d7c61 100644 --- a/sql/sql_get_diagnostics.h +++ b/sql/sql_get_diagnostics.h @@ -254,7 +254,8 @@ public: CURSOR_NAME, MESSAGE_TEXT, MYSQL_ERRNO, - RETURNED_SQLSTATE + RETURNED_SQLSTATE, + ROW_NUMBER }; /** diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 22374458c3d..45296f22f35 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -229,6 +229,7 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, } if (values.elements != table->s->visible_fields) { + thd->get_stmt_da()->reset_current_row_for_warning(1); my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), 1L); DBUG_RETURN(-1); } @@ -253,6 +254,7 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, if (fields.elements != values.elements) { + thd->get_stmt_da()->reset_current_row_for_warning(1); my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), 1L); DBUG_RETURN(-1); } @@ -699,7 +701,6 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, const bool was_insert_delayed= (table_list->lock_type == TL_WRITE_DELAYED); bool using_bulk_insert= 0; uint value_count; - ulong counter = 1; /* counter of iteration in bulk PS operation*/ ulonglong iteration= 0; ulonglong id; @@ -767,7 +768,7 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, value_count= values->elements; if ((res= mysql_prepare_insert(thd, table_list, fields, values, - update_fields, update_values, duplic, + update_fields, update_values, duplic, ignore, &unused_conds, FALSE))) { retval= thd->is_error(); @@ -828,12 +829,27 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, context->resolve_in_table_list_only(table_list); switch_to_nullable_trigger_fields(*values, table); + /* + Check assignability for the leftmost () in VALUES: + INSERT INTO t1 (a,b) VALUES (1,2), (3,4); + This checks if the values (1,2) can be assigned to fields (a,b). + The further values, e.g. (3,4) are not checked - they will be + checked during the execution time (when processing actual rows). + This is to preserve the "insert until the very first error"-style + behaviour for non-transactional tables. + */ + if (values->elements && + table_list->table->check_assignability_opt_fields(fields, *values, + ignore)) + goto abort; + while ((values= its++)) { - counter++; + thd->get_stmt_da()->inc_current_row_for_warning(); if (values->elements != value_count) { - my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), counter); + my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), + thd->get_stmt_da()->current_row_for_warning()); goto abort; } if (setup_fields(thd, Ref_ptr_array(), @@ -842,6 +858,7 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, switch_to_nullable_trigger_fields(*values, table); } its.rewind (); + thd->get_stmt_da()->reset_current_row_for_warning(0); /* Restore the current context. */ ctx_state.restore_state(context, table_list); @@ -1014,6 +1031,7 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, while ((values= its++)) { + thd->get_stmt_da()->inc_current_row_for_warning(); if (fields.elements || !value_count) { /* @@ -1130,7 +1148,6 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, if (unlikely(error)) break; info.accepted_rows++; - thd->get_stmt_da()->inc_current_row_for_warning(); } its.rewind(); iteration++; @@ -1163,8 +1180,13 @@ values_loop_end: table->file->ha_release_auto_increment(); if (using_bulk_insert) { - if (unlikely(table->file->ha_end_bulk_insert()) && - !error) + /* + if my_error() wasn't called yet on some specific row, end_bulk_insert() + can still do it, but the error shouldn't be for any specific row number + */ + if (!error) + thd->get_stmt_da()->reset_current_row_for_warning(0); + if (unlikely(table->file->ha_end_bulk_insert()) && !error) { table->file->print_error(my_errno,MYF(0)); error=1; @@ -1425,7 +1447,7 @@ static bool check_view_insertability(THD * thd, TABLE_LIST *view) DBUG_ASSERT(view->table != 0 && view->field_translation != 0); - (void) my_bitmap_init(&used_fields, used_fields_buff, table->s->fields, 0); + (void) my_bitmap_init(&used_fields, used_fields_buff, table->s->fields); bitmap_clear_all(&used_fields); view->contain_auto_increment= 0; @@ -1618,7 +1640,8 @@ static void prepare_for_positional_update(TABLE *table, TABLE_LIST *tables) int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, List<Item> &fields, List_item *values, List<Item> &update_fields, List<Item> &update_values, - enum_duplicates duplic, COND **where, + enum_duplicates duplic, bool ignore, + COND **where, bool select_insert) { SELECT_LEX *select_lex= thd->lex->first_select_lex(); @@ -1686,7 +1709,16 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, { select_lex->no_wrap_view_item= TRUE; res= check_update_fields(thd, context->table_list, update_fields, - update_values, false, &map); + update_values, false, &map) || + /* + Check that all col=expr pairs are compatible for assignment in + INSERT INTO t1 VALUES (...) + ON DUPLICATE KEY UPDATE col=expr [, col=expr]; + */ + TABLE::check_assignability_explicit_fields(update_fields, + update_values, + ignore); + select_lex->no_wrap_view_item= FALSE; } @@ -1694,6 +1726,8 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, ctx_state.restore_state(context, table_list); } + thd->get_stmt_da()->reset_current_row_for_warning(1); + if (res) DBUG_RETURN(res); @@ -2801,7 +2835,7 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) my_bitmap_init(©->has_value_set, (my_bitmap_map*) (bitmap + bitmaps_used*share->column_bitmap_size), - share->fields, FALSE); + share->fields); } copy->tmp_set.bitmap= 0; // To catch errors bzero((char*) bitmap, share->column_bitmap_size * bitmaps_used); @@ -3784,7 +3818,7 @@ int mysql_insert_select_prepare(THD *thd, select_result *sel_res) if ((res= mysql_prepare_insert(thd, lex->query_tables, lex->field_list, 0, lex->update_list, lex->value_list, - lex->duplicates, + lex->duplicates, lex->ignore, &select_lex->where, TRUE))) DBUG_RETURN(res); @@ -3878,6 +3912,17 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) check_insert_fields(thd, table_list, *fields, values, !insert_into_view, 1, &map)); + if (!res) + { + /* + Check that all colN=exprN pairs are compatible for assignment, e.g.: + INSERT INTO t1 (col1, col2) VALUES (expr1, expr2); + INSERT INTO t1 SET col1=expr1, col2=expr2; + */ + res= table_list->table->check_assignability_opt_fields(*fields, values, + lex->ignore); + } + if (!res && fields->elements) { Abort_on_warning_instant_set aws(thd, @@ -3931,7 +3976,15 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) } res= res || setup_fields(thd, Ref_ptr_array(), *info.update_values, - MARK_COLUMNS_READ, 0, NULL, 0); + MARK_COLUMNS_READ, 0, NULL, 0) || + /* + Check that all col=expr pairs are compatible for assignment in + INSERT INTO t1 SELECT ... FROM t2 + ON DUPLICATE KEY UPDATE col=expr [, col=expr] + */ + TABLE::check_assignability_explicit_fields(*info.update_fields, + *info.update_values, + lex->ignore); if (!res) { /* @@ -5100,7 +5153,8 @@ bool select_create::send_eof() { WSREP_DEBUG("select_create commit failed, thd: %llu err: %s %s", thd->thread_id, - wsrep_thd_transaction_state_str(thd), wsrep_thd_query(thd)); + wsrep_thd_transaction_state_str(thd), + wsrep_thd_query(thd)); mysql_mutex_unlock(&thd->LOCK_thd_data); abort_result_set(); DBUG_RETURN(true); diff --git a/sql/sql_insert.h b/sql/sql_insert.h index 80666a81c50..8b034c25877 100644 --- a/sql/sql_insert.h +++ b/sql/sql_insert.h @@ -27,6 +27,7 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, List<Item> &fields, List_item *values, List<Item> &update_fields, List<Item> &update_values, enum_duplicates duplic, + bool ignore, COND **where, bool select_insert); bool mysql_insert(THD *thd,TABLE_LIST *table,List<Item> &fields, List<List_item> &values, List<Item> &update_fields, diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 5967a4a452a..807fcd3ba12 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -1352,7 +1352,7 @@ void lex_unlock_plugins(LEX *lex) /* release used plugins */ if (lex->plugins.elements) /* No function call and no mutex if no plugins. */ { - plugin_unlock_list(0, (plugin_ref*)lex->plugins.buffer, + plugin_unlock_list(0, (plugin_ref*)lex->plugins.buffer, lex->plugins.elements); } reset_dynamic(&lex->plugins); @@ -6054,8 +6054,10 @@ int st_select_lex_unit::save_union_explain_part2(Explain_query *output) bool LEX::is_partition_management() const { return (sql_command == SQLCOM_ALTER_TABLE && - (alter_info.partition_flags == ALTER_PARTITION_ADD || - alter_info.partition_flags == ALTER_PARTITION_REORGANIZE)); + (alter_info.partition_flags & (ALTER_PARTITION_ADD | + ALTER_PARTITION_CONVERT_IN | + ALTER_PARTITION_CONVERT_OUT | + ALTER_PARTITION_REORGANIZE))); } @@ -7140,6 +7142,27 @@ bool LEX::sp_declare_cursor(THD *thd, const LEX_CSTRING *name, uint offp; sp_instr_cpush *i; + /* In some cases param_ctx can be NULL. e.g.: FOR rec IN (SELECT...) */ + if (param_ctx) + { + for (uint prm= 0; prm < param_ctx->context_var_count(); prm++) + { + const sp_variable *param= param_ctx->get_context_variable(prm); + if (param->mode != sp_variable::MODE_IN) + { + /* + PL/SQL supports the IN keyword in cursor parameters. + We also support this for compatibility. Note, OUT/INOUT parameters + will unlikely be ever supported. So "YET" may sound confusing here. + But it should be better than using a generic error. Adding a dedicated + error message for this small issue is not desirable. + */ + my_error(ER_NOT_SUPPORTED_YET, MYF(0), "OUT/INOUT cursor parameter"); + return true; + } + } + } + if (spcont->find_cursor(name, &offp, true)) { my_error(ER_SP_DUP_CURS, MYF(0), name->str); @@ -9698,6 +9721,7 @@ bool Lex_ident_sys_st::to_size_number(ulonglong *to) const } +#ifdef WITH_PARTITION_STORAGE_ENGINE bool LEX::part_values_current(THD *thd) { partition_element *elem= part_info->curr_part_elem; @@ -9705,7 +9729,7 @@ bool LEX::part_values_current(THD *thd) { if (unlikely(part_info->part_type != VERSIONING_PARTITION)) { - my_error(ER_PARTITION_WRONG_TYPE, MYF(0), "SYSTEM_TIME"); + part_type_error(thd, NULL, "SYSTEM_TIME", part_info); return true; } } @@ -9732,7 +9756,7 @@ bool LEX::part_values_history(THD *thd) { if (unlikely(part_info->part_type != VERSIONING_PARTITION)) { - my_error(ER_PARTITION_WRONG_TYPE, MYF(0), "SYSTEM_TIME"); + part_type_error(thd, NULL, "SYSTEM_TIME", part_info); return true; } } @@ -9753,6 +9777,7 @@ bool LEX::part_values_history(THD *thd) elem->type= partition_element::HISTORY; return false; } +#endif /* WITH_PARTITION_STORAGE_ENGINE */ bool LEX::last_field_generated_always_as_row_start_or_end(Lex_ident *p, @@ -11327,6 +11352,25 @@ bool LEX::stmt_alter_table_exchange_partition(Table_ident *table) } +bool LEX::stmt_alter_table(Table_ident *table) +{ + DBUG_ASSERT(sql_command == SQLCOM_ALTER_TABLE); + first_select_lex()->db= table->db; + if (first_select_lex()->db.str == NULL && + copy_db_to(&first_select_lex()->db)) + return true; + if (unlikely(check_table_name(table->table.str, table->table.length, + false)) || + (table->db.str && unlikely(check_db_name((LEX_STRING*) &table->db)))) + { + my_error(ER_WRONG_TABLE_NAME, MYF(0), table->table.str); + return true; + } + name= table->table; + return false; +} + + void LEX::stmt_purge_to(const LEX_CSTRING &to) { type= 0; diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 24185dbc734..c9a1f8678d8 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -291,7 +291,6 @@ class sp_pcontext; class sp_variable; class sp_expr_lex; class sp_assignment_lex; -class st_alter_tablespace; class partition_info; class Event_parse_data; class set_var_base; @@ -3531,12 +3530,6 @@ public: /* - Reference to a struct that contains information in various commands - to add/create/drop/change table spaces. - */ - st_alter_tablespace *alter_tablespace_info; - - /* The set of those tables whose fields are referenced in all subqueries of the query. TODO: possibly this it is incorrect to have used tables in LEX because @@ -3793,8 +3786,10 @@ public: bool table_or_sp_used(); bool is_partition_management() const; +#ifdef WITH_PARTITION_STORAGE_ENGINE bool part_values_current(THD *thd); bool part_values_history(THD *thd); +#endif /** @brief check if the statement is a single-level join @@ -4723,6 +4718,7 @@ public: void stmt_deallocate_prepare(const Lex_ident_sys_st &ident); bool stmt_alter_table_exchange_partition(Table_ident *table); + bool stmt_alter_table(Table_ident *table); void stmt_purge_to(const LEX_CSTRING &to); bool stmt_purge_before(Item *item); diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 2f1ee0b11bd..fe574db528f 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -659,6 +659,7 @@ int mysql_load(THD *thd, const sql_exchange *ex, TABLE_LIST *table_list, table->copy_blobs=1; thd->abort_on_warning= !ignore && thd->is_strict_mode(); + thd->get_stmt_da()->reset_current_row_for_warning(1); bool create_lookup_handler= handle_duplicates != DUP_ERROR; if ((table_list->table->file->ha_table_flags() & HA_DUPLICATE_POS)) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f2f622e78c8..9fe2506bbae 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -54,7 +54,6 @@ // check_mqh, // reset_mqh #include "sql_rename.h" // mysql_rename_tables -#include "sql_tablespace.h" // mysql_alter_tablespace #include "hostname.h" // hostname_cache_refresh #include "sql_test.h" // mysql_print_status #include "sql_select.h" // handle_select, mysql_select, @@ -878,7 +877,6 @@ void init_update_queries(void) sql_command_flags[SQLCOM_ALTER_PROCEDURE]|= CF_DISALLOW_IN_RO_TRANS; sql_command_flags[SQLCOM_ALTER_FUNCTION]|= CF_DISALLOW_IN_RO_TRANS; sql_command_flags[SQLCOM_TRUNCATE]|= CF_DISALLOW_IN_RO_TRANS; - sql_command_flags[SQLCOM_ALTER_TABLESPACE]|= CF_DISALLOW_IN_RO_TRANS; sql_command_flags[SQLCOM_REPAIR]|= CF_DISALLOW_IN_RO_TRANS; sql_command_flags[SQLCOM_OPTIMIZE]|= CF_DISALLOW_IN_RO_TRANS; sql_command_flags[SQLCOM_GRANT]|= CF_DISALLOW_IN_RO_TRANS; @@ -4140,7 +4138,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) If new master was not added, we still need to free mi. */ if (master_info_added) - master_info_index->remove_master_info(mi); + master_info_index->remove_master_info(mi, 1); else delete mi; } @@ -5906,12 +5904,6 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_XA_RECOVER: res= mysql_xa_recover(thd); break; - case SQLCOM_ALTER_TABLESPACE: - if (check_global_access(thd, CREATE_TABLESPACE_ACL)) - break; - if (!(res= mysql_alter_tablespace(thd, lex->alter_tablespace_info))) - my_ok(thd); - break; case SQLCOM_INSTALL_PLUGIN: if (! (res= mysql_install_plugin(thd, &thd->lex->comment, &thd->lex->ident))) @@ -6143,7 +6135,12 @@ finish: thd->wsrep_consistency_check= NO_CONSISTENCY_CHECK; if (wsrep_thd_is_toi(thd) || wsrep_thd_is_in_rsu(thd)) + { + WSREP_DEBUG("mysql_execute_command for %s", wsrep_thd_query(thd)); + THD_STAGE_INFO(thd, stage_waiting_isolation); wsrep_to_isolation_end(thd); + } + /* Force release of transactional locks if not in active MST and wsrep is on. */ @@ -7896,7 +7893,8 @@ static bool wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, }); #endif WSREP_DEBUG("wsrep retrying AC query: %lu %s", - thd->wsrep_retry_counter, wsrep_thd_query(thd)); + thd->wsrep_retry_counter, + wsrep_thd_query(thd)); wsrep_prepare_for_autocommit_retry(thd, rawbuf, length, parser_state); if (thd->lex->explain) delete_explain_query(thd->lex); diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index adb1ad391cf..9c7f25e8808 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -67,9 +67,9 @@ #include "opt_range.h" // store_key_image_to_rec #include "sql_alter.h" // Alter_table_ctx #include "sql_select.h" -#include "sql_tablespace.h" // check_tablespace_name #include "ddl_log.h" #include "tztime.h" // my_tz_OFFSET0 +#include "create_options.h" // engine_option_value #include <algorithm> using std::max; @@ -78,11 +78,6 @@ using std::min; #ifdef WITH_PARTITION_STORAGE_ENGINE #include "ha_partition.h" -#define ERROR_INJECT_CRASH(code) \ - DBUG_EVALUATE_IF(code, (DBUG_SUICIDE(), 0), 0) -#define ERROR_INJECT_ERROR(code) \ - DBUG_EVALUATE_IF(code, (my_error(ER_UNKNOWN_ERROR, MYF(0)), TRUE), 0) - /* Partition related functions declarations and some static constants; */ @@ -525,7 +520,7 @@ static bool create_full_part_field_array(THD *thd, TABLE *table, goto end; } if (unlikely(my_bitmap_init(&part_info->full_part_field_set, bitmap_buf, - table->s->fields, FALSE))) + table->s->fields))) { result= TRUE; goto end; @@ -1100,10 +1095,10 @@ static bool set_up_partition_bitmaps(THD *thd, partition_info *part_info) bitmap_bytes * 2)))) DBUG_RETURN(TRUE); - my_bitmap_init(&part_info->read_partitions, bitmap_buf, bitmap_bits, FALSE); + my_bitmap_init(&part_info->read_partitions, bitmap_buf, bitmap_bits); /* Use the second half of the allocated buffer for lock_partitions */ my_bitmap_init(&part_info->lock_partitions, bitmap_buf + (bitmap_bytes / 4), - bitmap_bits, FALSE); + bitmap_bits); part_info->bitmaps_are_initialized= TRUE; part_info->set_partition_bitmaps(NULL); DBUG_RETURN(FALSE); @@ -2211,12 +2206,10 @@ static int add_keyword_int(String *str, const char *keyword, longlong num) return err + str->append_longlong(num); } -static int add_partition_options(String *str, partition_element *p_elem) +static int add_server_part_options(String *str, partition_element *p_elem) { int err= 0; - if (p_elem->tablespace_name) - err+= add_keyword_string(str,"TABLESPACE", false, p_elem->tablespace_name); if (p_elem->nodegroup_id != UNDEF_NODEGROUP) err+= add_keyword_int(str,"NODEGROUP",(longlong)p_elem->nodegroup_id); if (p_elem->part_max_rows) @@ -2240,6 +2233,20 @@ static int add_partition_options(String *str, partition_element *p_elem) return err; } +static int add_engine_part_options(String *str, partition_element *p_elem) +{ + engine_option_value *opt= p_elem->option_list; + + for (; opt; opt= opt->next) + { + if (!opt->value.str) + continue; + if ((add_keyword_string(str, opt->name.str, opt->quoted_value, + opt->value.str))) + return 1; + } + return 0; +} /* Find the given field's Create_field object using name of field @@ -2463,7 +2470,7 @@ end: @retval != 0 Failure */ -static int add_key_with_algorithm(String *str, partition_info *part_info) +static int add_key_with_algorithm(String *str, const partition_info *part_info) { int err= 0; err+= str->append(STRING_WITH_LEN("KEY ")); @@ -2492,6 +2499,78 @@ char *generate_partition_syntax_for_frm(THD *thd, partition_info *part_info, return res; } + +/* + Generate the partition type syntax from the partition data structure. + + @return Operation status. + @retval 0 Success + @retval > 0 Failure + @retval -1 Fatal error +*/ + +int partition_info::gen_part_type(THD *thd, String *str) const +{ + int err= 0; + switch (part_type) + { + case RANGE_PARTITION: + err+= str->append(STRING_WITH_LEN("RANGE ")); + break; + case LIST_PARTITION: + err+= str->append(STRING_WITH_LEN("LIST ")); + break; + case HASH_PARTITION: + if (linear_hash_ind) + err+= str->append(STRING_WITH_LEN("LINEAR ")); + if (list_of_part_fields) + { + err+= add_key_with_algorithm(str, this); + err+= add_part_field_list(thd, str, part_field_list); + } + else + err+= str->append(STRING_WITH_LEN("HASH ")); + break; + case VERSIONING_PARTITION: + err+= str->append(STRING_WITH_LEN("SYSTEM_TIME ")); + break; + default: + DBUG_ASSERT(0); + /* We really shouldn't get here, no use in continuing from here */ + my_error(ER_OUT_OF_RESOURCES, MYF(ME_FATAL)); + return -1; + } + return err; +} + + +void part_type_error(THD *thd, partition_info *work_part_info, + const char *part_type, + partition_info *tab_part_info) +{ + StringBuffer<256> tab_part_type; + if (tab_part_info->gen_part_type(thd, &tab_part_type) < 0) + return; + tab_part_type.length(tab_part_type.length() - 1); + if (work_part_info) + { + DBUG_ASSERT(!part_type); + StringBuffer<256> work_part_type; + if (work_part_info->gen_part_type(thd, &work_part_type) < 0) + return; + work_part_type.length(work_part_type.length() - 1); + my_error(ER_PARTITION_WRONG_TYPE, MYF(0), work_part_type.c_ptr(), + tab_part_type.c_ptr()); + } + else + { + DBUG_ASSERT(part_type); + my_error(ER_PARTITION_WRONG_TYPE, MYF(0), part_type, + tab_part_type.c_ptr()); + } +} + + /* Generate the partition syntax from the partition data structure. Useful for support of generating defaults, SHOW CREATE TABLES @@ -2535,34 +2614,10 @@ char *generate_partition_syntax(THD *thd, partition_info *part_info, DBUG_ENTER("generate_partition_syntax"); err+= str.append(STRING_WITH_LEN(" PARTITION BY ")); - switch (part_info->part_type) - { - case RANGE_PARTITION: - err+= str.append(STRING_WITH_LEN("RANGE ")); - break; - case LIST_PARTITION: - err+= str.append(STRING_WITH_LEN("LIST ")); - break; - case HASH_PARTITION: - if (part_info->linear_hash_ind) - err+= str.append(STRING_WITH_LEN("LINEAR ")); - if (part_info->list_of_part_fields) - { - err+= add_key_with_algorithm(&str, part_info); - err+= add_part_field_list(thd, &str, part_info->part_field_list); - } - else - err+= str.append(STRING_WITH_LEN("HASH ")); - break; - case VERSIONING_PARTITION: - err+= str.append(STRING_WITH_LEN("SYSTEM_TIME ")); - break; - default: - DBUG_ASSERT(0); - /* We really shouldn't get here, no use in continuing from here */ - my_error(ER_OUT_OF_RESOURCES, MYF(ME_FATAL)); - DBUG_RETURN(NULL); - } + int err2= part_info->gen_part_type(thd, &str); + if (err2 < 0) + DBUG_RETURN(NULL); + err+= err2; if (part_info->part_type == VERSIONING_PARTITION) { Vers_part_info *vers_info= part_info->vers_info; @@ -2663,7 +2718,10 @@ char *generate_partition_syntax(THD *thd, partition_info *part_info, part_info->use_default_subpartitions) { if (show_partition_options) - err+= add_partition_options(&str, part_elem); + { + err+= add_server_part_options(&str, part_elem); + err+= add_engine_part_options(&str, part_elem); + } } else { @@ -2677,7 +2735,7 @@ char *generate_partition_syntax(THD *thd, partition_info *part_info, err+= append_identifier(thd, &str, part_elem->partition_name, strlen(part_elem->partition_name)); if (show_partition_options) - err+= add_partition_options(&str, part_elem); + err+= add_server_part_options(&str, part_elem); if (j != (num_subparts-1)) err+= str.append(STRING_WITH_LEN(",\n ")); else @@ -4714,8 +4772,6 @@ bool compare_partition_options(HA_CREATE_INFO *table_create_info, Note that there are not yet any engine supporting tablespace together with partitioning. TODO: when there are, add compare. */ - if (part_elem->tablespace_name || table_create_info->tablespace) - option_diffs[errors++]= "TABLESPACE"; if (part_elem->part_max_rows != table_create_info->max_rows) option_diffs[errors++]= "MAX_ROWS"; if (part_elem->part_min_rows != table_create_info->min_rows) @@ -4877,10 +4933,12 @@ uint prep_alter_part_table(THD *thd, TABLE *table, Alter_info *alter_info, if (alter_info->partition_flags & (ALTER_PARTITION_ADD | ALTER_PARTITION_DROP | + ALTER_PARTITION_CONVERT_OUT | ALTER_PARTITION_COALESCE | ALTER_PARTITION_REORGANIZE | ALTER_PARTITION_TABLE_REORG | - ALTER_PARTITION_REBUILD)) + ALTER_PARTITION_REBUILD | + ALTER_PARTITION_CONVERT_IN)) { /* You can't add column when we are doing alter related to partition @@ -5017,6 +5075,13 @@ uint prep_alter_part_table(THD *thd, TABLE *table, Alter_info *alter_info, if ((alter_info->partition_flags & ALTER_PARTITION_ADD) || (alter_info->partition_flags & ALTER_PARTITION_REORGANIZE)) { + if ((alter_info->partition_flags & ALTER_PARTITION_CONVERT_IN) && + !(tab_part_info->part_type == RANGE_PARTITION || + tab_part_info->part_type == LIST_PARTITION)) + { + my_error(ER_ONLY_ON_RANGE_LIST_PARTITION, MYF(0), "CONVERT TABLE TO"); + goto err; + } if (thd->work_part_info->part_type != tab_part_info->part_type) { if (thd->work_part_info->part_type == NOT_A_PARTITION) @@ -5052,7 +5117,7 @@ uint prep_alter_part_table(THD *thd, TABLE *table, Alter_info *alter_info, else if (thd->work_part_info->part_type == VERSIONING_PARTITION || tab_part_info->part_type == VERSIONING_PARTITION) { - my_error(ER_PARTITION_WRONG_TYPE, MYF(0), "SYSTEM_TIME"); + part_type_error(thd, thd->work_part_info, NULL, tab_part_info); } else { @@ -5372,8 +5437,12 @@ that are reorganised. tab_part_info->is_auto_partitioned= FALSE; } } - else if (alter_info->partition_flags & ALTER_PARTITION_DROP) + else if ((alter_info->partition_flags & ALTER_PARTITION_DROP) | + (alter_info->partition_flags & ALTER_PARTITION_CONVERT_OUT)) { + const char * const cmd= + (alter_info->partition_flags & ALTER_PARTITION_CONVERT_OUT) ? + "CONVERT" : "DROP"; /* Drop a partition from a range partition and list partitioning is always safe and can be made more or less immediate. It is necessary @@ -5402,7 +5471,7 @@ that are reorganised. if (!(tab_part_info->part_type == RANGE_PARTITION || tab_part_info->part_type == LIST_PARTITION)) { - my_error(ER_ONLY_ON_RANGE_LIST_PARTITION, MYF(0), "DROP"); + my_error(ER_ONLY_ON_RANGE_LIST_PARTITION, MYF(0), cmd); goto err; } if (num_parts_dropped >= tab_part_info->num_parts) @@ -5444,7 +5513,7 @@ that are reorganised. } while (++part_count < tab_part_info->num_parts); if (num_parts_found != num_parts_dropped) { - my_error(ER_DROP_PARTITION_NON_EXISTENT, MYF(0), "DROP"); + my_error(ER_PARTITION_DOES_NOT_EXIST, MYF(0)); goto err; } if (table->file->is_fk_defined_on_table_or_index(MAX_KEY)) @@ -5452,6 +5521,17 @@ that are reorganised. my_error(ER_ROW_IS_REFERENCED, MYF(0)); goto err; } + DBUG_ASSERT(!(alter_info->partition_flags & ALTER_PARTITION_CONVERT_OUT) || + num_parts_dropped == 1); + /* NOTE: num_parts is used in generate_partition_syntax() */ + tab_part_info->num_parts-= num_parts_dropped; + if ((alter_info->partition_flags & ALTER_PARTITION_CONVERT_OUT) && + tab_part_info->is_sub_partitioned()) + { + // TODO technically this can be converted to a *partitioned* table + my_error(ER_PARTITION_CONVERT_SUBPARTITIONED, MYF(0)); + goto err; + } } else if (alter_info->partition_flags & ALTER_PARTITION_REBUILD) { @@ -5459,7 +5539,7 @@ that are reorganised. tab_part_info->default_engine_type); if (set_part_state(alter_info, tab_part_info, PART_CHANGED)) { - my_error(ER_DROP_PARTITION_NON_EXISTENT, MYF(0), "REBUILD"); + my_error(ER_PARTITION_DOES_NOT_EXIST, MYF(0)); goto err; } if (!(*fast_alter_table)) @@ -5735,7 +5815,7 @@ the generated partition syntax in a correct manner. } while (++part_count < tab_part_info->num_parts); if (drop_count != num_parts_reorged) { - my_error(ER_DROP_PARTITION_NON_EXISTENT, MYF(0), "REORGANIZE"); + my_error(ER_PARTITION_DOES_NOT_EXIST, MYF(0)); goto err; } tab_part_info->num_parts= check_total_partitions; @@ -5795,7 +5875,7 @@ the generated partition syntax in a correct manner. goto err; } } - } // ADD, DROP, COALESCE, REORGANIZE, TABLE_REORG, REBUILD + } // ADD, DROP, COALESCE, REORGANIZE, TABLE_REORG, REBUILD, CONVERT else { /* @@ -6136,19 +6216,64 @@ static bool mysql_drop_partitions(ALTER_PARTITION_PARAM_TYPE *lpt) /* - Insert log entry into list + Convert partition to a table in an ALTER TABLE of partitions + SYNOPSIS - insert_part_info_log_entry_list() - log_entry + alter_partition_convert_out() + lpt Struct containing parameters + RETURN VALUES - NONE + TRUE Failure + FALSE Success + + DESCRIPTION + Rename partition table marked with PART_TO_BE_DROPPED into a separate table + under the name lpt->alter_ctx->(new_db, new_name). + + This is ddl-logged by write_log_convert_out_partition(). */ -static void insert_part_info_log_entry_list(partition_info *part_info, - DDL_LOG_MEMORY_ENTRY *log_entry) +static bool alter_partition_convert_out(ALTER_PARTITION_PARAM_TYPE *lpt) { - log_entry->next_active_log_entry= part_info->first_log_entry; - part_info->first_log_entry= log_entry; + partition_info *part_info= lpt->table->part_info; + THD *thd= lpt->thd; + int error; + handler *file= get_new_handler(NULL, thd->mem_root, part_info->default_engine_type); + + DBUG_ASSERT(lpt->thd->mdl_context.is_lock_owner(MDL_key::TABLE, + lpt->table->s->db.str, + lpt->table->s->table_name.str, + MDL_EXCLUSIVE)); + + char from_name[FN_REFLEN + 1], to_name[FN_REFLEN + 1]; + const char *path= lpt->table->s->path.str; + + build_table_filename(to_name, sizeof(to_name) - 1, lpt->alter_ctx->new_db.str, + lpt->alter_ctx->new_name.str, "", 0); + + for (const partition_element &e: part_info->partitions) + { + if (e.part_state != PART_TO_BE_DROPPED) + continue; + + if (unlikely((error= create_partition_name(from_name, sizeof(from_name), + path, e.partition_name, + NORMAL_PART_NAME, FALSE)))) + { + DBUG_ASSERT(thd->is_error()); + return true; + } + if (DBUG_IF("error_convert_partition_00") || + unlikely(error= file->ha_rename_table(from_name, to_name))) + { + my_error(ER_ERROR_ON_RENAME, MYF(0), from_name, to_name, my_errno); + lpt->table->file->print_error(error, MYF(0)); + return true; + } + break; + } + + return false; } @@ -6176,50 +6301,44 @@ static void release_part_info_log_entries(DDL_LOG_MEMORY_ENTRY *log_entry) /* - Log an delete/rename frm file + Log an rename frm file SYNOPSIS - write_log_replace_delete_frm() + write_log_replace_frm() lpt Struct for parameters next_entry Next reference to use in log record from_path Name to rename from to_path Name to rename to - replace_flag TRUE if replace, else delete RETURN VALUES TRUE Error FALSE Success DESCRIPTION - Support routine that writes a replace or delete of an frm file into the + Support routine that writes a replace of an frm file into the ddl log. It also inserts an entry that keeps track of used space into the partition info object */ -static bool write_log_replace_delete_frm(ALTER_PARTITION_PARAM_TYPE *lpt, - uint next_entry, - const char *from_path, - const char *to_path, - bool replace_flag) +bool write_log_replace_frm(ALTER_PARTITION_PARAM_TYPE *lpt, + uint next_entry, + const char *from_path, + const char *to_path) { DDL_LOG_ENTRY ddl_log_entry; DDL_LOG_MEMORY_ENTRY *log_entry; - DBUG_ENTER("write_log_replace_delete_frm"); + DBUG_ENTER("write_log_replace_frm"); bzero(&ddl_log_entry, sizeof(ddl_log_entry)); - if (replace_flag) - ddl_log_entry.action_type= DDL_LOG_REPLACE_ACTION; - else - ddl_log_entry.action_type= DDL_LOG_DELETE_ACTION; + ddl_log_entry.action_type= DDL_LOG_REPLACE_ACTION; ddl_log_entry.next_entry= next_entry; lex_string_set(&ddl_log_entry.handler_name, reg_ext); lex_string_set(&ddl_log_entry.name, to_path); + lex_string_set(&ddl_log_entry.from_name, from_path); - if (replace_flag) - lex_string_set(&ddl_log_entry.from_name, from_path); if (ddl_log_write_entry(&ddl_log_entry, &log_entry)) { - DBUG_RETURN(TRUE); + DBUG_RETURN(true); } - insert_part_info_log_entry_list(lpt->part_info, log_entry); - DBUG_RETURN(FALSE); + ddl_log_add_entry(lpt->part_info, log_entry); + DBUG_RETURN(false); } @@ -6298,7 +6417,7 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, *next_entry= log_entry->entry_pos; sub_elem->log_entry= log_entry; - insert_part_info_log_entry_list(part_info, log_entry); + ddl_log_add_entry(part_info, log_entry); } while (++j < num_subparts); } else @@ -6325,7 +6444,7 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, } *next_entry= log_entry->entry_pos; part_elem->log_entry= log_entry; - insert_part_info_log_entry_list(part_info, log_entry); + ddl_log_add_entry(part_info, log_entry); } } } while (++i < num_elements); @@ -6334,21 +6453,29 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, /* - Log dropped partitions + Log dropped or converted partitions SYNOPSIS - write_log_dropped_partitions() + log_drop_or_convert_action() lpt Struct containing parameters RETURN VALUES TRUE Error FALSE Success */ -static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, - uint *next_entry, - const char *path, - bool temp_list) +enum log_action_enum +{ + ACT_DROP = 0, + ACT_CONVERT_IN, + ACT_CONVERT_OUT +}; + +static bool log_drop_or_convert_action(ALTER_PARTITION_PARAM_TYPE *lpt, + uint *next_entry, const char *path, + const char *from_name, bool temp_list, + const log_action_enum convert_action) { DDL_LOG_ENTRY ddl_log_entry; + DBUG_ASSERT(convert_action == ACT_DROP || (from_name != NULL)); partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; char tmp_path[FN_REFLEN + 1]; @@ -6356,10 +6483,13 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, List_iterator<partition_element> temp_it(part_info->temp_partitions); uint num_temp_partitions= part_info->temp_partitions.elements; uint num_elements= part_info->partitions.elements; - DBUG_ENTER("write_log_dropped_partitions"); + DBUG_ENTER("log_drop_or_convert_action"); bzero(&ddl_log_entry, sizeof(ddl_log_entry)); - ddl_log_entry.action_type= DDL_LOG_DELETE_ACTION; + + ddl_log_entry.action_type= convert_action ? + DDL_LOG_RENAME_ACTION : + DDL_LOG_DELETE_ACTION; if (temp_list) num_elements= num_temp_partitions; while (num_elements--) @@ -6380,8 +6510,13 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, name_variant= TEMP_PART_NAME; else name_variant= NORMAL_PART_NAME; + DBUG_ASSERT(convert_action != ACT_CONVERT_IN || + part_elem->part_state == PART_TO_BE_ADDED); + DBUG_ASSERT(convert_action != ACT_CONVERT_OUT || + part_elem->part_state == PART_TO_BE_DROPPED); if (part_info->is_sub_partitioned()) { + DBUG_ASSERT(!convert_action); List_iterator<partition_element> sub_it(part_elem->subpartitions); uint num_subparts= part_info->num_subparts; uint j= 0; @@ -6403,7 +6538,7 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, } *next_entry= log_entry->entry_pos; sub_elem->log_entry= log_entry; - insert_part_info_log_entry_list(part_info, log_entry); + ddl_log_add_entry(part_info, log_entry); } while (++j < num_subparts); } else @@ -6415,14 +6550,25 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, part_elem->partition_name, name_variant, TRUE)) DBUG_RETURN(TRUE); - lex_string_set(&ddl_log_entry.name, tmp_path); + switch (convert_action) + { + case ACT_CONVERT_OUT: + ddl_log_entry.from_name= { from_name, strlen(from_name) }; + /* fall through */ + case ACT_DROP: + ddl_log_entry.name= { tmp_path, strlen(tmp_path) }; + break; + case ACT_CONVERT_IN: + ddl_log_entry.name= { from_name, strlen(from_name) }; + ddl_log_entry.from_name= { tmp_path, strlen(tmp_path) }; + } if (ddl_log_write_entry(&ddl_log_entry, &log_entry)) { DBUG_RETURN(TRUE); } *next_entry= log_entry->entry_pos; part_elem->log_entry= log_entry; - insert_part_info_log_entry_list(part_info, log_entry); + ddl_log_add_entry(part_info, log_entry); } } } @@ -6430,21 +6576,37 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, } -/* - Set execute log entry in ddl log for this partitioned table - SYNOPSIS - set_part_info_exec_log_entry() - part_info Partition info object - exec_log_entry Log entry - RETURN VALUES - NONE -*/ - -static void set_part_info_exec_log_entry(partition_info *part_info, - DDL_LOG_MEMORY_ENTRY *exec_log_entry) +inline +static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt, + uint *next_entry, const char *path, + bool temp_list) { - part_info->exec_log_entry= exec_log_entry; - exec_log_entry->next_active_log_entry= NULL; + return log_drop_or_convert_action(lpt, next_entry, path, NULL, temp_list, + ACT_DROP); +} + +inline +static bool write_log_convert_partition(ALTER_PARTITION_PARAM_TYPE *lpt, + uint *next_entry, const char *path) +{ + char other_table[FN_REFLEN + 1]; + const ulong f= lpt->alter_info->partition_flags; + DBUG_ASSERT((f & ALTER_PARTITION_CONVERT_IN) || (f & ALTER_PARTITION_CONVERT_OUT)); + const log_action_enum convert_action= (f & ALTER_PARTITION_CONVERT_IN) + ? ACT_CONVERT_IN : ACT_CONVERT_OUT; + build_table_filename(other_table, sizeof(other_table) - 1, lpt->alter_ctx->new_db.str, + lpt->alter_ctx->new_name.str, "", 0); + DDL_LOG_MEMORY_ENTRY *main_entry= lpt->part_info->main_entry; + bool res= log_drop_or_convert_action(lpt, next_entry, path, other_table, + false, convert_action); + /* + NOTE: main_entry is "drop shadow frm", we have to keep it like this + because partitioning crash-safety disables it at install shadow FRM phase. + This is needed to avoid spurious drop action when the shadow frm is replaced + by the backup frm and there is nothing to drop. + */ + lpt->part_info->main_entry= main_entry; + return res; } @@ -6452,10 +6614,9 @@ static void set_part_info_exec_log_entry(partition_info *part_info, Write the log entry to ensure that the shadow frm file is removed at crash. SYNOPSIS - write_log_drop_shadow_frm() + write_log_drop_frm() lpt Struct containing parameters - install_frm Should we log action to install shadow frm or should - the action be to remove the shadow frm file. + RETURN VALUES TRUE Error FALSE Success @@ -6464,36 +6625,53 @@ static void set_part_info_exec_log_entry(partition_info *part_info, file and its corresponding handler file. */ -static bool write_log_drop_shadow_frm(ALTER_PARTITION_PARAM_TYPE *lpt) +static bool write_log_drop_frm(ALTER_PARTITION_PARAM_TYPE *lpt, + DDL_LOG_STATE *drop_chain) { - partition_info *part_info= lpt->part_info; - DDL_LOG_MEMORY_ENTRY *log_entry; - DDL_LOG_MEMORY_ENTRY *exec_log_entry= NULL; - char shadow_path[FN_REFLEN + 1]; - DBUG_ENTER("write_log_drop_shadow_frm"); + char path[FN_REFLEN + 1]; + DBUG_ENTER("write_log_drop_frm"); + const DDL_LOG_STATE *main_chain= lpt->part_info; + const bool drop_backup= (drop_chain != main_chain); - build_table_shadow_filename(shadow_path, sizeof(shadow_path) - 1, lpt); + build_table_shadow_filename(path, sizeof(path) - 1, lpt, drop_backup); mysql_mutex_lock(&LOCK_gdl); - if (write_log_replace_delete_frm(lpt, 0UL, NULL, - (const char*)shadow_path, FALSE)) + if (ddl_log_delete_frm(drop_chain, (const char*)path)) goto error; - log_entry= part_info->first_log_entry; - if (ddl_log_write_execute_entry(log_entry->entry_pos, - &exec_log_entry)) + + if (drop_backup && (lpt->alter_info->partition_flags & ALTER_PARTITION_CONVERT_IN)) + { + TABLE_LIST *table_from= lpt->table_list->next_local; + build_table_filename(path, sizeof(path) - 1, table_from->db.str, + table_from->table_name.str, "", 0); + + if (ddl_log_delete_frm(drop_chain, (const char*) path)) + goto error; + } + + if (ddl_log_write_execute_entry(drop_chain->list->entry_pos, + drop_backup ? + main_chain->execute_entry->entry_pos : 0, + &drop_chain->execute_entry)) goto error; mysql_mutex_unlock(&LOCK_gdl); - set_part_info_exec_log_entry(part_info, exec_log_entry); DBUG_RETURN(FALSE); error: - release_part_info_log_entries(part_info->first_log_entry); + release_part_info_log_entries(drop_chain->list); mysql_mutex_unlock(&LOCK_gdl); - part_info->first_log_entry= NULL; + drop_chain->list= NULL; my_error(ER_DDL_LOG_ERROR, MYF(0)); DBUG_RETURN(TRUE); } +static inline +bool write_log_drop_shadow_frm(ALTER_PARTITION_PARAM_TYPE *lpt) +{ + return write_log_drop_frm(lpt, lpt->part_info); +} + + /* Log renaming of shadow frm to real frm name and dropping of old frm SYNOPSIS @@ -6511,20 +6689,20 @@ static bool write_log_rename_frm(ALTER_PARTITION_PARAM_TYPE *lpt) { partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; - DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->exec_log_entry; + DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->execute_entry; char path[FN_REFLEN + 1]; char shadow_path[FN_REFLEN + 1]; - DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->first_log_entry; + DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->list; DBUG_ENTER("write_log_rename_frm"); - part_info->first_log_entry= NULL; + part_info->list= NULL; build_table_filename(path, sizeof(path) - 1, lpt->db.str, lpt->table_name.str, "", 0); build_table_shadow_filename(shadow_path, sizeof(shadow_path) - 1, lpt); mysql_mutex_lock(&LOCK_gdl); - if (write_log_replace_delete_frm(lpt, 0UL, shadow_path, path, TRUE)) + if (write_log_replace_frm(lpt, 0UL, shadow_path, path)) goto error; - log_entry= part_info->first_log_entry; - part_info->frm_log_entry= log_entry; + log_entry= part_info->list; + part_info->main_entry= log_entry; if (ddl_log_write_execute_entry(log_entry->entry_pos, &exec_log_entry)) goto error; @@ -6533,10 +6711,10 @@ static bool write_log_rename_frm(ALTER_PARTITION_PARAM_TYPE *lpt) DBUG_RETURN(FALSE); error: - release_part_info_log_entries(part_info->first_log_entry); + release_part_info_log_entries(part_info->list); mysql_mutex_unlock(&LOCK_gdl); - part_info->first_log_entry= old_first_log_entry; - part_info->frm_log_entry= NULL; + part_info->list= old_first_log_entry; + part_info->main_entry= NULL; my_error(ER_DDL_LOG_ERROR, MYF(0)); DBUG_RETURN(TRUE); } @@ -6561,25 +6739,25 @@ static bool write_log_drop_partition(ALTER_PARTITION_PARAM_TYPE *lpt) { partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; - DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->exec_log_entry; + DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->execute_entry; char tmp_path[FN_REFLEN + 1]; char path[FN_REFLEN + 1]; uint next_entry= 0; - DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->first_log_entry; + DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->list; DBUG_ENTER("write_log_drop_partition"); - part_info->first_log_entry= NULL; + part_info->list= NULL; build_table_filename(path, sizeof(path) - 1, lpt->db.str, lpt->table_name.str, "", 0); build_table_shadow_filename(tmp_path, sizeof(tmp_path) - 1, lpt); mysql_mutex_lock(&LOCK_gdl); if (write_log_dropped_partitions(lpt, &next_entry, (const char*)path, FALSE)) goto error; - if (write_log_replace_delete_frm(lpt, next_entry, (const char*)tmp_path, - (const char*)path, TRUE)) + if (write_log_replace_frm(lpt, next_entry, (const char*)tmp_path, + (const char*)path)) goto error; - log_entry= part_info->first_log_entry; - part_info->frm_log_entry= log_entry; + log_entry= part_info->list; + part_info->main_entry= log_entry; if (ddl_log_write_execute_entry(log_entry->entry_pos, &exec_log_entry)) goto error; @@ -6588,15 +6766,44 @@ static bool write_log_drop_partition(ALTER_PARTITION_PARAM_TYPE *lpt) DBUG_RETURN(FALSE); error: - release_part_info_log_entries(part_info->first_log_entry); + release_part_info_log_entries(part_info->list); mysql_mutex_unlock(&LOCK_gdl); - part_info->first_log_entry= old_first_log_entry; - part_info->frm_log_entry= NULL; + part_info->list= old_first_log_entry; + part_info->main_entry= NULL; my_error(ER_DDL_LOG_ERROR, MYF(0)); DBUG_RETURN(TRUE); } +static bool write_log_convert_partition(ALTER_PARTITION_PARAM_TYPE *lpt) +{ + partition_info *part_info= lpt->part_info; + char tmp_path[FN_REFLEN + 1]; + char path[FN_REFLEN + 1]; + uint next_entry= part_info->list ? part_info->list->entry_pos : 0; + + build_table_filename(path, sizeof(path) - 1, lpt->db.str, lpt->table_name.str, "", 0); + build_table_shadow_filename(tmp_path, sizeof(tmp_path) - 1, lpt); + + mysql_mutex_lock(&LOCK_gdl); + + if (write_log_convert_partition(lpt, &next_entry, (const char*)path)) + goto error; + DBUG_ASSERT(next_entry == part_info->list->entry_pos); + if (ddl_log_write_execute_entry(part_info->list->entry_pos, + &part_info->execute_entry)) + goto error; + mysql_mutex_unlock(&LOCK_gdl); + return false; + +error: + mysql_mutex_unlock(&LOCK_gdl); + part_info->main_entry= NULL; + my_error(ER_DDL_LOG_ERROR, MYF(0)); + return true; +} + + /* Write the log entries to ensure that the add partition command is not executed at all if a crash before it has completed @@ -6618,11 +6825,10 @@ static bool write_log_add_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt) { partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; - DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->exec_log_entry; char tmp_path[FN_REFLEN + 1]; char path[FN_REFLEN + 1]; uint next_entry= 0; - DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->first_log_entry; + DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->list; /* write_log_drop_shadow_frm(lpt) must have been run first */ DBUG_ASSERT(old_first_log_entry); DBUG_ENTER("write_log_add_change_partition"); @@ -6637,20 +6843,18 @@ static bool write_log_add_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt) if (write_log_dropped_partitions(lpt, &next_entry, (const char*)path, FALSE)) goto error; - log_entry= part_info->first_log_entry; + log_entry= part_info->list; if (ddl_log_write_execute_entry(log_entry->entry_pos, - /* Reuse the old execute ddl_log_entry */ - &exec_log_entry)) + &part_info->execute_entry)) goto error; mysql_mutex_unlock(&LOCK_gdl); - set_part_info_exec_log_entry(part_info, exec_log_entry); DBUG_RETURN(FALSE); error: - release_part_info_log_entries(part_info->first_log_entry); + release_part_info_log_entries(part_info->list); mysql_mutex_unlock(&LOCK_gdl); - part_info->first_log_entry= old_first_log_entry; + part_info->list= old_first_log_entry; my_error(ER_DDL_LOG_ERROR, MYF(0)); DBUG_RETURN(TRUE); } @@ -6682,10 +6886,10 @@ static bool write_log_final_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt) { partition_info *part_info= lpt->part_info; DDL_LOG_MEMORY_ENTRY *log_entry; - DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->exec_log_entry; + DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->execute_entry; char path[FN_REFLEN + 1]; char shadow_path[FN_REFLEN + 1]; - DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->first_log_entry; + DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->list; uint next_entry= 0; DBUG_ENTER("write_log_final_change_partition"); @@ -6693,7 +6897,7 @@ static bool write_log_final_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt) Do not link any previous log entry. Replace the revert operations with forced retry operations. */ - part_info->first_log_entry= NULL; + part_info->list= NULL; build_table_filename(path, sizeof(path) - 1, lpt->db.str, lpt->table_name.str, "", 0); build_table_shadow_filename(shadow_path, sizeof(shadow_path) - 1, lpt); mysql_mutex_lock(&LOCK_gdl); @@ -6703,10 +6907,10 @@ static bool write_log_final_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt) lpt->alter_info->partition_flags & ALTER_PARTITION_REORGANIZE)) goto error; - if (write_log_replace_delete_frm(lpt, next_entry, shadow_path, path, TRUE)) + if (write_log_replace_frm(lpt, next_entry, shadow_path, path)) goto error; - log_entry= part_info->first_log_entry; - part_info->frm_log_entry= log_entry; + log_entry= part_info->list; + part_info->main_entry= log_entry; /* Overwrite the revert execute log entry with this retry execute entry */ if (ddl_log_write_execute_entry(log_entry->entry_pos, &exec_log_entry)) @@ -6716,10 +6920,10 @@ static bool write_log_final_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt) DBUG_RETURN(FALSE); error: - release_part_info_log_entries(part_info->first_log_entry); + release_part_info_log_entries(part_info->list); mysql_mutex_unlock(&LOCK_gdl); - part_info->first_log_entry= old_first_log_entry; - part_info->frm_log_entry= NULL; + part_info->list= old_first_log_entry; + part_info->main_entry= NULL; my_error(ER_DDL_LOG_ERROR, MYF(0)); DBUG_RETURN(TRUE); } @@ -6736,11 +6940,15 @@ error: FALSE Success */ +/* + TODO: Partitioning atomic DDL refactoring: this should be replaced with + ddl_log_complete(). +*/ static void write_log_completed(ALTER_PARTITION_PARAM_TYPE *lpt, bool dont_crash) { partition_info *part_info= lpt->part_info; - DDL_LOG_MEMORY_ENTRY *log_entry= part_info->exec_log_entry; + DDL_LOG_MEMORY_ENTRY *log_entry= part_info->execute_entry; DBUG_ENTER("write_log_completed"); DBUG_ASSERT(log_entry); @@ -6756,11 +6964,11 @@ static void write_log_completed(ALTER_PARTITION_PARAM_TYPE *lpt, */ ; } - release_part_info_log_entries(part_info->first_log_entry); - release_part_info_log_entries(part_info->exec_log_entry); + release_part_info_log_entries(part_info->list); + release_part_info_log_entries(part_info->execute_entry); mysql_mutex_unlock(&LOCK_gdl); - part_info->exec_log_entry= NULL; - part_info->first_log_entry= NULL; + part_info->execute_entry= NULL; + part_info->list= NULL; DBUG_VOID_RETURN; } @@ -6774,14 +6982,18 @@ static void write_log_completed(ALTER_PARTITION_PARAM_TYPE *lpt, NONE */ +/* + TODO: Partitioning atomic DDL refactoring: this should be replaced with + ddl_log_release_entries(). +*/ static void release_log_entries(partition_info *part_info) { mysql_mutex_lock(&LOCK_gdl); - release_part_info_log_entries(part_info->first_log_entry); - release_part_info_log_entries(part_info->exec_log_entry); + release_part_info_log_entries(part_info->list); + release_part_info_log_entries(part_info->execute_entry); mysql_mutex_unlock(&LOCK_gdl); - part_info->first_log_entry= NULL; - part_info->exec_log_entry= NULL; + part_info->list= NULL; + part_info->execute_entry= NULL; } @@ -6864,10 +7076,15 @@ static int alter_close_table(ALTER_PARTITION_PARAM_TYPE *lpt) @param close_table Table is still open, close it before reverting */ +/* + TODO: Partitioning atomic DDL refactoring: this should be replaced with + correct combination of ddl_log_revert() / ddl_log_complete() +*/ static void handle_alter_part_error(ALTER_PARTITION_PARAM_TYPE *lpt, bool action_completed, bool drop_partition, - bool frm_install) + bool frm_install, + bool reopen) { THD *thd= lpt->thd; partition_info *part_info= lpt->part_info->get_clone(thd); @@ -6911,8 +7128,11 @@ static void handle_alter_part_error(ALTER_PARTITION_PARAM_TYPE *lpt, close_all_tables_for_name(thd, table->s, HA_EXTRA_NOT_USED, NULL); } - if (part_info->first_log_entry && - ddl_log_execute_entry(thd, part_info->first_log_entry->entry_pos)) + if (!reopen) + DBUG_VOID_RETURN; + + if (part_info->list && + ddl_log_execute_entry(thd, part_info->list->entry_pos)) { /* We couldn't recover from error, most likely manual interaction @@ -7074,6 +7294,64 @@ bool log_partition_alter_to_ddl_log(ALTER_PARTITION_PARAM_TYPE *lpt) } +extern bool alter_partition_convert_in(ALTER_PARTITION_PARAM_TYPE *lpt); + +/** + Check that definition of source table fits definition of partition being + added and every row stored in the table conforms partition's expression. + + @param lpt Structure containing parameters required for checking + @param[in,out] part_file_name_buf Buffer for storing a partition name + @param part_file_name_buf_sz Size of buffer for storing a partition name + @param part_file_name_len Length of partition prefix stored in the buffer + on invocation of function + + @return false on success, true on error +*/ + +static bool check_table_data(ALTER_PARTITION_PARAM_TYPE *lpt) +{ + /* + TODO: if destination is partitioned by range(X) and source is indexed by X + then just get min(X) and max(X) from index. + */ + THD *thd= lpt->thd; + TABLE *table_to= lpt->table_list->table; + TABLE *table_from= lpt->table_list->next_local->table; + + DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, + table_to->s->db.str, + table_to->s->table_name.str, + MDL_EXCLUSIVE)); + + DBUG_ASSERT(thd->mdl_context.is_lock_owner(MDL_key::TABLE, + table_from->s->db.str, + table_from->s->table_name.str, + MDL_EXCLUSIVE)); + + uint32 new_part_id; + partition_element *part_elem; + const char* partition_name= thd->lex->part_info->curr_part_elem->partition_name; + part_elem= table_to->part_info->get_part_elem(partition_name, + nullptr, 0, &new_part_id); + if (unlikely(!part_elem)) + return true; + + if (unlikely(new_part_id == NOT_A_PARTITION_ID)) + { + DBUG_ASSERT(table_to->part_info->is_sub_partitioned()); + my_error(ER_PARTITION_INSTEAD_OF_SUBPARTITION, MYF(0)); + return true; + } + + if (verify_data_with_partition(table_from, table_to, new_part_id)) + { + return true; + } + + return false; +} + /** Actually perform the change requested by ALTER TABLE of partitions @@ -7098,11 +7376,23 @@ bool log_partition_alter_to_ddl_log(ALTER_PARTITION_PARAM_TYPE *lpt) uint fast_alter_partition_table(THD *thd, TABLE *table, Alter_info *alter_info, + Alter_table_ctx *alter_ctx, HA_CREATE_INFO *create_info, - TABLE_LIST *table_list, - const LEX_CSTRING *db, - const LEX_CSTRING *table_name) + TABLE_LIST *table_list) { + /* + TODO: Partitioning atomic DDL refactoring. + + DDL log chain state is stored in partition_info: + + struct st_ddl_log_memory_entry *first_log_entry; + struct st_ddl_log_memory_entry *exec_log_entry; + struct st_ddl_log_memory_entry *frm_log_entry; + + Make it stored and used in DDL_LOG_STATE like it was done in MDEV-17567. + This requires mysql_write_frm() refactoring (see comment there). + */ + /* Set-up struct used to write frm files */ partition_info *part_info; ALTER_PARTITION_PARAM_TYPE lpt_obj; @@ -7120,13 +7410,14 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, lpt->table_list= table_list; lpt->part_info= part_info; lpt->alter_info= alter_info; + lpt->alter_ctx= alter_ctx; lpt->create_info= create_info; lpt->db_options= create_info->table_options_with_row_type(); lpt->table= table; lpt->key_info_buffer= 0; lpt->key_count= 0; - lpt->db= *db; - lpt->table_name= *table_name; + lpt->db= alter_ctx->db; + lpt->table_name= alter_ctx->table_name; lpt->org_tabledef_version= table->s->tabledef_version; lpt->copied= 0; lpt->deleted= 0; @@ -7246,49 +7537,138 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, to test if recovery is properly done. */ if (write_log_drop_shadow_frm(lpt) || - ERROR_INJECT_CRASH("crash_drop_partition_1") || - ERROR_INJECT_ERROR("fail_drop_partition_1") || + ERROR_INJECT("drop_partition_1") || mysql_write_frm(lpt, WFRM_WRITE_SHADOW) || - ERROR_INJECT_CRASH("crash_drop_partition_2") || - ERROR_INJECT_ERROR("fail_drop_partition_2") || + ERROR_INJECT("drop_partition_2") || wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED) || - ERROR_INJECT_CRASH("crash_drop_partition_3") || - ERROR_INJECT_ERROR("fail_drop_partition_3") || + ERROR_INJECT("drop_partition_3") || write_log_drop_partition(lpt) || (action_completed= TRUE, FALSE) || - ERROR_INJECT_CRASH("crash_drop_partition_4") || - ERROR_INJECT_ERROR("fail_drop_partition_4") || + ERROR_INJECT("drop_partition_4") || alter_close_table(lpt) || - ERROR_INJECT_CRASH("crash_drop_partition_5") || - ERROR_INJECT_ERROR("fail_drop_partition_5") || - ERROR_INJECT_CRASH("crash_drop_partition_6") || - ERROR_INJECT_ERROR("fail_drop_partition_6") || + ERROR_INJECT("drop_partition_5") || + ERROR_INJECT("drop_partition_6") || (frm_install= TRUE, FALSE) || mysql_write_frm(lpt, WFRM_INSTALL_SHADOW) || log_partition_alter_to_ddl_log(lpt) || (frm_install= FALSE, FALSE) || - ERROR_INJECT_CRASH("crash_drop_partition_7") || - ERROR_INJECT_ERROR("fail_drop_partition_7") || + ERROR_INJECT("drop_partition_7") || mysql_drop_partitions(lpt) || - ERROR_INJECT_CRASH("crash_drop_partition_8") || - ERROR_INJECT_ERROR("fail_drop_partition_8") || + ERROR_INJECT("drop_partition_8") || (write_log_completed(lpt, FALSE), FALSE) || ((!thd->lex->no_write_to_binlog) && (write_bin_log(thd, FALSE, thd->query(), thd->query_length()), FALSE)) || - ERROR_INJECT_CRASH("crash_drop_partition_9") || - ERROR_INJECT_ERROR("fail_drop_partition_9")) + ERROR_INJECT("drop_partition_9")) { - handle_alter_part_error(lpt, action_completed, TRUE, frm_install); + handle_alter_part_error(lpt, action_completed, TRUE, frm_install, true); goto err; } if (alter_partition_lock_handling(lpt)) goto err; } + else if (alter_info->partition_flags & ALTER_PARTITION_CONVERT_OUT) + { + DDL_LOG_STATE chain_drop_backup; + bzero(&chain_drop_backup, sizeof(chain_drop_backup)); + + if (mysql_write_frm(lpt, WFRM_WRITE_CONVERTED_TO) || + ERROR_INJECT("convert_partition_1") || + write_log_drop_shadow_frm(lpt) || + ERROR_INJECT("convert_partition_2") || + mysql_write_frm(lpt, WFRM_WRITE_SHADOW) || + ERROR_INJECT("convert_partition_3") || + wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED) || + ERROR_INJECT("convert_partition_4") || + write_log_convert_partition(lpt) || + ERROR_INJECT("convert_partition_5") || + alter_close_table(lpt) || + ERROR_INJECT("convert_partition_6") || + alter_partition_convert_out(lpt) || + ERROR_INJECT("convert_partition_7") || + write_log_drop_frm(lpt, &chain_drop_backup) || + mysql_write_frm(lpt, WFRM_INSTALL_SHADOW|WFRM_BACKUP_ORIGINAL) || + log_partition_alter_to_ddl_log(lpt) || + ERROR_INJECT("convert_partition_8") || + ((!thd->lex->no_write_to_binlog) && + ((thd->binlog_xid= thd->query_id), + ddl_log_update_xid(lpt->part_info, thd->binlog_xid), + write_bin_log(thd, false, thd->query(), thd->query_length()), + (thd->binlog_xid= 0))) || + ERROR_INJECT("convert_partition_9")) + { + DDL_LOG_STATE main_state= *lpt->part_info; + handle_alter_part_error(lpt, true, true, false, false); + ddl_log_complete(&chain_drop_backup); + (void) ddl_log_revert(thd, &main_state); + if (thd->locked_tables_mode) + thd->locked_tables_list.reopen_tables(thd, false); + goto err; + } + ddl_log_complete(lpt->part_info); + ERROR_INJECT("convert_partition_10"); + (void) ddl_log_revert(thd, &chain_drop_backup); + if (alter_partition_lock_handling(lpt) || + ERROR_INJECT("convert_partition_11")) + goto err; + } + else if ((alter_info->partition_flags & ALTER_PARTITION_CONVERT_IN)) + { + DDL_LOG_STATE chain_drop_backup; + bzero(&chain_drop_backup, sizeof(chain_drop_backup)); + TABLE *table_from= table_list->next_local->table; + + if (wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED) || + wait_while_table_is_used(thd, table_from, HA_EXTRA_PREPARE_FOR_RENAME) || + ERROR_INJECT("convert_partition_1") || + compare_table_with_partition(thd, table_from, table, NULL, 0) || + ERROR_INJECT("convert_partition_2") || + check_table_data(lpt)) + goto err; + + if (write_log_drop_shadow_frm(lpt) || + ERROR_INJECT("convert_partition_3") || + mysql_write_frm(lpt, WFRM_WRITE_SHADOW) || + ERROR_INJECT("convert_partition_4") || + alter_close_table(lpt) || + ERROR_INJECT("convert_partition_5") || + write_log_convert_partition(lpt) || + ERROR_INJECT("convert_partition_6") || + alter_partition_convert_in(lpt) || + ERROR_INJECT("convert_partition_7") || + (frm_install= true, false) || + write_log_drop_frm(lpt, &chain_drop_backup) || + mysql_write_frm(lpt, WFRM_INSTALL_SHADOW|WFRM_BACKUP_ORIGINAL) || + log_partition_alter_to_ddl_log(lpt) || + (frm_install= false, false) || + ERROR_INJECT("convert_partition_8") || + ((!thd->lex->no_write_to_binlog) && + ((thd->binlog_xid= thd->query_id), + ddl_log_update_xid(lpt->part_info, thd->binlog_xid), + write_bin_log(thd, false, thd->query(), thd->query_length()), + (thd->binlog_xid= 0))) || + ERROR_INJECT("convert_partition_9")) + { + DDL_LOG_STATE main_state= *lpt->part_info; + handle_alter_part_error(lpt, true, true, false, false); + ddl_log_complete(&chain_drop_backup); + (void) ddl_log_revert(thd, &main_state); + if (thd->locked_tables_mode) + thd->locked_tables_list.reopen_tables(thd, false); + goto err; + } + ddl_log_complete(lpt->part_info); + ERROR_INJECT("convert_partition_10"); + (void) ddl_log_revert(thd, &chain_drop_backup); + if (alter_partition_lock_handling(lpt) || + ERROR_INJECT("convert_partition_11")) + goto err; + } else if ((alter_info->partition_flags & ALTER_PARTITION_ADD) && (part_info->part_type == RANGE_PARTITION || part_info->part_type == LIST_PARTITION)) { + DBUG_ASSERT(!(alter_info->partition_flags & ALTER_PARTITION_CONVERT_IN)); /* ADD RANGE/LIST PARTITIONS In this case there are no tuples removed and no tuples are added. @@ -7320,43 +7700,33 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, 12)Complete query */ if (write_log_drop_shadow_frm(lpt) || - ERROR_INJECT_CRASH("crash_add_partition_1") || - ERROR_INJECT_ERROR("fail_add_partition_1") || + ERROR_INJECT("add_partition_1") || mysql_write_frm(lpt, WFRM_WRITE_SHADOW) || - ERROR_INJECT_CRASH("crash_add_partition_2") || - ERROR_INJECT_ERROR("fail_add_partition_2") || + ERROR_INJECT("add_partition_2") || wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED) || - ERROR_INJECT_CRASH("crash_add_partition_3") || - ERROR_INJECT_ERROR("fail_add_partition_3") || + ERROR_INJECT("add_partition_3") || write_log_add_change_partition(lpt) || - ERROR_INJECT_CRASH("crash_add_partition_4") || - ERROR_INJECT_ERROR("fail_add_partition_4") || + ERROR_INJECT("add_partition_4") || mysql_change_partitions(lpt) || - ERROR_INJECT_CRASH("crash_add_partition_5") || - ERROR_INJECT_ERROR("fail_add_partition_5") || + ERROR_INJECT("add_partition_5") || alter_close_table(lpt) || - ERROR_INJECT_CRASH("crash_add_partition_6") || - ERROR_INJECT_ERROR("fail_add_partition_6") || - ERROR_INJECT_CRASH("crash_add_partition_7") || - ERROR_INJECT_ERROR("fail_add_partition_7") || + ERROR_INJECT("add_partition_6") || + ERROR_INJECT("add_partition_7") || write_log_rename_frm(lpt) || (action_completed= TRUE, FALSE) || - ERROR_INJECT_CRASH("crash_add_partition_8") || - ERROR_INJECT_ERROR("fail_add_partition_8") || + ERROR_INJECT("add_partition_8") || (frm_install= TRUE, FALSE) || mysql_write_frm(lpt, WFRM_INSTALL_SHADOW) || log_partition_alter_to_ddl_log(lpt) || (frm_install= FALSE, FALSE) || - ERROR_INJECT_CRASH("crash_add_partition_9") || - ERROR_INJECT_ERROR("fail_add_partition_9") || + ERROR_INJECT("add_partition_9") || (write_log_completed(lpt, FALSE), FALSE) || ((!thd->lex->no_write_to_binlog) && (write_bin_log(thd, FALSE, thd->query(), thd->query_length()), FALSE)) || - ERROR_INJECT_CRASH("crash_add_partition_10") || - ERROR_INJECT_ERROR("fail_add_partition_10")) + ERROR_INJECT("add_partition_10")) { - handle_alter_part_error(lpt, action_completed, FALSE, frm_install); + handle_alter_part_error(lpt, action_completed, FALSE, frm_install, true); goto err; } if (alter_partition_lock_handling(lpt)) @@ -7419,49 +7789,37 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, 13) Complete query. */ if (write_log_drop_shadow_frm(lpt) || - ERROR_INJECT_CRASH("crash_change_partition_1") || - ERROR_INJECT_ERROR("fail_change_partition_1") || + ERROR_INJECT("change_partition_1") || mysql_write_frm(lpt, WFRM_WRITE_SHADOW) || - ERROR_INJECT_CRASH("crash_change_partition_2") || - ERROR_INJECT_ERROR("fail_change_partition_2") || + ERROR_INJECT("change_partition_2") || write_log_add_change_partition(lpt) || - ERROR_INJECT_CRASH("crash_change_partition_3") || - ERROR_INJECT_ERROR("fail_change_partition_3") || + ERROR_INJECT("change_partition_3") || mysql_change_partitions(lpt) || - ERROR_INJECT_CRASH("crash_change_partition_4") || - ERROR_INJECT_ERROR("fail_change_partition_4") || + ERROR_INJECT("change_partition_4") || wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED) || - ERROR_INJECT_CRASH("crash_change_partition_5") || - ERROR_INJECT_ERROR("fail_change_partition_5") || + ERROR_INJECT("change_partition_5") || alter_close_table(lpt) || - ERROR_INJECT_CRASH("crash_change_partition_6") || - ERROR_INJECT_ERROR("fail_change_partition_6") || + ERROR_INJECT("change_partition_6") || write_log_final_change_partition(lpt) || (action_completed= TRUE, FALSE) || - ERROR_INJECT_CRASH("crash_change_partition_7") || - ERROR_INJECT_ERROR("fail_change_partition_7") || - ERROR_INJECT_CRASH("crash_change_partition_8") || - ERROR_INJECT_ERROR("fail_change_partition_8") || + ERROR_INJECT("change_partition_7") || + ERROR_INJECT("change_partition_8") || ((frm_install= TRUE), FALSE) || mysql_write_frm(lpt, WFRM_INSTALL_SHADOW) || log_partition_alter_to_ddl_log(lpt) || (frm_install= FALSE, FALSE) || - ERROR_INJECT_CRASH("crash_change_partition_9") || - ERROR_INJECT_ERROR("fail_change_partition_9") || + ERROR_INJECT("change_partition_9") || mysql_drop_partitions(lpt) || - ERROR_INJECT_CRASH("crash_change_partition_10") || - ERROR_INJECT_ERROR("fail_change_partition_10") || + ERROR_INJECT("change_partition_10") || mysql_rename_partitions(lpt) || - ERROR_INJECT_CRASH("crash_change_partition_11") || - ERROR_INJECT_ERROR("fail_change_partition_11") || + ERROR_INJECT("change_partition_11") || (write_log_completed(lpt, FALSE), FALSE) || ((!thd->lex->no_write_to_binlog) && (write_bin_log(thd, FALSE, thd->query(), thd->query_length()), FALSE)) || - ERROR_INJECT_CRASH("crash_change_partition_12") || - ERROR_INJECT_ERROR("fail_change_partition_12")) + ERROR_INJECT("change_partition_12")) { - handle_alter_part_error(lpt, action_completed, FALSE, frm_install); + handle_alter_part_error(lpt, action_completed, FALSE, frm_install, true); goto err; } if (alter_partition_lock_handling(lpt)) diff --git a/sql/sql_partition.h b/sql/sql_partition.h index 57e6d0600ed..a90eaae0bae 100644 --- a/sql/sql_partition.h +++ b/sql/sql_partition.h @@ -55,6 +55,7 @@ typedef struct st_lock_param_type THD *thd; HA_CREATE_INFO *create_info; Alter_info *alter_info; + Alter_table_ctx *alter_ctx; TABLE *table; KEY *key_info_buffer; LEX_CSTRING db; @@ -64,6 +65,7 @@ typedef struct st_lock_param_type uint key_count; uint db_options; size_t pack_frm_len; + // TODO: remove duplicate data: part_info can be accessed via table->part_info partition_info *part_info; } ALTER_PARTITION_PARAM_TYPE; @@ -255,10 +257,9 @@ typedef int (*get_partitions_in_range_iter)(partition_info *part_info, #ifdef WITH_PARTITION_STORAGE_ENGINE uint fast_alter_partition_table(THD *thd, TABLE *table, Alter_info *alter_info, + Alter_table_ctx *alter_ctx, HA_CREATE_INFO *create_info, - TABLE_LIST *table_list, - const LEX_CSTRING *db, - const LEX_CSTRING *table_name); + TABLE_LIST *table_list); bool set_part_state(Alter_info *alter_info, partition_info *tab_part_info, enum partition_state part_state); uint prep_alter_part_table(THD *thd, TABLE *table, Alter_info *alter_info, @@ -278,7 +279,16 @@ bool verify_data_with_partition(TABLE *table, TABLE *part_table, uint32 part_id); bool compare_partition_options(HA_CREATE_INFO *table_create_info, partition_element *part_elem); +bool compare_table_with_partition(THD *thd, TABLE *table, + TABLE *part_table, + partition_element *part_elem, + uint part_id); bool partition_key_modified(TABLE *table, const MY_BITMAP *fields); +bool write_log_replace_frm(ALTER_PARTITION_PARAM_TYPE *lpt, + uint next_entry, + const char *from_path, + const char *to_path); + #else #define partition_key_modified(X,Y) 0 #endif diff --git a/sql/sql_partition_admin.cc b/sql/sql_partition_admin.cc index 6ca96300b7a..fcc08b69af4 100644 --- a/sql/sql_partition_admin.cc +++ b/sql/sql_partition_admin.cc @@ -193,10 +193,8 @@ static bool check_exchange_partition(TABLE *table, TABLE *part_table) @param part_table Partitioned table. @param part_elem Partition element to use for partition specific compare. */ -static bool compare_table_with_partition(THD *thd, TABLE *table, - TABLE *part_table, - partition_element *part_elem, - uint part_id) +bool compare_table_with_partition(THD *thd, TABLE *table, TABLE *part_table, + partition_element *part_elem, uint part_id) { HA_CREATE_INFO table_create_info, part_create_info; Alter_info part_alter_info; @@ -298,7 +296,7 @@ static bool compare_table_with_partition(THD *thd, TABLE *table, The workaround is to use REORGANIZE PARTITION to rewrite the frm file and then use EXCHANGE PARTITION when they are the same. */ - if (compare_partition_options(&table_create_info, part_elem)) + if (part_elem && compare_partition_options(&table_create_info, part_elem)) DBUG_RETURN(TRUE); DBUG_RETURN(FALSE); @@ -1002,4 +1000,53 @@ bool Sql_cmd_alter_table_truncate_partition::execute(THD *thd) DBUG_RETURN(error); } + +/** + Move a table specified in the CONVERT TABLE <table_name> TO PARTITION ... + to the new partition. + + @param lpt A structure containing parameters regarding to the statement + ALTER TABLE ... TO PARTITION ... + @param part_file_name a file name of the partition being added + + @return false on success, true on error +*/ + +bool alter_partition_convert_in(ALTER_PARTITION_PARAM_TYPE *lpt) +{ + char part_file_name[2*FN_REFLEN+1]; + THD *thd= lpt->thd; + const char *path= lpt->table_list->table->s->path.str; + TABLE_LIST *table_from= lpt->table_list->next_local; + + const char *partition_name= + thd->lex->part_info->curr_part_elem->partition_name; + + if (create_partition_name(part_file_name, sizeof(part_file_name), path, + partition_name, NORMAL_PART_NAME, false)) + return true; + + char from_file_name[FN_REFLEN+1]; + + build_table_filename(from_file_name, sizeof(from_file_name), + table_from->db.str, table_from->table_name.str, "", 0); + + handler *file= get_new_handler(nullptr, thd->mem_root, + table_from->table->file->ht); + if (unlikely(!file)) + return true; + + close_all_tables_for_name(thd, table_from->table->s, + HA_EXTRA_PREPARE_FOR_RENAME, nullptr); + + bool res= file->ha_rename_table(from_file_name, part_file_name); + + if (res) + my_error(ER_ERROR_ON_RENAME, MYF(0), from_file_name, + part_file_name, my_errno); + + delete file; + return res; +} + #endif /* WITH_PARTITION_STORAGE_ENGINE */ diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 7bd9c7c10d1..5a077a934ac 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -459,7 +459,7 @@ static int item_val_real(struct st_mysql_value *value, double *buf) static struct st_plugin_dl *plugin_dl_find(const LEX_CSTRING *dl) { - uint i; + size_t i; struct st_plugin_dl *tmp; DBUG_ENTER("plugin_dl_find"); for (i= 0; i < plugin_dl_array.elements; i++) @@ -476,7 +476,7 @@ static struct st_plugin_dl *plugin_dl_find(const LEX_CSTRING *dl) static st_plugin_dl *plugin_dl_insert_or_reuse(struct st_plugin_dl *plugin_dl) { - uint i; + size_t i; struct st_plugin_dl *tmp; DBUG_ENTER("plugin_dl_insert_or_reuse"); for (i= 0; i < plugin_dl_array.elements; i++) @@ -1071,6 +1071,8 @@ plugin_ref plugin_lock_by_name(THD *thd, const LEX_CSTRING *name, int type) plugin_ref rc= NULL; st_plugin_int *plugin; DBUG_ENTER("plugin_lock_by_name"); + if (!name->length) + DBUG_RETURN(NULL); mysql_mutex_lock(&LOCK_plugin); if ((plugin= plugin_find_internal(name, type))) rc= intern_plugin_lock(lex, plugin_int_to_ref(plugin)); @@ -1081,7 +1083,7 @@ plugin_ref plugin_lock_by_name(THD *thd, const LEX_CSTRING *name, int type) static st_plugin_int *plugin_insert_or_reuse(struct st_plugin_int *plugin) { - uint i; + size_t i; struct st_plugin_int *tmp; DBUG_ENTER("plugin_insert_or_reuse"); for (i= 0; i < plugin_array.elements; i++) @@ -1264,24 +1266,18 @@ static void plugin_deinitialize(struct st_plugin_int *plugin, bool ref_check) remove_status_vars(show_vars); } - if (plugin_type_deinitialize[plugin->plugin->type]) - { - if ((*plugin_type_deinitialize[plugin->plugin->type])(plugin)) - { - sql_print_error("Plugin '%s' of type %s failed deinitialization", - plugin->name.str, plugin_type_names[plugin->plugin->type].str); - } - } - else if (plugin->plugin->deinit) + plugin_type_init deinit= plugin_type_deinitialize[plugin->plugin->type]; + if (!deinit) + deinit= (plugin_type_init)(plugin->plugin->deinit); + + if (deinit && deinit(plugin)) { - DBUG_PRINT("info", ("Deinitializing plugin: '%s'", plugin->name.str)); - if (plugin->plugin->deinit(plugin)) - { - DBUG_PRINT("warning", ("Plugin '%s' deinit function returned error.", - plugin->name.str)); - } + if (THD *thd= current_thd) + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + WARN_PLUGIN_BUSY, ER_THD(thd, WARN_PLUGIN_BUSY)); } - plugin->state= PLUGIN_IS_UNINITIALIZED; + else + plugin->state= PLUGIN_IS_UNINITIALIZED; // free to unload if (ref_check && plugin->ref_count) sql_print_error("Plugin '%s' has ref_count=%d after deinitialization.", @@ -1289,10 +1285,13 @@ static void plugin_deinitialize(struct st_plugin_int *plugin, bool ref_check) plugin_variables_deinit(plugin); } -static void plugin_del(struct st_plugin_int *plugin) +static void plugin_del(struct st_plugin_int *plugin, uint del_mask) { DBUG_ENTER("plugin_del"); mysql_mutex_assert_owner(&LOCK_plugin); + del_mask|= PLUGIN_IS_UNINITIALIZED | PLUGIN_IS_DISABLED; // always use these + if (!(plugin->state & del_mask)) + DBUG_VOID_RETURN; /* Free allocated strings before deleting the plugin. */ plugin_vars_free_values(plugin->plugin->system_vars); restore_ptr_backup(plugin->nbackups, plugin->ptr_backup); @@ -1310,7 +1309,7 @@ static void plugin_del(struct st_plugin_int *plugin) static void reap_plugins(void) { - uint count; + size_t count; struct st_plugin_int *plugin, **reap, **list; mysql_mutex_assert_owner(&LOCK_plugin); @@ -1342,19 +1341,19 @@ static void reap_plugins(void) list= reap; while ((plugin= *(--list))) - plugin_deinitialize(plugin, true); + plugin_deinitialize(plugin, true); mysql_mutex_lock(&LOCK_plugin); while ((plugin= *(--reap))) - plugin_del(plugin); + plugin_del(plugin, 0); my_afree(reap); } static void intern_plugin_unlock(LEX *lex, plugin_ref plugin) { - int i; + ssize_t i; st_plugin_int *pi; DBUG_ENTER("intern_plugin_unlock"); @@ -1420,7 +1419,7 @@ void plugin_unlock(THD *thd, plugin_ref plugin) } -void plugin_unlock_list(THD *thd, plugin_ref *list, uint count) +void plugin_unlock_list(THD *thd, plugin_ref *list, size_t count) { LEX *lex= thd ? thd->lex : 0; DBUG_ENTER("plugin_unlock_list"); @@ -1593,7 +1592,7 @@ static void init_plugin_psi_keys(void) {} */ int plugin_init(int *argc, char **argv, int flags) { - uint i; + size_t i; struct st_maria_plugin **builtins; struct st_maria_plugin *plugin; struct st_plugin_int tmp, *plugin_ptr, **reap; @@ -1785,7 +1784,7 @@ int plugin_init(int *argc, char **argv, int flags) reaped_mandatory_plugin= TRUE; plugin_deinitialize(plugin_ptr, true); mysql_mutex_lock(&LOCK_plugin); - plugin_del(plugin_ptr); + plugin_del(plugin_ptr, 0); } mysql_mutex_unlock(&LOCK_plugin); @@ -2024,7 +2023,7 @@ error: void plugin_shutdown(void) { - uint i, count= plugin_array.elements; + size_t i, count= plugin_array.elements; struct st_plugin_int **plugins, *plugin; struct st_plugin_dl **dl; DBUG_ENTER("plugin_shutdown"); @@ -2073,12 +2072,14 @@ void plugin_shutdown(void) plugins= (struct st_plugin_int **) my_alloca(sizeof(void*) * (count+1)); /* - If we have any plugins which did not die cleanly, we force shutdown + If we have any plugins which did not die cleanly, we force shutdown. + Don't re-deinit() plugins that failed deinit() earlier (already dying) */ for (i= 0; i < count; i++) { plugins[i]= *dynamic_element(&plugin_array, i, struct st_plugin_int **); - /* change the state to ensure no reaping races */ + if (plugins[i]->state == PLUGIN_IS_DYING) + plugins[i]->state= PLUGIN_IS_UNINITIALIZED; if (plugins[i]->state == PLUGIN_IS_DELETED) plugins[i]->state= PLUGIN_IS_DYING; } @@ -2114,9 +2115,7 @@ void plugin_shutdown(void) if (plugins[i]->ref_count) sql_print_error("Plugin '%s' has ref_count=%d after shutdown.", plugins[i]->name.str, plugins[i]->ref_count); - if (plugins[i]->state & PLUGIN_IS_UNINITIALIZED || - plugins[i]->state & PLUGIN_IS_DISABLED) - plugin_del(plugins[i]); + plugin_del(plugins[i], PLUGIN_IS_DYING); } /* @@ -2355,7 +2354,7 @@ static bool do_uninstall(THD *thd, TABLE *table, const LEX_CSTRING *name) of the delete from the plugin table, so that it is not replicated in row based mode. */ - table->file->row_logging= 0; // No logging + table->file->row_logging= 0; // No logging error= table->file->ha_delete_row(table->record[0]); if (unlikely(error)) { @@ -2468,7 +2467,7 @@ wsrep_error_label: bool plugin_foreach_with_mask(THD *thd, plugin_foreach_func *func, int type, uint state_mask, void *arg) { - uint idx, total= 0; + size_t idx, total= 0; struct st_plugin_int *plugin; plugin_ref *plugins; my_bool res= FALSE; @@ -3319,7 +3318,7 @@ static void cleanup_variables(struct system_variables *vars) void plugin_thdvar_cleanup(THD *thd) { - uint idx; + size_t idx; plugin_ref *list; DBUG_ENTER("plugin_thdvar_cleanup"); @@ -4315,7 +4314,7 @@ void add_plugin_options(DYNAMIC_ARRAY *options, MEM_ROOT *mem_root) if (!initialized) return; - for (uint idx= 0; idx < plugin_array.elements; idx++) + for (size_t idx= 0; idx < plugin_array.elements; idx++) { p= *dynamic_element(&plugin_array, idx, struct st_plugin_int **); @@ -4422,7 +4421,7 @@ int thd_setspecific(MYSQL_THD thd, MYSQL_THD_KEY_T key, void *value) DBUG_ASSERT(key != INVALID_THD_KEY); if (key == INVALID_THD_KEY || (!thd && !(thd= current_thd))) return EINVAL; - + memcpy(intern_sys_var_ptr(thd, key, true), &value, sizeof(void*)); return 0; } diff --git a/sql/sql_plugin.h b/sql/sql_plugin.h index eaf0b40f34a..d4df8c6468f 100644 --- a/sql/sql_plugin.h +++ b/sql/sql_plugin.h @@ -172,7 +172,7 @@ extern plugin_ref plugin_lock(THD *thd, plugin_ref ptr); extern plugin_ref plugin_lock_by_name(THD *thd, const LEX_CSTRING *name, int type); extern void plugin_unlock(THD *thd, plugin_ref plugin); -extern void plugin_unlock_list(THD *thd, plugin_ref *list, uint count); +extern void plugin_unlock_list(THD *thd, plugin_ref *list, size_t count); extern bool mysql_install_plugin(THD *thd, const LEX_CSTRING *name, const LEX_CSTRING *dl); extern bool mysql_uninstall_plugin(THD *thd, const LEX_CSTRING *name, diff --git a/sql/sql_plugin_services.inl b/sql/sql_plugin_services.inl index 86b2fb69b22..3a66e982e7b 100644 --- a/sql/sql_plugin_services.inl +++ b/sql/sql_plugin_services.inl @@ -218,7 +218,7 @@ static struct my_print_error_service_st my_print_error_handler= my_printv_error }; -struct json_service_st json_handler= +static struct json_service_st json_handler= { json_type, json_get_array_item, @@ -233,6 +233,97 @@ static struct thd_mdl_service_st thd_mdl_handler= thd_mdl_context }; +struct sql_service_st sql_service_handler= +{ + mysql_init, + mysql_real_connect_local, + mysql_real_connect, + mysql_errno, + mysql_error, + mysql_real_query, + mysql_affected_rows, + mysql_num_rows, + mysql_store_result, + mysql_free_result, + mysql_fetch_row, + mysql_close, + mysql_options, + mysql_fetch_lengths, + mysql_set_character_set, + mysql_num_fields, + mysql_select_db +}; + +#define DEFINE_warning_function(name, ret) { \ + static query_id_t last_query_id= -1; \ + THD *thd= current_thd; \ + if((thd ? thd->query_id : 0) != last_query_id) \ + { \ + my_error(ER_PROVIDER_NOT_LOADED, MYF(ME_ERROR_LOG|ME_WARNING), name); \ + last_query_id= thd ? thd->query_id : 0; \ + } \ + return ret; \ +} + +#include <providers/lzma.h> +static struct provider_service_lzma_st provider_handler_lzma= +{ + DEFINE_lzma_stream_buffer_decode([]) DEFINE_warning_function("LZMA compression", LZMA_PROG_ERROR), + DEFINE_lzma_easy_buffer_encode([]) DEFINE_warning_function("LZMA compression", LZMA_PROG_ERROR), + + false // .is_loaded +}; +struct provider_service_lzma_st *provider_service_lzma= &provider_handler_lzma; + +#include <providers/lzo/lzo1x.h> +static struct provider_service_lzo_st provider_handler_lzo= +{ + DEFINE_lzo1x_1_15_compress([]) DEFINE_warning_function("LZO compression", LZO_E_INTERNAL_ERROR), + DEFINE_lzo1x_decompress_safe([]) DEFINE_warning_function("LZO compression", LZO_E_INTERNAL_ERROR), + + false // .is_loaded +}; +struct provider_service_lzo_st *provider_service_lzo= &provider_handler_lzo; + +#include <providers/bzlib.h> +static struct provider_service_bzip2_st provider_handler_bzip2= +{ + DEFINE_BZ2_bzBuffToBuffCompress([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzBuffToBuffDecompress([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzCompress([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzCompressEnd([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzCompressInit([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzDecompress([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzDecompressEnd([]) DEFINE_warning_function("BZip2 compression", -1), + DEFINE_BZ2_bzDecompressInit([]) DEFINE_warning_function("BZip2 compression", -1), + + false // .is_loaded +}; +struct provider_service_bzip2_st *provider_service_bzip2= &provider_handler_bzip2; + +#include <providers/snappy-c.h> +static struct provider_service_snappy_st provider_handler_snappy= +{ + DEFINE_snappy_max_compressed_length([]) -> size_t DEFINE_warning_function("Snappy compression", 0), + DEFINE_snappy_compress([]) DEFINE_warning_function("Snappy compression", SNAPPY_INVALID_INPUT), + DEFINE_snappy_uncompressed_length([]) DEFINE_warning_function("Snappy compression", SNAPPY_INVALID_INPUT), + DEFINE_snappy_uncompress([]) DEFINE_warning_function("Snappy compression", SNAPPY_INVALID_INPUT), + + false // .is_loaded +}; +struct provider_service_snappy_st *provider_service_snappy= &provider_handler_snappy; + +#include <providers/lz4.h> +static struct provider_service_lz4_st provider_handler_lz4= +{ + DEFINE_LZ4_compressBound([]) DEFINE_warning_function("LZ4 compression", 0), + DEFINE_LZ4_compress_default([]) DEFINE_warning_function("LZ4 compression", 0), + DEFINE_LZ4_decompress_safe([]) DEFINE_warning_function("LZ4 compression", -1), + + false // .is_loaded +}; +struct provider_service_lz4_st *provider_service_lz4= &provider_handler_lz4; + static struct st_service_ref list_of_services[]= { { "base64_service", VERSION_base64, &base64_handler }, @@ -257,5 +348,11 @@ static struct st_service_ref list_of_services[]= { "thd_wait_service", VERSION_thd_wait, &thd_wait_handler }, { "wsrep_service", VERSION_wsrep, &wsrep_handler }, { "json_service", VERSION_json, &json_handler }, - { "thd_mdl_service", VERSION_thd_mdl, &thd_mdl_handler } + { "thd_mdl_service", VERSION_thd_mdl, &thd_mdl_handler }, + { "sql_service", VERSION_sql_service, &sql_service_handler }, + { "provider_service_bzip2", VERSION_provider_bzip2, &provider_handler_bzip2 }, + { "provider_service_lz4", VERSION_provider_lz4, &provider_handler_lz4 }, + { "provider_service_lzma", VERSION_provider_lzma, &provider_handler_lzma }, + { "provider_service_lzo", VERSION_provider_lzo, &provider_handler_lzo }, + { "provider_service_snappy", VERSION_provider_snappy, &provider_handler_snappy } }; diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index ace6ad8478d..4ff5a679ffd 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -133,6 +133,7 @@ static const uint PARAMETER_FLAG_UNSIGNED= 128U << 8; #include "wsrep_trans_observer.h" #endif /* WITH_WSREP */ #include "xa.h" // xa_recover_get_fields +#include "sql_audit.h" // mysql_audit_release /** A result class used to send cursor rows using the binary protocol. @@ -1287,7 +1288,8 @@ static bool mysql_test_insert_common(Prepared_statement *stmt, List<List_item> &values_list, List<Item> &update_fields, List<Item> &update_values, - enum_duplicates duplic) + enum_duplicates duplic, + bool ignore) { THD *thd= stmt->thd; List_iterator_fast<List_item> its(values_list); @@ -1314,7 +1316,6 @@ static bool mysql_test_insert_common(Prepared_statement *stmt, if ((values= its++)) { uint value_count; - ulong counter= 0; Item *unused_conds= 0; if (table_list->table) @@ -1324,7 +1325,8 @@ static bool mysql_test_insert_common(Prepared_statement *stmt, } if (mysql_prepare_insert(thd, table_list, fields, values, update_fields, - update_values, duplic, &unused_conds, FALSE)) + update_values, duplic, ignore, + &unused_conds, FALSE)) goto error; value_count= values->elements; @@ -1340,16 +1342,18 @@ static bool mysql_test_insert_common(Prepared_statement *stmt, } while ((values= its++)) { - counter++; if (values->elements != value_count) { - my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), counter); + my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), + thd->get_stmt_da()->current_row_for_warning()); goto error; } if (setup_fields(thd, Ref_ptr_array(), *values, COLUMNS_READ, 0, NULL, 0)) goto error; + thd->get_stmt_da()->inc_current_row_for_warning(); } + thd->get_stmt_da()->reset_current_row_for_warning(1); } DBUG_RETURN(FALSE); @@ -1377,7 +1381,7 @@ static bool mysql_test_insert(Prepared_statement *stmt, List<List_item> &values_list, List<Item> &update_fields, List<Item> &update_values, - enum_duplicates duplic) + enum_duplicates duplic, bool ignore) { THD *thd= stmt->thd; @@ -1393,7 +1397,7 @@ static bool mysql_test_insert(Prepared_statement *stmt, } return mysql_test_insert_common(stmt, table_list, fields, values_list, - update_fields, update_values, duplic); + update_fields, update_values, duplic, ignore); } @@ -2471,14 +2475,14 @@ static bool check_prepared_statement(Prepared_statement *stmt) res= mysql_test_insert(stmt, tables, lex->field_list, lex->many_values, lex->update_list, lex->value_list, - lex->duplicates); + lex->duplicates, lex->ignore); break; case SQLCOM_LOAD: res= mysql_test_insert_common(stmt, tables, lex->field_list, lex->many_values, lex->update_list, lex->value_list, - lex->duplicates); + lex->duplicates, lex->ignore); break; case SQLCOM_UPDATE: @@ -4050,19 +4054,22 @@ Execute_sql_statement(LEX_STRING sql_text) executions without having to cleanup/reset THD in between. */ -bool -Execute_sql_statement::execute_server_code(THD *thd) +static bool execute_server_code(THD *thd, + const char *sql_text, size_t sql_len) { PSI_statement_locker *parent_locker; bool error; + query_id_t save_query_id= thd->query_id; + query_id_t next_id= next_query_id(); - if (alloc_query(thd, m_sql_text.str, m_sql_text.length)) + if (alloc_query(thd, sql_text, sql_len)) return TRUE; Parser_state parser_state; if (parser_state.init(thd, thd->query(), thd->query_length())) return TRUE; + thd->query_id= next_id; parser_state.m_lip.multi_statements= FALSE; lex_start(thd); @@ -4080,17 +4087,23 @@ Execute_sql_statement::execute_server_code(THD *thd) /* report error issued during command execution */ if (likely(error == 0) && thd->spcont == NULL) - general_log_write(thd, COM_STMT_EXECUTE, + general_log_write(thd, COM_QUERY, thd->query(), thd->query_length()); end: thd->lex->restore_set_statement_var(); + thd->query_id= save_query_id; delete_explain_query(thd->lex); lex_end(thd->lex); return error; } +bool Execute_sql_statement::execute_server_code(THD *thd) +{ + return ::execute_server_code(thd, m_sql_text.str, m_sql_text.length); +} + /*************************************************************************** Prepared_statement ****************************************************************************/ @@ -4860,7 +4873,9 @@ Prepared_statement::execute_server_runnable(Server_runnable *server_runnable) Statement stmt_backup; bool error; Query_arena *save_stmt_arena= thd->stmt_arena; + Reprepare_observer *save_reprepare_observer= thd->m_reprepare_observer; Item_change_list save_change_list; + thd->Item_change_list::move_elements_to(&save_change_list); state= STMT_CONVENTIONAL_EXECUTION; @@ -4870,12 +4885,15 @@ Prepared_statement::execute_server_runnable(Server_runnable *server_runnable) thd->set_n_backup_statement(this, &stmt_backup); thd->set_n_backup_active_arena(this, &stmt_backup); + thd->stmt_arena= this; + thd->m_reprepare_observer= 0; error= server_runnable->execute_server_code(thd); thd->cleanup_after_query(); + thd->m_reprepare_observer= save_reprepare_observer; thd->restore_active_arena(this, &stmt_backup); thd->restore_backup_statement(this, &stmt_backup); thd->stmt_arena= save_stmt_arena; @@ -5593,14 +5611,6 @@ Ed_connection::store_result_set() return ed_result_set; } -/* - MENT-56 - Protocol_local and service_sql for plugins to enable 'local' SQL query execution. -*/ - -#ifndef EMBEDDED_LIBRARY -// This part is mostly copied from libmysqld/lib_sql.cc -// TODO: get rid of code duplications #include <mysql.h> #include "../libmysqld/embedded_priv.h" @@ -5616,12 +5626,30 @@ public: char **next_field; MYSQL_FIELD *next_mysql_field; MEM_ROOT *alloc; + THD *new_thd; + Security_context empty_ctx; + ulonglong client_capabilities; + + my_bool do_log_bin; - Protocol_local(THD *thd_arg, ulong prealloc= 0) : + Protocol_local(THD *thd_arg, THD *new_thd_arg, ulong prealloc) : Protocol_text(thd_arg, prealloc), - cur_data(0), first_data(0), data_tail(&first_data), alloc(0) - {} + cur_data(0), first_data(0), data_tail(&first_data), alloc(0), + new_thd(new_thd_arg), do_log_bin(FALSE) + {} + void set_binlog_vars(my_bool *sav_log_bin) + { + *sav_log_bin= thd->variables.sql_log_bin; + thd->variables.sql_log_bin= do_log_bin; + thd->set_binlog_bit(); + } + void restore_binlog_vars(my_bool sav_log_bin) + { + do_log_bin= thd->variables.sql_log_bin; + thd->variables.sql_log_bin= sav_log_bin; + thd->set_binlog_bit(); + } protected: bool net_store_data(const uchar *from, size_t length); bool net_store_data_cs(const uchar *from, size_t length, @@ -5692,6 +5720,20 @@ MYSQL_DATA *Protocol_local::alloc_new_dataset() } +void Protocol_local::clear_data_list() +{ + while (first_data) + { + MYSQL_DATA *data= first_data; + first_data= data->embedded_info->next; + free_rows(data); + } + data_tail= &first_data; + free_rows(cur_data); + cur_data= 0; +} + + static char *dup_str_aux(MEM_ROOT *root, const char *from, uint length, CHARSET_INFO *fromcs, CHARSET_INFO *tocs) { @@ -5985,7 +6027,6 @@ bool Protocol_local::send_result_set_metadata(List<Item> *list, uint flags) { List_iterator_fast<Item> it(*list); Item *item; -// Protocol_local prot(thd); DBUG_ENTER("send_result_set_metadata"); // if (!thd->mysql) // bootstrap file handling @@ -5996,7 +6037,7 @@ bool Protocol_local::send_result_set_metadata(List<Item> *list, uint flags) for (uint pos= 0 ; (item= it++); pos++) { - if (/*prot.*/store_item_metadata(thd, item, pos)) + if (store_item_metadata(thd, item, pos)) goto err; } @@ -6010,6 +6051,7 @@ bool Protocol_local::send_result_set_metadata(List<Item> *list, uint flags) DBUG_RETURN(1); /* purecov: inspected */ } + static void list_fields_send_default(THD *thd, Protocol_local *p, Field *fld, uint pos) { @@ -6097,19 +6139,6 @@ bool Protocol_local::store_null() #include <sql_common.h> #include <errmsg.h> -struct local_results -{ - struct st_mysql_data *cur_data; - struct st_mysql_data *first_data; - struct st_mysql_data **data_tail; - void clear_data_list(); - struct st_mysql_data *alloc_new_dataset(); - char **next_field; - MYSQL_FIELD *next_mysql_field; - MEM_ROOT *alloc; -}; - - static void embedded_get_error(MYSQL *mysql, MYSQL_DATA *data) { NET *net= &mysql->net; @@ -6124,11 +6153,11 @@ static void embedded_get_error(MYSQL *mysql, MYSQL_DATA *data) static my_bool loc_read_query_result(MYSQL *mysql) { - local_results *thd= (local_results *) mysql->thd; + Protocol_local *p= (Protocol_local *) mysql->thd; - MYSQL_DATA *res= thd->first_data; - DBUG_ASSERT(!thd->cur_data); - thd->first_data= res->embedded_info->next; + MYSQL_DATA *res= p->first_data; + DBUG_ASSERT(!p->cur_data); + p->first_data= res->embedded_info->next; if (res->embedded_info->last_errno && !res->embedded_info->fields_list) { @@ -6156,7 +6185,7 @@ static my_bool loc_read_query_result(MYSQL *mysql) if (res->embedded_info->fields_list) { mysql->status=MYSQL_STATUS_GET_RESULT; - thd->cur_data= res; + p->cur_data= res; } else my_free(res); @@ -6165,174 +6194,273 @@ static my_bool loc_read_query_result(MYSQL *mysql) } -static MYSQL_METHODS local_methods= +static my_bool +loc_advanced_command(MYSQL *mysql, enum enum_server_command command, + const uchar *header, ulong header_length, + const uchar *arg, ulong arg_length, my_bool skip_check, + MYSQL_STMT *stmt) { - loc_read_query_result, /* read_query_result */ - NULL/*loc_advanced_command*/, /* advanced_command */ - NULL/*loc_read_rows*/, /* read_rows */ - NULL/*loc_use_result*/, /* use_result */ - NULL/*loc_fetch_lengths*/, /* fetch_lengths */ - NULL/*loc_flush_use_result*/, /* flush_use_result */ - NULL/*loc_read_change_user_result*/ /* read_change_user_result */ -}; + my_bool result= 1; + Protocol_local *p= (Protocol_local *) mysql->thd; + NET *net= &mysql->net; + if (p->thd && p->thd->killed != NOT_KILLED) + { + if (p->thd->killed < KILL_CONNECTION) + p->thd->killed= NOT_KILLED; + else + return 1; + } -extern "C" MYSQL *mysql_real_connect_local(MYSQL *mysql, - const char *host, const char *user, const char *passwd, const char *db) -{ - //char name_buff[USERNAME_LENGTH]; + p->clear_data_list(); + /* Check that we are calling the client functions in right order */ + if (mysql->status != MYSQL_STATUS_READY) + { + set_mysql_error(mysql, CR_COMMANDS_OUT_OF_SYNC, unknown_sqlstate); + goto end; + } - DBUG_ENTER("mysql_real_connect_local"); + /* Clear result variables */ + p->thd->clear_error(1); + mysql->affected_rows= ~(my_ulonglong) 0; + mysql->field_count= 0; + net_clear_error(net); - /* Test whether we're already connected */ - if (mysql->server_version) + /* + We have to call free_old_query before we start to fill mysql->fields + for new query. In the case of embedded server we collect field data + during query execution (not during data retrieval as it is in remote + client). So we have to call free_old_query here + */ + free_old_query(mysql); + + if (header) { - set_mysql_error(mysql, CR_ALREADY_CONNECTED, unknown_sqlstate); - DBUG_RETURN(0); + arg= header; + arg_length= header_length; } - if (!host || !host[0]) - host= mysql->options.host; + if (p->new_thd) + { + THD *thd_orig= current_thd; + set_current_thd(p->thd); + p->thd->thread_stack= (char*) &result; + p->thd->set_time(); + result= execute_server_code(p->thd, (const char *)arg, arg_length); + p->thd->cleanup_after_query(); + mysql_audit_release(p->thd); + p->end_statement(); + set_current_thd(thd_orig); + } + else + { + Ed_connection con(p->thd); + Security_context *ctx_orig= p->thd->security_ctx; + ulonglong cap_orig= p->thd->client_capabilities; + MYSQL_LEX_STRING sql_text; + my_bool log_bin_orig; + p->set_binlog_vars(&log_bin_orig); + + DBUG_ASSERT(current_thd == p->thd); + sql_text.str= (char *) arg; + sql_text.length= arg_length; + p->thd->security_ctx= &p->empty_ctx; + p->thd->client_capabilities= p->client_capabilities; + result= con.execute_direct(p, sql_text); + p->thd->client_capabilities= cap_orig; + p->thd->security_ctx= ctx_orig; + p->restore_binlog_vars(log_bin_orig); + } + if (skip_check) + result= 0; + p->cur_data= 0; - mysql->methods= &local_methods; +end: + return result; +} - if (!db || !db[0]) - db=mysql->options.db; - if (!user || !user[0]) - user=mysql->options.user; +/* + reads dataset from the next query result - mysql->user= my_strdup(PSI_INSTRUMENT_ME, user, MYF(0)); + SYNOPSIS + loc_read_rows() + mysql connection handle + other parameters are not used + NOTES + It just gets next MYSQL_DATA from the result's queue - mysql->info_buffer= (char *) my_malloc(PSI_INSTRUMENT_ME, - MYSQL_ERRMSG_SIZE, MYF(0)); - //mysql->thd= create_embedded_thd(client_flag); + RETURN + pointer to MYSQL_DATA with the coming recordset +*/ - //init_embedded_mysql(mysql, client_flag); +static MYSQL_DATA * +loc_read_rows(MYSQL *mysql, MYSQL_FIELD *mysql_fields __attribute__((unused)), + unsigned int fields __attribute__((unused))) +{ + MYSQL_DATA *result= ((Protocol_local *)mysql->thd)->cur_data; + ((Protocol_local *)mysql->thd)->cur_data= 0; + if (result->embedded_info->last_errno) + { + embedded_get_error(mysql, result); + return NULL; + } + *result->embedded_info->prev_ptr= NULL; + return result; +} - //if (mysql_init_character_set(mysql)) - // goto error; - //if (check_embedded_connection(mysql, db)) - // goto error; +/************************************************************************** + Get column lengths of the current row + If one uses mysql_use_result, res->lengths contains the length information, + else the lengths are calculated from the offset between pointers. +**************************************************************************/ - mysql->server_status= SERVER_STATUS_AUTOCOMMIT; +static void loc_fetch_lengths(ulong *to, MYSQL_ROW column, + unsigned int field_count) +{ + MYSQL_ROW end; - //if (mysql->options.init_commands) - //{ - // DYNAMIC_ARRAY *init_commands= mysql->options.init_commands; - // char **ptr= (char**)init_commands->buffer; - // char **end= ptr + init_commands->elements; -// - // for (; ptr<end; ptr++) - // { - // MYSQL_RES *res; - // if (mysql_query(mysql,*ptr)) - // goto error; - // if (mysql->fields) - // { - // if (!(res= (*mysql->methods->use_result)(mysql))) - // goto error; - // mysql_free_result(res); - // } - // } - //} + for (end=column + field_count; column != end ; column++,to++) + *to= *column ? *(uint *)((*column) - sizeof(uint)) : 0; +} - DBUG_PRINT("exit",("Mysql handler: %p", mysql)); - DBUG_RETURN(mysql); -//error: - DBUG_PRINT("error",("message: %u (%s)", - mysql->net.last_errno, - mysql->net.last_error)); +static void loc_flush_use_result(MYSQL *mysql, my_bool) +{ + Protocol_local *p= (Protocol_local *) mysql->thd; + if (p->cur_data) { - /* Free alloced memory */ - my_bool free_me=mysql->free_me; - free_old_query(mysql); - mysql->free_me=0; - mysql_close(mysql); - mysql->free_me=free_me; + free_rows(p->cur_data); + p->cur_data= 0; + } + else if (p->first_data) + { + MYSQL_DATA *data= p->first_data; + p->first_data= data->embedded_info->next; + free_rows(data); } - DBUG_RETURN(0); } -extern "C" int execute_sql_command(const char *command, - char *hosts, char *names, char *filters) +static void loc_on_close_free(MYSQL *mysql) { - MYSQL_LEX_STRING sql_text; - THD *thd= current_thd; - THD *new_thd= 0; - int result; - my_bool qc_save= 0; - Reprepare_observer *save_reprepare_observer= nullptr; + Protocol_local *p= (Protocol_local *) mysql->thd; + THD *thd= p->new_thd; + delete p; + if (thd) + { + delete thd; + local_connection_thread_count--; + } + my_free(mysql->info_buffer); + mysql->info_buffer= 0; +} - if (!thd) +static MYSQL_RES *loc_use_result(MYSQL *mysql) +{ + return mysql_store_result(mysql); +} + +static MYSQL_METHODS local_methods= +{ + loc_read_query_result, /* read_query_result */ + loc_advanced_command, /* advanced_command */ + loc_read_rows, /* read_rows */ + loc_use_result, /* use_result */ + loc_fetch_lengths, /* fetch_lengths */ + loc_flush_use_result, /* flush_use_result */ + NULL, /* read_change_user_result */ + loc_on_close_free /* on_close_free */ +#ifdef EMBEDDED_LIBRARY + ,NULL, /* list_fields */ + NULL, /* read_prepare_result */ + NULL, /* stmt_execute */ + NULL, /* read_binary_rows */ + NULL, /* unbuffered_fetch */ + NULL, /* read_statistics */ + NULL, /* next_result */ + NULL /* read_rows_from_cursor */ +#endif +}; + + +Atomic_counter<uint32_t> local_connection_thread_count; + +extern "C" MYSQL *mysql_real_connect_local(MYSQL *mysql) +{ + THD *thd_orig= current_thd; + THD *new_thd; + Protocol_local *p; + ulonglong client_flag; + DBUG_ENTER("mysql_real_connect_local"); + + /* Test whether we're already connected */ + if (mysql->server_version) { + set_mysql_error(mysql, CR_ALREADY_CONNECTED, unknown_sqlstate); + DBUG_RETURN(0); + } + + mysql->methods= &local_methods; + mysql->user= NULL; + client_flag= mysql->options.client_flag; + client_flag|= CLIENT_MULTI_RESULTS;; + client_flag&= ~(CLIENT_COMPRESS | CLIENT_PLUGIN_AUTH); + + mysql->info_buffer= (char *) my_malloc(PSI_INSTRUMENT_ME, + MYSQL_ERRMSG_SIZE, MYF(0)); + if (!thd_orig || thd_orig->lock) + { + /* + When we start with the empty current_thd (that happens when plugins + are loaded during the server start) or when some tables are locked + with the current_thd already (that happens when INSTALL PLUGIN + calls the plugin_init or with queries), we create the new THD for + the local connection. So queries with this MYSQL will be run with + it rather than the current THD. + */ + new_thd= new THD(0); - new_thd->thread_stack= (char*) &sql_text; + local_connection_thread_count++; + new_thd->thread_stack= (char*) &thd_orig; new_thd->store_globals(); new_thd->security_ctx->skip_grants(); new_thd->query_cache_is_applicable= 0; new_thd->variables.wsrep_on= 0; + new_thd->variables.sql_log_bin= 0; + new_thd->set_binlog_bit(); + new_thd->client_capabilities= client_flag; + + /* + TOSO: decide if we should turn the auditing off + for such threads. + We can do it like this: + new_thd->audit_class_mask[0]= ~0; + */ bzero((char*) &new_thd->net, sizeof(new_thd->net)); - thd= new_thd; + set_current_thd(thd_orig); + thd_orig= new_thd; } else - { - if (thd->lock) - /* Doesn't work if the thread opened/locked tables already. */ - return 2; - - qc_save= thd->query_cache_is_applicable; - thd->query_cache_is_applicable= 0; - save_reprepare_observer= thd->m_reprepare_observer; - thd->m_reprepare_observer= nullptr; - } - sql_text.str= (char *) command; - sql_text.length= strlen(command); - { - Protocol_local p(thd); - Ed_connection con(thd); - result= con.execute_direct(&p, sql_text); - if (!result && p.first_data) - { - int nr= (int) p.first_data->rows; - MYSQL_ROWS *rows= p.first_data->data; - - while (nr--) - { - strcpy(hosts, rows->data[0]); - hosts+= strlen(hosts) + 1; - strcpy(names, rows->data[1]); - names+= strlen(names) + 1; - if (filters) - { - strcpy(filters, rows->data[2]); - filters+= strlen(filters) + 1; - } - rows= rows->next; - } - } - if (p.first_data) - { - if (p.alloc) - free_root(p.alloc, MYF(0)); - my_free(p.first_data); - } - } + new_thd= NULL; + p= new Protocol_local(thd_orig, new_thd, 0); if (new_thd) - delete new_thd; + new_thd->protocol= p; else { - thd->query_cache_is_applicable= qc_save; - thd->m_reprepare_observer= save_reprepare_observer; + p->empty_ctx.init(); + p->empty_ctx.skip_grants(); + p->client_capabilities= client_flag; } - *hosts= 0; - return result; + mysql->thd= p; + mysql->server_status= SERVER_STATUS_AUTOCOMMIT; + + + DBUG_PRINT("exit",("Mysql handler: %p", mysql)); + DBUG_RETURN(mysql); } -#endif /*!EMBEDDED_LIBRARY*/ diff --git a/sql/sql_prepare.h b/sql/sql_prepare.h index 1ea773a7ca8..ff6e986ec87 100644 --- a/sql/sql_prepare.h +++ b/sql/sql_prepare.h @@ -353,4 +353,6 @@ private: size_t m_column_count; /* TODO: change to point to metadata */ }; +extern Atomic_counter<uint32_t> local_connection_thread_count; + #endif // SQL_PREPARE_H diff --git a/sql/sql_priv.h b/sql/sql_priv.h index 76260ec51e7..2f1f7166432 100644 --- a/sql/sql_priv.h +++ b/sql/sql_priv.h @@ -335,6 +335,8 @@ #define UNCACHEABLE_DEPENDENT (UNCACHEABLE_DEPENDENT_GENERATED | \ UNCACHEABLE_DEPENDENT_INJECTED) +#define FAKE_SELECT_LEX_ID UINT_MAX + /* Used to check GROUP BY list in the MODE_ONLY_FULL_GROUP_BY mode */ #define UNDEF_POS (-1) @@ -418,16 +420,6 @@ inline int hexchar_to_int(char c) /* This must match the path length limit in the ER_NOT_RW_DIR error msg. */ #define ER_NOT_RW_DIR_PATHSIZE 200 -#define IS_TABLESPACES_TABLESPACE_NAME 0 -#define IS_TABLESPACES_ENGINE 1 -#define IS_TABLESPACES_TABLESPACE_TYPE 2 -#define IS_TABLESPACES_LOGFILE_GROUP_NAME 3 -#define IS_TABLESPACES_EXTENT_SIZE 4 -#define IS_TABLESPACES_AUTOEXTEND_SIZE 5 -#define IS_TABLESPACES_MAXIMUM_SIZE 6 -#define IS_TABLESPACES_NODEGROUP_ID 7 -#define IS_TABLESPACES_TABLESPACE_COMMENT 8 - bool db_name_is_in_ignore_db_dirs_list(const char *dbase); #endif /* MYSQL_SERVER */ diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc index 3448e157e10..8f0f15a982a 100644 --- a/sql/sql_reload.cc +++ b/sql/sql_reload.cc @@ -406,7 +406,7 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, /* If not default connection and 'all' is used */ mi->release(); mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index->remove_master_info(mi)) + if (master_info_index->remove_master_info(mi, 0)) result= 1; mysql_mutex_unlock(&LOCK_active_mi); } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b00c7de9cbc..bf1732ade6b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -240,9 +240,7 @@ static bool test_if_cheaper_ordering(const JOIN_TAB *tab, ha_rows *new_select_limit, uint *new_used_key_parts= NULL, uint *saved_best_key_parts= NULL); -static int test_if_order_by_key(JOIN *join, - ORDER *order, TABLE *table, uint idx, - uint *used_key_parts); +static int test_if_order_by_key(JOIN *, ORDER *, TABLE *, uint, uint *); static bool test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order, ha_rows select_limit, bool no_changes, const key_map *map); @@ -2617,7 +2615,7 @@ int JOIN::optimize_stage2() if (!conds && outer_join) { /* Handle the case where we have an OUTER JOIN without a WHERE */ - conds= (Item*) &Item_true; + conds= (Item*) Item_true; } if (impossible_where) @@ -2787,9 +2785,7 @@ int JOIN::optimize_stage2() if (conds && const_table_map != found_const_table_map && (select_options & SELECT_DESCRIBE)) - { - conds= (Item*) &Item_false; - } + conds= (Item*) Item_false; /* Cache constant expressions in WHERE, HAVING, ON clauses. */ cache_const_exprs(); @@ -3101,7 +3097,7 @@ int JOIN::optimize_stage2() having= having->remove_eq_conds(thd, &select_lex->having_value, true); if (select_lex->having_value == Item::COND_FALSE) { - having= (Item*) &Item_false; + having= (Item*) Item_false; zero_result_cause= "Impossible HAVING noticed after reading const tables"; error= 0; select_lex->mark_const_derived(zero_result_cause); @@ -5049,6 +5045,7 @@ mysql_select(THD *thd, TABLE_LIST *tables, List<Item> &fields, COND *conds, } } + thd->get_stmt_da()->reset_current_row_for_warning(1); /* Look for a table owned by an engine with the select_handler interface */ select_lex->pushdown_select= find_select_handler(thd, select_lex); @@ -5711,7 +5708,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, if (join->cond_value == Item::COND_FALSE) { join->impossible_where= true; - conds= (Item*) &Item_false; + conds= (Item*) Item_false; } join->cond_equal= NULL; @@ -7196,7 +7193,7 @@ update_ref_and_keys(THD *thd, DYNAMIC_ARRAY *keyuse,JOIN_TAB *join_tab, /* set a barrier for the array of SARGABLE_PARAM */ (*sargables)[0].field= 0; - if (my_init_dynamic_array2(thd->mem_root->m_psi_key, keyuse, sizeof(KEYUSE), + if (my_init_dynamic_array2(thd->mem_root->psi_key, keyuse, sizeof(KEYUSE), thd->alloc(sizeof(KEYUSE) * 20), 20, 64, MYF(MY_THREAD_SPECIFIC))) DBUG_RETURN(TRUE); @@ -8108,7 +8105,7 @@ best_access_path(JOIN *join, /* quick_range couldn't use key! */ records= (double) s->records/rec; trace_access_idx.add("used_range_estimates", false) - .add("cause", "not available"); + .add("reason", "not available"); } } else @@ -8146,16 +8143,14 @@ best_access_path(JOIN *join, } else { + trace_access_idx.add("used_range_estimates", false); if (table->opt_range_keys.is_set(key)) { - trace_access_idx.add("used_range_estimates",false) - .add("cause", - "not better than ref estimates"); + trace_access_idx.add("reason", "not better than ref estimates"); } else { - trace_access_idx.add("used_range_estimates", false) - .add("cause", "not available"); + trace_access_idx.add("reason", "not available"); } } } @@ -8468,7 +8463,7 @@ best_access_path(JOIN *join, trace_access_idx.add("chosen", false) .add("cause", cause ? cause : "cost"); } - cause= NULL; + cause= nullptr; } /* for each key */ records= best_records; } @@ -8512,11 +8507,11 @@ best_access_path(JOIN *join, join->allowed_outer_join_with_cache)) // (2) { double fanout; + double join_sel; + bool stats_found= 0, force_estimate= 0; Json_writer_object trace_access_hash(thd); trace_access_hash.add("type", "hash"); trace_access_hash.add("index", "hj-key"); - double join_sel; - bool stats_found= 0, force_estimate= 0; /* Estimate the cost of the hash join access to the table */ double rnd_records= apply_selectivity_for_table(s, use_cond_selectivity, &force_estimate); @@ -8590,6 +8585,7 @@ best_access_path(JOIN *join, best_uses_jbuf= TRUE; best_filter= 0; best_type= JT_HASH; + trace_access_hash.add("rnd_records", rnd_records); trace_access_hash.add("records", records); trace_access_hash.add("cost", best); trace_access_hash.add("chosen", true); @@ -12415,7 +12411,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) below to check if we should use 'quick' instead. */ DBUG_PRINT("info", ("Item_int")); - tmp= (Item*) &Item_true; + tmp= (Item*) Item_true; } } @@ -12963,9 +12959,9 @@ static bool generate_derived_keys(DYNAMIC_ARRAY *keyuse_array) { KEYUSE *keyuse= dynamic_element(keyuse_array, 0, KEYUSE*); - uint elements= keyuse_array->elements; + size_t elements= keyuse_array->elements; TABLE *prev_table= 0; - for (uint i= 0; i < elements; i++, keyuse++) + for (size_t i= 0; i < elements; i++, keyuse++) { if (!keyuse->table) break; @@ -16040,7 +16036,7 @@ COND *Item_cond_and::build_equal_items(THD *thd, if (!cond_args->elements && !cond_equal.current_level.elements && !eq_list.elements) - return (Item*) &Item_true; + return (Item*) Item_true; List_iterator_fast<Item_equal> it(cond_equal.current_level); while ((item_equal= it++)) @@ -16147,7 +16143,7 @@ COND *Item_func_eq::build_equal_items(THD *thd, Item_equal *item_equal; int n= cond_equal.current_level.elements + eq_list.elements; if (n == 0) - return (Item*) &Item_true; + return (Item*) Item_true; else if (n == 1) { if ((item_equal= cond_equal.current_level.pop())) @@ -16551,7 +16547,7 @@ Item *eliminate_item_equal(THD *thd, COND *cond, COND_EQUAL *upper_levels, List<Item> eq_list; Item_func_eq *eq_item= 0; if (((Item *) item_equal)->const_item() && !item_equal->val_int()) - return (Item*) &Item_false; + return (Item*) Item_false; Item *item_const= item_equal->get_const(); Item_equal_fields_iterator it(*item_equal); Item *head; @@ -16696,7 +16692,7 @@ Item *eliminate_item_equal(THD *thd, COND *cond, COND_EQUAL *upper_levels, switch (eq_list.elements) { case 0: - res= cond ? cond : (Item*) &Item_true; + res= cond ? cond : (Item*) Item_true; break; case 1: if (!cond || cond->is_bool_literal()) @@ -16949,9 +16945,9 @@ static void update_const_equal_items(THD *thd, COND *cond, JOIN_TAB *tab, Item *item; while ((item= li++)) update_const_equal_items(thd, item, tab, - (((Item_cond*) cond)->top_level() && - ((Item_cond*) cond)->functype() == - Item_func::COND_AND_FUNC)); + cond->is_top_level_item() && + ((Item_cond*) cond)->functype() == + Item_func::COND_AND_FUNC); } else if (cond->type() == Item::FUNC_ITEM && ((Item_func*) cond)->functype() == Item_func::MULT_EQUAL_FUNC) @@ -18537,7 +18533,7 @@ Item_func_isnull::remove_eq_conds(THD *thd, Item::cond_result *cond_value, */ - Item *item0= (Item*) &Item_false; + Item *item0= (Item*) Item_false; Item *eq_cond= new(thd->mem_root) Item_func_eq(thd, args[0], item0); if (!eq_cond) return this; @@ -19108,20 +19104,19 @@ setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps, uint field_count) DBUG_ASSERT(table->s->virtual_fields == 0); - my_bitmap_init(&table->def_read_set, (my_bitmap_map*) bitmaps, field_count, - FALSE); + my_bitmap_init(&table->def_read_set, (my_bitmap_map*) bitmaps, field_count); bitmaps+= bitmap_size; my_bitmap_init(&table->tmp_set, - (my_bitmap_map*) bitmaps, field_count, FALSE); + (my_bitmap_map*) bitmaps, field_count); bitmaps+= bitmap_size; my_bitmap_init(&table->eq_join_set, - (my_bitmap_map*) bitmaps, field_count, FALSE); + (my_bitmap_map*) bitmaps, field_count); bitmaps+= bitmap_size; my_bitmap_init(&table->cond_set, - (my_bitmap_map*) bitmaps, field_count, FALSE); + (my_bitmap_map*) bitmaps, field_count); bitmaps+= bitmap_size; my_bitmap_init(&table->has_value_set, - (my_bitmap_map*) bitmaps, field_count, FALSE); + (my_bitmap_map*) bitmaps, field_count); /* write_set and all_set are copies of read_set */ table->def_write_set= table->def_read_set; table->s->all_set= table->def_read_set; @@ -19244,7 +19239,7 @@ TABLE *Create_tmp_table::start(THD *thd, (ulong) m_rows_limit, MY_TEST(m_group))); if (use_temp_pool && !(test_flags & TEST_KEEP_TMP_TABLES)) - m_temp_pool_slot = bitmap_lock_set_next(&temp_pool); + m_temp_pool_slot = temp_pool_set_next(); if (m_temp_pool_slot != MY_BIT_NONE) // we got a slot sprintf(path, "%s-%s-%lx-%i", tmp_file_prefix, param->tmp_name, @@ -20155,7 +20150,7 @@ void Create_tmp_table::cleanup_on_failure(THD *thd, TABLE *table) if (table) free_tmp_table(thd, table); if (m_temp_pool_slot != MY_BIT_NONE) - bitmap_lock_clear_bit(&temp_pool, m_temp_pool_slot); + temp_pool_clear_bit(m_temp_pool_slot); } @@ -20922,7 +20917,7 @@ free_tmp_table(THD *thd, TABLE *entry) (*ptr)->free(); if (entry->temp_pool_slot != MY_BIT_NONE) - bitmap_lock_clear_bit(&temp_pool, entry->temp_pool_slot); + temp_pool_clear_bit(entry->temp_pool_slot); plugin_unlock(0, entry->s->db_plugin); entry->alias.free(); @@ -21618,7 +21613,7 @@ sub_select(JOIN *join,JOIN_TAB *join_tab,bool end_of_records) if (join_tab->on_precond && !join_tab->on_precond->val_int()) rc= NESTED_LOOP_NO_MORE_ROWS; } - join->thd->get_stmt_da()->reset_current_row_for_warning(); + join->thd->get_stmt_da()->reset_current_row_for_warning(1); if (rc != NESTED_LOOP_NO_MORE_ROWS && (rc= join_tab_execution_startup(join_tab)) < 0) @@ -29490,7 +29485,7 @@ JOIN::reoptimize(Item *added_where, table_map join_tables, { DYNAMIC_ARRAY added_keyuse; SARGABLE_PARAM *sargables= 0; /* Used only as a dummy parameter. */ - uint org_keyuse_elements; + size_t org_keyuse_elements; /* Re-run the REF optimizer to take into account the new conditions. */ if (update_ref_and_keys(thd, &added_keyuse, join_tab, table_count, added_where, @@ -29512,7 +29507,7 @@ JOIN::reoptimize(Item *added_where, table_map join_tables, reset_query_plan(); if (!keyuse.buffer && - my_init_dynamic_array(thd->mem_root->m_psi_key, &keyuse, sizeof(KEYUSE), + my_init_dynamic_array(thd->mem_root->psi_key, &keyuse, sizeof(KEYUSE), 20, 64, MYF(MY_THREAD_SPECIFIC))) { delete_dynamic(&added_keyuse); @@ -29885,6 +29880,7 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, DBUG_ASSERT (ref_key != (int) nr); possible_key.add("can_resolve_order", true); + possible_key.add("direction", direction); bool is_covering= (table->covering_keys.is_set(nr) || (table->file->index_flags(nr, 0, 1) & HA_CLUSTERED_INDEX)); @@ -30441,7 +30437,7 @@ AGGR_OP::end_send() Reset the counter before copying rows from internal temporary table to INSERT table. */ - join_tab->join->thd->get_stmt_da()->reset_current_row_for_warning(); + join_tab->join->thd->get_stmt_da()->reset_current_row_for_warning(1); while (rc == NESTED_LOOP_OK) { int error; @@ -30658,7 +30654,7 @@ void JOIN::make_notnull_conds_for_range_scans() Found a IS NULL conjunctive predicate for a null-rejected field in the WHERE clause */ - conds= (Item*) &Item_false; + conds= (Item*) Item_false; cond_equal= 0; impossible_where= true; DBUG_VOID_RETURN; @@ -30681,7 +30677,7 @@ void JOIN::make_notnull_conds_for_range_scans() Found a IS NULL conjunctive predicate for a null-rejected field of the inner table of an outer join with ON expression tbl->on_expr */ - tbl->on_expr= (Item*) &Item_false; + tbl->on_expr= (Item*) Item_false; } } } @@ -30833,7 +30829,7 @@ void build_notnull_conds_for_inner_nest_of_outer_join(JOIN *join, if (used_tables && build_notnull_conds_for_range_scans(join, nest_tbl->on_expr, used_tables)) { - nest_tbl->on_expr= (Item*) &Item_false; + nest_tbl->on_expr= (Item*) Item_false; } li.rewind(); @@ -30847,7 +30843,7 @@ void build_notnull_conds_for_inner_nest_of_outer_join(JOIN *join, } else if (build_notnull_conds_for_range_scans(join, tbl->on_expr, tbl->table->map)) - tbl->on_expr= (Item*) &Item_false; + tbl->on_expr= (Item*) Item_false; } } } diff --git a/sql/sql_show.cc b/sql/sql_show.cc index c163d0d5fbe..0cc05ebdfcc 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2009, 2022, MariaDB + Copyright (c) 2009, 2023, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -231,6 +231,9 @@ static my_bool show_plugins(THD *thd, plugin_ref plugin, case PLUGIN_IS_DISABLED: table->field[2]->store(STRING_WITH_LEN("DISABLED"), cs); break; + case PLUGIN_IS_DYING: + table->field[2]->store(STRING_WITH_LEN("INACTIVE"), cs); + break; case PLUGIN_IS_FREED: // filtered in fill_plugins, used in fill_all_plugins table->field[2]->store(STRING_WITH_LEN("NOT INSTALLED"), cs); break; @@ -324,7 +327,7 @@ int fill_plugins(THD *thd, TABLE_LIST *tables, COND *cond) TABLE *table= tables->table; if (plugin_foreach_with_mask(thd, show_plugins, MYSQL_ANY_PLUGIN, - ~(PLUGIN_IS_FREED | PLUGIN_IS_DYING), table)) + ~PLUGIN_IS_FREED, table)) DBUG_RETURN(1); DBUG_RETURN(0); @@ -354,7 +357,7 @@ int fill_all_plugins(THD *thd, TABLE_LIST *tables, COND *cond) plugin_dl_foreach(thd, 0, show_plugins, table); const char *wstr= lookup.db_value.str, *wend= wstr + lookup.db_value.length; - for (uint i=0; i < (uint) dirp->number_of_files; i++) + for (size_t i=0; i < dirp->number_of_files; i++) { FILEINFO *file= dirp->dir_entry+i; LEX_CSTRING dl= { file->name, strlen(file->name) }; @@ -952,7 +955,7 @@ find_files(THD *thd, Dynamic_array<LEX_CSTRING*> *files, LEX_CSTRING *db, if (!db) /* Return databases */ { - for (uint i=0; i < (uint) dirp->number_of_files; i++) + for (size_t i=0; i < dirp->number_of_files; i++) { FILEINFO *file= dirp->dir_entry+i; #ifdef USE_SYMDIR @@ -2360,6 +2363,9 @@ int show_create_table_ex(THD *thd, TABLE_LIST *table_list, packet->append_parenthesized((long) key_part->length / key_part->field->charset()->mbmaxlen); } + if (table->file->index_flags(i, j, 0) & HA_READ_ORDER && + key_part->key_part_flag & HA_REVERSE_SORT) /* same in SHOW KEYS */ + packet->append(STRING_WITH_LEN(" DESC")); } if (key_info->without_overlaps) @@ -2446,7 +2452,7 @@ int show_create_table_ex(THD *thd, TABLE_LIST *table_list, add_table_options(thd, table, create_info_arg, table_list->schema_table != 0, 0, packet); - if (DBUG_EVALUATE_IF("sysvers_hide", 0, table->versioned())) + if (!DBUG_IF("sysvers_hide") && table->versioned()) packet->append(STRING_WITH_LEN(" WITH SYSTEM VERSIONING")); #ifdef WITH_PARTITION_STORAGE_ENGINE @@ -6755,7 +6761,7 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, for (uint i=0 ; i < show_table->s->keys ; i++,key_info++) { if ((key_info->flags & HA_INVISIBLE_KEY) && - DBUG_EVALUATE_IF("test_invisible_index", 0, 1)) + !DBUG_IF("test_invisible_index")) continue; KEY_PART_INFO *key_part= key_info->key_part; LEX_CSTRING *str; @@ -6763,7 +6769,7 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, for (uint j=0 ; j < key_info->user_defined_key_parts ; j++,key_part++) { if (key_part->field->invisible >= INVISIBLE_SYSTEM && - DBUG_EVALUATE_IF("test_completely_invisible", 0, 1)) + !DBUG_IF("test_completely_invisible")) { /* NOTE: we will get SEQ_IN_INDEX gap inside the result if this key_part @@ -7423,13 +7429,7 @@ static void store_schema_partitions_record(THD *thd, TABLE *schema_table, table->field[23]->store(STRING_WITH_LEN("default"), cs); table->field[24]->set_notnull(); - if (part_elem->tablespace_name) - table->field[24]->store(part_elem->tablespace_name, - strlen(part_elem->tablespace_name), cs); - else - { - table->field[24]->set_null(); - } + table->field[24]->set_null(); // Tablespace } return; } @@ -8245,7 +8245,6 @@ TABLE *create_schema_table(THD *thd, TABLE_LIST *table_list) bool keep_row_order; TMP_TABLE_PARAM *tmp_table_param; SELECT_LEX *select_lex; - my_bitmap_map *bitmaps; DBUG_ENTER("create_schema_table"); for (; !fields->end_marker(); fields++) @@ -8266,8 +8265,9 @@ TABLE *create_schema_table(THD *thd, TABLE_LIST *table_list) table_list->alias, !need_all_fields, keep_row_order))) DBUG_RETURN(0); - bitmaps= (my_bitmap_map*) thd->alloc(bitmap_buffer_size(field_count)); - my_bitmap_init(&table->def_read_set, bitmaps, field_count, FALSE); + my_bitmap_map* bitmaps= + (my_bitmap_map*) thd->alloc(bitmap_buffer_size(field_count)); + my_bitmap_init(&table->def_read_set, bitmaps, field_count); table->read_set= &table->def_read_set; bitmap_clear_all(table->read_set); table_list->schema_table_param= tmp_table_param; @@ -9860,23 +9860,17 @@ int initialize_schema_table(st_plugin_int *plugin) int finalize_schema_table(st_plugin_int *plugin) { + int deinit_status= 0; ST_SCHEMA_TABLE *schema_table= (ST_SCHEMA_TABLE *)plugin->data; DBUG_ENTER("finalize_schema_table"); if (schema_table) { if (plugin->plugin->deinit) - { - DBUG_PRINT("info", ("Deinitializing plugin: '%s'", plugin->name.str)); - if (plugin->plugin->deinit(NULL)) - { - DBUG_PRINT("warning", ("Plugin '%s' deinit function returned error.", - plugin->name.str)); - } - } + deinit_status= plugin->plugin->deinit(NULL); my_free(schema_table); } - DBUG_RETURN(0); + DBUG_RETURN(deinit_status); } diff --git a/sql/sql_signal.cc b/sql/sql_signal.cc index 8e973f9b0b3..4e86cc4d782 100644 --- a/sql/sql_signal.cc +++ b/sql/sql_signal.cc @@ -44,6 +44,7 @@ const LEX_CSTRING Diag_condition_item_names[]= { STRING_WITH_LEN("CURSOR_NAME") }, { STRING_WITH_LEN("MESSAGE_TEXT") }, { STRING_WITH_LEN("MYSQL_ERRNO") }, + { STRING_WITH_LEN("ROW_NUMBER") }, { STRING_WITH_LEN("CONDITION_IDENTIFIER") }, { STRING_WITH_LEN("CONDITION_NUMBER") }, @@ -309,6 +310,26 @@ int Sql_cmd_common_signal::eval_signal_informations(THD *thd, Sql_condition *con cond->m_sql_errno= (int) code; } + set= m_set_signal_information.m_item[DIAG_ROW_NUMBER]; + if (set != NULL) + { + if (set->is_null()) + { + thd->raise_error_printf(ER_WRONG_VALUE_FOR_VAR, + "ROW_NUMBER", "NULL"); + goto end; + } + longlong row_number_value= set->val_int(); + if (row_number_value < 0) + { + str= set->val_str(& str_value); + thd->raise_error_printf(ER_WRONG_VALUE_FOR_VAR, + "ROW_NUMBER", str->c_ptr_safe()); + goto end; + } + cond->m_row_number= (ulong) row_number_value; + } + /* The various item->val_xxx() methods don't return an error code, but flag thd in case of failure. @@ -419,7 +440,8 @@ bool Sql_cmd_resignal::execute(THD *thd) DBUG_RETURN(result); } - Sql_condition signaled_err(thd->mem_root, *signaled, signaled->message); + Sql_condition signaled_err(thd->mem_root, *signaled, signaled->message, + signaled->m_row_number); if (m_cond) { diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index d0b6ac20d1d..d300fe43b67 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -28,11 +28,15 @@ #include "sql_base.h" #include "key.h" #include "sql_statistics.h" +#include "opt_histogram_json.h" #include "opt_range.h" #include "uniques.h" #include "sql_show.h" #include "sql_partition.h" +#include <vector> +#include <string> + /* The system variable 'use_stat_tables' can take one of the following values: @@ -57,8 +61,11 @@ the collected statistics in the persistent statistical tables only when the value of the variable 'use_stat_tables' is not equal to "never". -*/ - +*/ + +Histogram_base *create_histogram(MEM_ROOT *mem_root, Histogram_type hist_type, + THD *owner); + /* Currently there are only 3 persistent statistical tables */ static const uint STATISTICS_TABLES= 3; @@ -178,12 +185,12 @@ TABLE_FIELD_TYPE column_stat_fields[COLUMN_STAT_N_FIELDS] = }, { { STRING_WITH_LEN("hist_type") }, - { STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB')") }, + { STRING_WITH_LEN("enum('SINGLE_PREC_HB','DOUBLE_PREC_HB','JSON_HB')") }, { STRING_WITH_LEN("utf8mb3") } }, { { STRING_WITH_LEN("histogram") }, - { STRING_WITH_LEN("varbinary(255)") }, + { STRING_WITH_LEN("longblob") }, { NULL, 0 } } }; @@ -307,7 +314,7 @@ public: inline void init(THD *thd, Field * table_field); inline bool add(); - inline void finish(ha_rows rows, double sample_fraction); + inline bool finish(MEM_ROOT *mem_root, ha_rows rows, double sample_fraction); inline void cleanup(); }; @@ -1064,15 +1071,23 @@ public: stat_field->store(stats->get_avg_frequency()); break; case COLUMN_STAT_HIST_SIZE: - stat_field->store(stats->histogram.get_size()); + // Note: this is dumb. the histogram size is stored with the + // histogram! + stat_field->store(stats->histogram? + stats->histogram->get_size() : 0); break; case COLUMN_STAT_HIST_TYPE: - stat_field->store(stats->histogram.get_type() + 1); + if (stats->histogram) + stat_field->store(stats->histogram->get_type() + 1); + else + stat_field->set_null(); break; case COLUMN_STAT_HISTOGRAM: - stat_field->store((char *)stats->histogram.get_values(), - stats->histogram.get_size(), &my_charset_bin); - break; + if (stats->histogram) + stats->histogram->serialize(stat_field); + else + stat_field->set_null(); + break; } } } @@ -1100,6 +1115,8 @@ public: void get_stat_values() { table_field->read_stats->set_all_nulls(); + // default: hist_type=NULL means there's no histogram + table_field->read_stats->histogram_type_on_disk= INVALID_HISTOGRAM; if (table_field->read_stats->min_value) table_field->read_stats->min_value->set_null(); @@ -1111,7 +1128,7 @@ public: char buff[MAX_FIELD_WIDTH]; String val(buff, sizeof(buff), &my_charset_bin); - for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HIST_TYPE; i++) + for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HISTOGRAM; i++) { Field *stat_field= stat_table->field[i]; @@ -1155,13 +1172,28 @@ public: table_field->read_stats->set_avg_frequency(stat_field->val_real()); break; case COLUMN_STAT_HIST_SIZE: - table_field->read_stats->histogram.set_size(stat_field->val_int()); + /* + Ignore the contents of mysql.column_stats.hist_size. We take the + size from the mysql.column_stats.histogram column, itself. + */ break; case COLUMN_STAT_HIST_TYPE: - Histogram_type hist_type= (Histogram_type) (stat_field->val_int() - - 1); - table_field->read_stats->histogram.set_type(hist_type); - break; + { + /* + Save the histogram type. The histogram itself will be read in + read_histograms_for_table(). + */ + Histogram_type hist_type= (Histogram_type) (stat_field->val_int() - + 1); + table_field->read_stats->histogram_type_on_disk= hist_type; + break; + } + case COLUMN_STAT_HISTOGRAM: + /* + Do nothing here: we take the histogram length from the 'histogram' + column itself + */ + break; } } } @@ -1182,9 +1214,9 @@ public: The method assumes that the value of histogram size and the pointer to the histogram location has been already set in the fields size and values of read_stats->histogram. - */ + */ - void get_histogram_value() + Histogram_base * load_histogram(MEM_ROOT *mem_root) { if (find_stat()) { @@ -1194,14 +1226,60 @@ public: Field *stat_field= stat_table->field[fldno]; table_field->read_stats->set_not_null(fldno); stat_field->val_str(&val); - memcpy(table_field->read_stats->histogram.get_values(), - val.ptr(), table_field->read_stats->histogram.get_size()); + Histogram_type hist_type= + table_field->read_stats->histogram_type_on_disk; + + Histogram_base *hist; + if (!(hist= create_histogram(mem_root, hist_type, NULL))) + return NULL; + Field *field= table->field[table_field->field_index]; + if (!hist->parse(mem_root, db_name->str, table_name->str, + field, hist_type, + val.ptr(), val.length())) + { + table_field->read_stats->histogram= hist; + return hist; + } + else + delete hist; } + return NULL; } - }; +bool Histogram_binary::parse(MEM_ROOT *mem_root, const char*, const char*, + Field*, Histogram_type type_arg, + const char *hist_data, size_t hist_data_len) +{ + /* On-disk an in-memory formats are the same. Just copy the data. */ + type= type_arg; + size= (uint8) hist_data_len; // 'size' holds the size of histogram in bytes + if (!(values= (uchar*)alloc_root(mem_root, hist_data_len))) + return true; + + memcpy(values, hist_data, hist_data_len); + return false; +} + +/* + Save the histogram data info a table field. +*/ +void Histogram_binary::serialize(Field *field) +{ + field->store((char*)values, size, &my_charset_bin); +} + +void Histogram_binary::init_for_collection(MEM_ROOT *mem_root, + Histogram_type htype_arg, + ulonglong size_arg) +{ + type= htype_arg; + values= (uchar*)alloc_root(mem_root, (size_t)size_arg); + size= (uint8) size_arg; +} + + /* An object of the class Index_stat is created to read statistical data on tables from the statistical table table_stat, to update @@ -1512,62 +1590,39 @@ public: } }; -/* - Histogram_builder is a helper class that is used to build histograms - for columns -*/ - -class Histogram_builder +class Histogram_binary_builder : public Histogram_builder { - Field *column; /* table field for which the histogram is built */ - uint col_length; /* size of this field */ - ha_rows records; /* number of records the histogram is built for */ Field *min_value; /* pointer to the minimal value for the field */ Field *max_value; /* pointer to the maximal value for the field */ - Histogram *histogram; /* the histogram location */ + Histogram_binary *histogram; /* the histogram location */ uint hist_width; /* the number of points in the histogram */ double bucket_capacity; /* number of rows in a bucket of the histogram */ uint curr_bucket; /* number of the current bucket to be built */ - ulonglong count; /* number of values retrieved */ - ulonglong count_distinct; /* number of distinct values retrieved */ - /* number of distinct values that occured only once */ - ulonglong count_distinct_single_occurence; -public: - Histogram_builder(Field *col, uint col_len, ha_rows rows) - : column(col), col_length(col_len), records(rows) +public: + Histogram_binary_builder(Field *col, uint col_len, ha_rows rows) + : Histogram_builder(col, col_len, rows) { Column_statistics *col_stats= col->collected_stats; min_value= col_stats->min_value; max_value= col_stats->max_value; - histogram= &col_stats->histogram; + histogram= (Histogram_binary*)col_stats->histogram; hist_width= histogram->get_width(); bucket_capacity= (double) records / (hist_width + 1); curr_bucket= 0; - count= 0; - count_distinct= 0; - count_distinct_single_occurence= 0; } - ulonglong get_count_distinct() const { return count_distinct; } - ulonglong get_count_single_occurence() const + int next(void *elem, element_count elem_cnt) override { - return count_distinct_single_occurence; - } - - int next(void *elem, element_count elem_cnt) - { - count_distinct++; - if (elem_cnt == 1) - count_distinct_single_occurence++; - count+= elem_cnt; + counters.next(elem, elem_cnt); + ulonglong count= counters.get_count(); if (curr_bucket == hist_width) return 0; if (count > bucket_capacity * (curr_bucket + 1)) { column->store_field_value((uchar *) elem, col_length); histogram->set_value(curr_bucket, - column->pos_in_interval(min_value, max_value)); + column->pos_in_interval(min_value, max_value)); curr_bucket++; while (curr_bucket != hist_width && count > bucket_capacity * (curr_bucket + 1)) @@ -1578,25 +1633,51 @@ public: } return 0; } + void finalize() override {} }; +Histogram_builder *Histogram_binary::create_builder(Field *col, uint col_len, + ha_rows rows) +{ + return new Histogram_binary_builder(col, col_len, rows); +} + + +Histogram_base *create_histogram(MEM_ROOT *mem_root, Histogram_type hist_type, + THD *owner) +{ + Histogram_base *res= NULL; + switch (hist_type) { + case SINGLE_PREC_HB: + case DOUBLE_PREC_HB: + res= new Histogram_binary(); + break; + case JSON_HB: + res= new Histogram_json_hb(); + break; + default: + DBUG_ASSERT(0); + } + + if (res) + res->set_owner(owner); + return res; +} + + C_MODE_START -int histogram_build_walk(void *elem, element_count elem_cnt, void *arg) +static int histogram_build_walk(void *elem, element_count elem_cnt, void *arg) { Histogram_builder *hist_builder= (Histogram_builder *) arg; return hist_builder->next(elem, elem_cnt); } - - -static int count_distinct_single_occurence_walk(void *elem, - element_count count, void *arg) +int basic_stats_collector_walk(void *elem, element_count count, + void *arg) { - ((ulonglong*)arg)[0]+= 1; - if (count == 1) - ((ulonglong*)arg)[1]+= 1; + ((Basic_stats_collector*)arg)->next(elem, count); return 0; } @@ -1681,23 +1762,35 @@ public: */ void walk_tree() { - ulonglong counts[2] = {0, 0}; - tree->walk(table_field->table, - count_distinct_single_occurence_walk, counts); - distincts= counts[0]; - distincts_single_occurence= counts[1]; + Basic_stats_collector stats_collector; + tree->walk(table_field->table, basic_stats_collector_walk, + (void*)&stats_collector ); + distincts= stats_collector.get_count_distinct(); + distincts_single_occurence= stats_collector.get_count_single_occurence(); } /* @brief Calculate a histogram of the tree */ - void walk_tree_with_histogram(ha_rows rows) + bool walk_tree_with_histogram(ha_rows rows) { - Histogram_builder hist_builder(table_field, tree_key_length, rows); - tree->walk(table_field->table, histogram_build_walk, (void *) &hist_builder); - distincts= hist_builder.get_count_distinct(); - distincts_single_occurence= hist_builder.get_count_single_occurence(); + Histogram_base *hist= table_field->collected_stats->histogram; + Histogram_builder *hist_builder= + hist->create_builder(table_field, tree_key_length, rows); + + if (tree->walk(table_field->table, histogram_build_walk, + (void*)hist_builder)) + { + delete hist_builder; + return true; // Error + } + hist_builder->finalize(); + distincts= hist_builder->counters.get_count_distinct(); + distincts_single_occurence= hist_builder->counters. + get_count_single_occurence(); + delete hist_builder; + return false; } ulonglong get_count_distinct() @@ -1712,20 +1805,11 @@ public: /* @brief - Get the size of the histogram in bytes built for table_field - */ - uint get_hist_size() - { - return table_field->collected_stats->histogram.get_size(); - } - - /* - @brief Get the pointer to the histogram built for table_field */ - uchar *get_histogram() + Histogram_base *get_histogram() { - return table_field->collected_stats->histogram.get_values(); + return table_field->collected_stats->histogram; } }; @@ -2125,26 +2209,13 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) ulonglong *idx_avg_frequency= (ulonglong*) alloc_root(&table->mem_root, sizeof(ulonglong) * key_parts); - uint hist_size= thd->variables.histogram_size; - Histogram_type hist_type= (Histogram_type) (thd->variables.histogram_type); - uchar *histogram= NULL; - if (hist_size > 0) - { - if ((histogram= (uchar *) alloc_root(&table->mem_root, - hist_size * columns))) - bzero(histogram, hist_size * columns); - - } - - if (!table_stats || !column_stats || !index_stats || !idx_avg_frequency || - (hist_size && !histogram)) + if (!table_stats || !column_stats || !index_stats || !idx_avg_frequency) DBUG_RETURN(1); table->collected_stats= table_stats; table_stats->column_stats= column_stats; table_stats->index_stats= index_stats; table_stats->idx_avg_frequency= idx_avg_frequency; - table_stats->histograms= histogram; memset(column_stats, 0, sizeof(Column_statistics) * columns); @@ -2152,10 +2223,7 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) { if (bitmap_is_set(table->read_set, (*field_ptr)->field_index)) { - column_stats->histogram.set_size(hist_size); - column_stats->histogram.set_type(hist_type); - column_stats->histogram.set_values(histogram); - histogram+= hist_size; + column_stats->histogram = NULL; (*field_ptr)->collected_stats= column_stats++; } } @@ -2177,6 +2245,25 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) DBUG_RETURN(0); } +/* + Free the "local" statistics for table. + We only free the statistics that is not on MEM_ROOT and needs to be + explicitly freed. +*/ +void free_statistics_for_table(THD *thd, TABLE *table) +{ + for (Field **field_ptr= table->field; *field_ptr; field_ptr++) + { + // Only delete the histograms that are exclusivly owned by this thread + if ((*field_ptr)->collected_stats && + (*field_ptr)->collected_stats->histogram && + (*field_ptr)->collected_stats->histogram->get_owner() == thd) + { + delete (*field_ptr)->collected_stats->histogram; + (*field_ptr)->collected_stats->histogram= NULL; + } + } +} /** @brief @@ -2383,7 +2470,8 @@ bool Column_statistics_collected::add() */ inline -void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) +bool Column_statistics_collected::finish(MEM_ROOT *mem_root, ha_rows rows, + double sample_fraction) { double val; @@ -2401,13 +2489,32 @@ void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) } if (count_distinct) { - uint hist_size= count_distinct->get_hist_size(); + uint hist_size= current_thd->variables.histogram_size; + Histogram_type hist_type= + (Histogram_type) (current_thd->variables.histogram_type); + bool have_histogram= false; + if (hist_size != 0 && hist_type != INVALID_HISTOGRAM) + { + have_histogram= true; + histogram= create_histogram(mem_root, hist_type, current_thd); + histogram->init_for_collection(mem_root, hist_type, hist_size); + } /* Compute cardinality statistics and optionally histogram. */ - if (hist_size == 0) + if (!have_histogram) count_distinct->walk_tree(); else - count_distinct->walk_tree_with_histogram(rows - nulls); + { + if (count_distinct->walk_tree_with_histogram(rows - nulls)) + { + delete histogram; + histogram= NULL; + + delete count_distinct; + count_distinct= NULL; + return true; // Error + } + } ulonglong distincts= count_distinct->get_count_distinct(); ulonglong distincts_single_occurence= @@ -2442,15 +2549,14 @@ void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) set_not_null(COLUMN_STAT_AVG_FREQUENCY); } else - hist_size= 0; - histogram.set_size(hist_size); + have_histogram= false; + set_not_null(COLUMN_STAT_HIST_SIZE); - if (hist_size && distincts) + if (have_histogram && distincts && histogram) { set_not_null(COLUMN_STAT_HIST_TYPE); - histogram.set_values(count_distinct->get_histogram()); set_not_null(COLUMN_STAT_HISTOGRAM); - } + } delete count_distinct; count_distinct= NULL; } @@ -2459,7 +2565,8 @@ void Column_statistics_collected::finish(ha_rows rows, double sample_fraction) val= 1.0; set_avg_frequency(val); set_not_null(COLUMN_STAT_AVG_FREQUENCY); - } + } + return false; } @@ -2708,7 +2815,10 @@ int collect_statistics_for_table(THD *thd, TABLE *table) continue; bitmap_set_bit(table->write_set, table_field->field_index); if (!rc) - table_field->collected_stats->finish(rows, sample_fraction); + { + rc= table_field->collected_stats->finish(&table->mem_root, rows, + sample_fraction); + } else table_field->collected_stats->cleanup(); } @@ -2788,7 +2898,7 @@ int update_statistics_for_table(THD *thd, TABLE *table) start_new_trans new_trans(thd); - if (open_stat_tables(thd, tables, TRUE)) + if ((open_stat_tables(thd, tables, TRUE))) DBUG_RETURN(rc); save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); @@ -2914,16 +3024,17 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) /* Read statistics from the statistical table column_stats */ stat_table= stat_tables[COLUMN_STAT].table; - ulong total_hist_size= 0; + bool have_histograms= false; Column_stat column_stat(stat_table, table); for (field_ptr= table_share->field; *field_ptr; field_ptr++) { table_field= *field_ptr; column_stat.set_key_fields(table_field); column_stat.get_stat_values(); - total_hist_size+= table_field->read_stats->histogram.get_size(); + if (table_field->read_stats->histogram_type_on_disk != INVALID_HISTOGRAM) + have_histograms= true; } - table_share->stats_cb.total_hist_size= total_hist_size; + table_share->stats_cb.have_histograms= have_histograms; /* Read statistics from the statistical table index_stats */ stat_table= stat_tables[INDEX_STAT].table; @@ -3019,6 +3130,9 @@ void delete_stat_values_for_table_share(TABLE_SHARE *table_share) delete column_stats->max_value; column_stats->max_value= NULL; } + + delete column_stats->histogram; + column_stats->histogram=NULL; } } @@ -3063,28 +3177,28 @@ int read_histograms_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) if (stats_cb->start_histograms_load()) { - uchar *histogram= (uchar *) alloc_root(&stats_cb->mem_root, - stats_cb->total_hist_size); - if (!histogram) - { - stats_cb->abort_histograms_load(); - DBUG_RETURN(1); - } - memset(histogram, 0, stats_cb->total_hist_size); - Column_stat column_stat(stat_tables[COLUMN_STAT].table, table); + + /* + The process of histogram loading makes use of the field it is for. Mark + all fields as readable/writable in order to allow that. + */ + MY_BITMAP *old_sets[2]; + dbug_tmp_use_all_columns(table, old_sets, &table->read_set, &table->write_set); + for (Field **field_ptr= table->s->field; *field_ptr; field_ptr++) { Field *table_field= *field_ptr; - if (uint hist_size= table_field->read_stats->histogram.get_size()) + if (table_field->read_stats->histogram_type_on_disk != INVALID_HISTOGRAM) { column_stat.set_key_fields(table_field); - table_field->read_stats->histogram.set_values(histogram); - column_stat.get_histogram_value(); - histogram+= hist_size; + table_field->read_stats->histogram= + column_stat.load_histogram(&stats_cb->mem_root); } } stats_cb->end_histograms_load(); + + dbug_tmp_restore_column_maps(&table->read_set, &table->write_set, old_sets); } table->histograms_are_read= true; DBUG_RETURN(0); @@ -3773,15 +3887,11 @@ double get_column_range_cardinality(Field *field, if (avg_frequency > 1.0 + 0.000001 && col_stats->min_max_values_are_provided()) { - Histogram *hist= &col_stats->histogram; - if (hist->is_usable(thd)) + Histogram_base *hist = col_stats->histogram; + if (hist && hist->is_usable(thd)) { - store_key_image_to_rec(field, (uchar *) min_endp->key, - field->key_length()); - double pos= field->pos_in_interval(col_stats->min_value, - col_stats->max_value); res= col_non_nulls * - hist->point_selectivity(pos, + hist->point_selectivity(field, min_endp, avg_frequency / col_non_nulls); } } @@ -3796,34 +3906,41 @@ double get_column_range_cardinality(Field *field, { if (col_stats->min_max_values_are_provided()) { - double sel, min_mp_pos, max_mp_pos; - - if (min_endp && !(field->null_ptr && min_endp->key[0])) + Histogram_base *hist= col_stats->histogram; + double avg_frequency= col_stats->get_avg_frequency(); + double sel; + if (hist && hist->is_usable(thd)) { - store_key_image_to_rec(field, (uchar *) min_endp->key, - field->key_length()); - min_mp_pos= field->pos_in_interval(col_stats->min_value, - col_stats->max_value); + sel= hist->range_selectivity(field, min_endp, max_endp, + avg_frequency / col_non_nulls); + res= col_non_nulls * sel; } else - min_mp_pos= 0.0; - if (max_endp) { - store_key_image_to_rec(field, (uchar *) max_endp->key, - field->key_length()); - max_mp_pos= field->pos_in_interval(col_stats->min_value, - col_stats->max_value); - } - else - max_mp_pos= 1.0; + double min_mp_pos, max_mp_pos; + if (min_endp && !(field->null_ptr && min_endp->key[0])) + { + store_key_image_to_rec(field, (uchar *) min_endp->key, + field->key_length()); + min_mp_pos= + field->pos_in_interval(col_stats->min_value, col_stats->max_value); + } + else + min_mp_pos= 0.0; + if (max_endp) + { + store_key_image_to_rec(field, (uchar *) max_endp->key, + field->key_length()); + max_mp_pos= + field->pos_in_interval(col_stats->min_value, col_stats->max_value); + } + else + max_mp_pos= 1.0; - Histogram *hist= &col_stats->histogram; - if (hist->is_usable(thd)) - sel= hist->range_selectivity(min_mp_pos, max_mp_pos); - else - sel= (max_mp_pos - min_mp_pos); - res= col_non_nulls * sel; - set_if_bigger(res, col_stats->get_avg_frequency()); + sel = (max_mp_pos - min_mp_pos); + res= col_non_nulls * sel; + set_if_bigger(res, avg_frequency); + } } else res= col_non_nulls; @@ -3833,13 +3950,13 @@ double get_column_range_cardinality(Field *field, return res; } - - /* Estimate selectivity of "col=const" using a histogram - @param pos Position of the "const" between column's min_value and - max_value. This is a number in [0..1] range. + @param field the field to estimate its selectivity. + + @param endpoint The constant + @param avg_sel Average selectivity of condition "col=const" in this table. It is calcuated as (#non_null_values / #distinct_values). @@ -3868,9 +3985,15 @@ double get_column_range_cardinality(Field *field, value. */ -double Histogram::point_selectivity(double pos, double avg_sel) +double Histogram_binary::point_selectivity(Field *field, key_range *endpoint, + double avg_sel) { double sel; + Column_statistics *col_stats= field->read_stats; + store_key_image_to_rec(field, (uchar *) endpoint->key, + field->key_length()); + double pos= field->pos_in_interval(col_stats->min_value, + col_stats->max_value); /* Find the bucket that contains the value 'pos'. */ uint min= find_bucket(pos, TRUE); uint pos_value= (uint) (pos * prec_factor()); @@ -3915,6 +4038,43 @@ double Histogram::point_selectivity(double pos, double avg_sel) return sel; } + +double Histogram_binary::range_selectivity(Field *field, + key_range *min_endp, + key_range *max_endp, + double avg_sel) +{ + double sel, min_mp_pos, max_mp_pos; + Column_statistics *col_stats= field->read_stats; + + if (min_endp && !(field->null_ptr && min_endp->key[0])) + { + store_key_image_to_rec(field, (uchar *) min_endp->key, + field->key_length()); + min_mp_pos= + field->pos_in_interval(col_stats->min_value, col_stats->max_value); + } + else + min_mp_pos= 0.0; + if (max_endp) + { + store_key_image_to_rec(field, (uchar *) max_endp->key, + field->key_length()); + max_mp_pos= + field->pos_in_interval(col_stats->min_value, col_stats->max_value); + } + else + max_mp_pos= 1.0; + + double bucket_sel= 1.0 / (get_width() + 1); + uint min= find_bucket(min_mp_pos, TRUE); + uint max= find_bucket(max_mp_pos, FALSE); + sel= bucket_sel * (max - min + 1); + + set_if_bigger(sel, avg_sel); + return sel; +} + /* Check whether the table is one of the persistent statistical tables. */ diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 35b3aa33acc..c0df15ea4ad 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -16,6 +16,9 @@ #ifndef SQL_STATISTICS_H #define SQL_STATISTICS_H +#include <vector> +#include <string> + /* For COMPLEMENTARY_FOR_QUERIES and PREFERABLY_FOR_QUERIES they are similar to the COMPLEMENTARY and PREFERABLY respectively except that @@ -42,7 +45,9 @@ typedef enum enum_histogram_type { SINGLE_PREC_HB, - DOUBLE_PREC_HB + DOUBLE_PREC_HB, + JSON_HB, + INVALID_HISTOGRAM } Histogram_type; enum enum_stat_tables @@ -120,6 +125,7 @@ int read_statistics_for_tables(THD *thd, TABLE_LIST *tables); int collect_statistics_for_table(THD *thd, TABLE *table); void delete_stat_values_for_table_share(TABLE_SHARE *table_share); int alloc_statistics_for_table(THD *thd, TABLE *table); +void free_statistics_for_table(THD *thd, TABLE *table); int update_statistics_for_table(THD *thd, TABLE *table); int delete_statistics_for_table(THD *thd, const LEX_CSTRING *db, const LEX_CSTRING *tab); int delete_statistics_for_column(THD *thd, TABLE *tab, Field *col); @@ -140,9 +146,82 @@ double get_column_range_cardinality(Field *field, bool is_stat_table(const LEX_CSTRING *db, LEX_CSTRING *table); bool is_eits_usable(Field* field); -class Histogram +class Histogram_builder; + +/* + Common base for all histograms +*/ +class Histogram_base { +public: + virtual bool parse(MEM_ROOT *mem_root, + const char *db_name, const char *table_name, + Field *field, Histogram_type type_arg, + const char *hist_data, size_t hist_data_len)= 0; + virtual void serialize(Field *to_field)= 0; + + virtual Histogram_type get_type()=0; + + virtual uint get_width()=0; + + /* + The creation-time workflow is: + * create a histogram + * init_for_collection() + * create_builder() + * feed the data to the builder + * serialize(); + */ + virtual void init_for_collection(MEM_ROOT *mem_root, Histogram_type htype_arg, + ulonglong size)=0; + virtual Histogram_builder *create_builder(Field *col, uint col_len, + ha_rows rows)=0; + + /* + This function checks that histograms should be usable only when + 1) the level of optimizer_use_condition_selectivity > 3 + */ + bool is_usable(THD *thd) + { + return thd->variables.optimizer_use_condition_selectivity > 3; + } + + + virtual double point_selectivity(Field *field, key_range *endpoint, + double avg_sel)=0; + virtual double range_selectivity(Field *field, key_range *min_endp, + key_range *max_endp, double avg_sel)=0; + + /* + Legacy: return the size of the histogram on disk. + This will be stored in mysql.column_stats.hist_size column. + The value is not really needed as one can look at + LENGTH(mysql.column_stats.histogram) directly. + */ + virtual uint get_size()=0; + virtual ~Histogram_base()= default; + + Histogram_base() : owner(NULL) {} + + /* + Memory management: a histogram may be (exclusively) "owned" by a particular + thread (done for histograms that are being collected). By default, a + histogram has owner==NULL and is not owned by any particular thread. + */ + THD *get_owner() { return owner; } + void set_owner(THD *thd) { owner=thd; } +private: + THD *owner; +}; + + +/* + A Height-balanced histogram that stores numeric fractions +*/ + +class Histogram_binary : public Histogram_base +{ private: Histogram_type type; uint8 size; /* Size of values array, in bytes */ @@ -155,22 +234,25 @@ private: return ((uint) (1 << 8) - 1); case DOUBLE_PREC_HB: return ((uint) (1 << 16) - 1); + default: + DBUG_ASSERT(0); } return 1; } public: - uint get_width() + uint get_width() override { switch (type) { case SINGLE_PREC_HB: return size; case DOUBLE_PREC_HB: return size / 2; + default: + DBUG_ASSERT(0); } return 0; } - private: uint get_value(uint i) { @@ -180,6 +262,8 @@ private: return (uint) (((uint8 *) values)[i]); case DOUBLE_PREC_HB: return (uint) uint2korr(values + i * 2); + default: + DBUG_ASSERT(0); } return 0; } @@ -224,70 +308,124 @@ private: } public: + uint get_size() override {return (uint)size;} - uint get_size() { return (uint) size; } - - Histogram_type get_type() { return type; } - - uchar *get_values() { return (uchar *) values; } + Histogram_type get_type() override { return type; } - void set_size (ulonglong sz) { size= (uint8) sz; } - - void set_type (Histogram_type t) { type= t; } - - void set_values (uchar *vals) { values= (uchar *) vals; } - - bool is_available() { return get_size() > 0 && get_values(); } - - /* - This function checks that histograms should be usable only when - 1) the level of optimizer_use_condition_selectivity > 3 - 2) histograms have been collected - */ - bool is_usable(THD *thd) - { - return thd->variables.optimizer_use_condition_selectivity > 3 && - is_available(); - } + bool parse(MEM_ROOT *mem_root, const char*, const char*, Field*, + Histogram_type type_arg, const char *hist_data, + size_t hist_data_len) override; + void serialize(Field *to_field) override; + void init_for_collection(MEM_ROOT *mem_root, Histogram_type htype_arg, + ulonglong size) override; + Histogram_builder *create_builder(Field *col, uint col_len, + ha_rows rows) override; void set_value(uint i, double val) { switch (type) { - case SINGLE_PREC_HB: + case SINGLE_PREC_HB: ((uint8 *) values)[i]= (uint8) (val * prec_factor()); return; case DOUBLE_PREC_HB: int2store(values + i * 2, val * prec_factor()); return; + default: + DBUG_ASSERT(0); + return; } } void set_prev_value(uint i) { switch (type) { - case SINGLE_PREC_HB: + case SINGLE_PREC_HB: ((uint8 *) values)[i]= ((uint8 *) values)[i-1]; return; case DOUBLE_PREC_HB: int2store(values + i * 2, uint2korr(values + i * 2 - 2)); return; + default: + DBUG_ASSERT(0); + return; } } - double range_selectivity(double min_pos, double max_pos) - { - double sel; - double bucket_sel= 1.0/(get_width() + 1); - uint min= find_bucket(min_pos, TRUE); - uint max= find_bucket(max_pos, FALSE); - sel= bucket_sel * (max - min + 1); - return sel; - } - + double range_selectivity(Field *field, key_range *min_endp, + key_range *max_endp, double avg_sel) override; + /* Estimate selectivity of "col=const" using a histogram */ - double point_selectivity(double pos, double avg_sel); + double point_selectivity(Field *field, key_range *endpoint, + double avg_sel) override; +}; + + +/* + This is used to collect the the basic statistics from a Unique object: + - count of values + - count of distinct values + - count of distinct values that have occurred only once +*/ + +class Basic_stats_collector +{ + ulonglong count; /* number of values retrieved */ + ulonglong count_distinct; /* number of distinct values retrieved */ + /* number of distinct values that occured only once */ + ulonglong count_distinct_single_occurence; + +public: + Basic_stats_collector() + { + count= 0; + count_distinct= 0; + count_distinct_single_occurence= 0; + } + + ulonglong get_count_distinct() const { return count_distinct; } + ulonglong get_count_single_occurence() const + { + return count_distinct_single_occurence; + } + ulonglong get_count() const { return count; } + + void next(void *elem, element_count elem_cnt) + { + count_distinct++; + if (elem_cnt == 1) + count_distinct_single_occurence++; + count+= elem_cnt; + } +}; + + +/* + Histogram_builder is a helper class that is used to build histograms + for columns. + + Do not create directly, call Histogram->get_builder(...); +*/ + +class Histogram_builder +{ +protected: + Field *column; /* table field for which the histogram is built */ + uint col_length; /* size of this field */ + ha_rows records; /* number of records the histogram is built for */ + + Histogram_builder(Field *col, uint col_len, ha_rows rows) : + column(col), col_length(col_len), records(rows) + {} + +public: + // A histogram builder will also collect the counters + Basic_stats_collector counters; + + virtual int next(void *elem, element_count elem_cnt)=0; + virtual void finalize()=0; + virtual ~Histogram_builder(){} }; @@ -308,7 +446,6 @@ public: /* Array of records per key for index prefixes */ ulonglong *idx_avg_frequency; - uchar *histograms; /* Sequence of histograms */ }; @@ -370,8 +507,10 @@ private: ulonglong avg_frequency; public: + /* Histogram type as specified in mysql.column_stats.hist_type */ + Histogram_type histogram_type_on_disk; - Histogram histogram; + Histogram_base *histogram; uint32 no_values_provided_bitmap() { diff --git a/sql/sql_table.cc b/sql/sql_table.cc index a281bdd7bcc..88b29d2b59e 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -19,6 +19,7 @@ /* drop and alter of tables */ #include "mariadb.h" +#include "sql_class.h" #include "sql_priv.h" #include "unireg.h" #include "debug_sync.h" @@ -59,6 +60,9 @@ #include "debug.h" // debug_crash_here() #include <algorithm> #include "wsrep_mysqld.h" +#include "rpl_mi.h" +#include "rpl_rli.h" +#include "log.h" #include "sql_debug.h" #ifdef _WIN32 @@ -91,6 +95,12 @@ static int mysql_prepare_create_table(THD *, HA_CREATE_INFO *, Alter_info *, static uint blob_length_by_type(enum_field_types type); static bool fix_constraints_names(THD *, List<Virtual_column_info> *, const HA_CREATE_INFO *); +static bool wait_for_master(THD *thd); +static int process_master_state(THD *thd, int alter_result, + uint64 &start_alter_id, bool if_exists); +static bool +write_bin_log_start_alter_rollback(THD *thd, uint64 &start_alter_id, + bool &partial_alter, bool if_exists); /** @brief Helper function for explain_filename @@ -669,10 +679,12 @@ void build_lower_case_table_filename(char *buff, size_t bufflen, */ uint build_table_shadow_filename(char *buff, size_t bufflen, - ALTER_PARTITION_PARAM_TYPE *lpt) + ALTER_PARTITION_PARAM_TYPE *lpt, + bool backup) { char tmp_name[FN_REFLEN]; - my_snprintf(tmp_name, sizeof (tmp_name), "%s-shadow-%lx-%s", tmp_file_prefix, + my_snprintf(tmp_name, sizeof (tmp_name), "%s-%s-%lx-%s", tmp_file_prefix, + backup ? "backup" : "shadow", (ulong) current_thd->thread_id, lpt->table_name.str); return build_table_filename(buff, bufflen, lpt->db.str, tmp_name, "", FN_IS_TMP); @@ -706,6 +718,11 @@ uint build_table_shadow_filename(char *buff, size_t bufflen, tables since it only handles partitioned data if it exists. */ + +/* + TODO: Partitioning atomic DDL refactoring: WFRM_WRITE_SHADOW + should be merged with create_table_impl(frm_only == true). +*/ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) { /* @@ -719,8 +736,11 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) char shadow_frm_name[FN_REFLEN+1]; char frm_name[FN_REFLEN+1]; #ifdef WITH_PARTITION_STORAGE_ENGINE + char bak_path[FN_REFLEN+1]; + char bak_frm_name[FN_REFLEN+1]; char *part_syntax_buf; uint syntax_len; + partition_info *part_info= lpt->part_info; #endif DBUG_ENTER("mysql_write_frm"); @@ -779,6 +799,94 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) goto end; } } +#ifdef WITH_PARTITION_STORAGE_ENGINE + if (flags & WFRM_WRITE_CONVERTED_TO) + { + THD *thd= lpt->thd; + Alter_table_ctx *alter_ctx= lpt->alter_ctx; + HA_CREATE_INFO *create_info= lpt->create_info; + + LEX_CSTRING new_path= { alter_ctx->get_new_path(), 0 }; + partition_info *work_part_info= thd->work_part_info; + handlerton *db_type= create_info->db_type; + DBUG_ASSERT(lpt->table->part_info); + DBUG_ASSERT(lpt->table->part_info == part_info); + handler *file= ((ha_partition *)(lpt->table->file))->get_child_handlers()[0]; + DBUG_ASSERT(file); + new_path.length= strlen(new_path.str); + strxnmov(frm_name, sizeof(frm_name) - 1, new_path.str, reg_ext, NullS); + create_info->alias= alter_ctx->table_name; + thd->work_part_info= NULL; + create_info->db_type= work_part_info->default_engine_type; + /* NOTE: partitioned temporary tables are not supported. */ + DBUG_ASSERT(!create_info->tmp_table()); + if (ddl_log_create_table(thd, part_info, create_info->db_type, &new_path, + &alter_ctx->new_db, &alter_ctx->new_name, true) || + ERROR_INJECT("create_before_create_frm")) + DBUG_RETURN(TRUE); + + if (mysql_prepare_create_table(thd, create_info, lpt->alter_info, + &lpt->db_options, file, + &lpt->key_info_buffer, &lpt->key_count, + C_ALTER_TABLE, alter_ctx->new_db, + alter_ctx->new_name)) + DBUG_RETURN(TRUE); + + lpt->create_info->table_options= lpt->db_options; + LEX_CUSTRING frm= build_frm_image(thd, alter_ctx->new_name, create_info, + lpt->alter_info->create_list, + lpt->key_count, lpt->key_info_buffer, + file); + if (unlikely(!frm.str)) + DBUG_RETURN(TRUE); + + thd->work_part_info= work_part_info; + create_info->db_type= db_type; + + ERROR_INJECT("alter_partition_after_create_frm"); + + error= writefile(frm_name, alter_ctx->new_db.str, alter_ctx->new_name.str, + create_info->tmp_table(), frm.str, frm.length); + my_free((void *) frm.str); + if (unlikely(error) || ERROR_INJECT("alter_partition_after_write_frm")) + { + mysql_file_delete(key_file_frm, frm_name, MYF(0)); + DBUG_RETURN(TRUE); + } + + DBUG_RETURN(false); + } + if (flags & WFRM_BACKUP_ORIGINAL) + { + build_table_filename(path, sizeof(path) - 1, lpt->db.str, + lpt->table_name.str, "", 0); + strxnmov(frm_name, sizeof(frm_name), path, reg_ext, NullS); + + build_table_shadow_filename(bak_path, sizeof(bak_path) - 1, lpt, true); + strxmov(bak_frm_name, bak_path, reg_ext, NullS); + + DDL_LOG_MEMORY_ENTRY *main_entry= part_info->main_entry; + mysql_mutex_lock(&LOCK_gdl); + if (write_log_replace_frm(lpt, part_info->list->entry_pos, + (const char*) bak_path, + (const char*) path) || + ddl_log_write_execute_entry(part_info->list->entry_pos, + &part_info->execute_entry)) + { + mysql_mutex_unlock(&LOCK_gdl); + DBUG_RETURN(TRUE); + } + mysql_mutex_unlock(&LOCK_gdl); + part_info->main_entry= main_entry; + if (mysql_file_rename(key_file_frm, frm_name, bak_frm_name, MYF(MY_WME))) + DBUG_RETURN(TRUE); + if (lpt->table->file->ha_create_partitioning_metadata(bak_path, path, + CHF_RENAME_FLAG)) + DBUG_RETURN(TRUE); + } +#else /* !WITH_PARTITION_STORAGE_ENGINE */ + DBUG_ASSERT(!(flags & WFRM_BACKUP_ORIGINAL)); +#endif /* !WITH_PARTITION_STORAGE_ENGINE */ if (flags & WFRM_INSTALL_SHADOW) { #ifdef WITH_PARTITION_STORAGE_ENGINE @@ -800,20 +908,25 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) completing this we write a new phase to the log entry that will deactivate it. */ - if (mysql_file_delete(key_file_frm, frm_name, MYF(MY_WME)) || + if (!(flags & WFRM_BACKUP_ORIGINAL) && ( + mysql_file_delete(key_file_frm, frm_name, MYF(MY_WME)) #ifdef WITH_PARTITION_STORAGE_ENGINE - lpt->table->file->ha_create_partitioning_metadata(path, shadow_path, + || lpt->table->file->ha_create_partitioning_metadata(path, shadow_path, CHF_DELETE_FLAG) || - ddl_log_increment_phase(part_info->frm_log_entry->entry_pos) || - (ddl_log_sync(), FALSE) || - mysql_file_rename(key_file_frm, - shadow_frm_name, frm_name, MYF(MY_WME)) || - lpt->table->file->ha_create_partitioning_metadata(path, shadow_path, - CHF_RENAME_FLAG)) -#else - mysql_file_rename(key_file_frm, - shadow_frm_name, frm_name, MYF(MY_WME))) + ddl_log_increment_phase(part_info->main_entry->entry_pos) || + (ddl_log_sync(), FALSE) +#endif + )) + { + error= 1; + goto err; + } + if (mysql_file_rename(key_file_frm, shadow_frm_name, frm_name, MYF(MY_WME)) +#ifdef WITH_PARTITION_STORAGE_ENGINE + || lpt->table->file->ha_create_partitioning_metadata(path, shadow_path, + CHF_RENAME_FLAG) #endif + ) { error= 1; goto err; @@ -852,8 +965,8 @@ bool mysql_write_frm(ALTER_PARTITION_PARAM_TYPE *lpt, uint flags) err: #ifdef WITH_PARTITION_STORAGE_ENGINE - ddl_log_increment_phase(part_info->frm_log_entry->entry_pos); - part_info->frm_log_entry= NULL; + ddl_log_increment_phase(part_info->main_entry->entry_pos); + part_info->main_entry= NULL; (void) ddl_log_sync(); #endif ; @@ -891,7 +1004,16 @@ int write_bin_log(THD *thd, bool clear_error, int errcode= 0; thd_proc_info(thd, "Writing to binlog"); if (clear_error) + { + if (global_system_variables.log_warnings > 2) + { + uint err_clear= thd->is_error() ? thd->get_stmt_da()->sql_errno() : 0; + if (err_clear) + sql_print_warning("Error code %d of query '%s' is cleared at its " + "binary logging.", err_clear, query); + } thd->clear_error(); + } else errcode= query_error_code(thd, TRUE); error= thd->binlog_query(THD::STMT_QUERY_TYPE, @@ -903,21 +1025,40 @@ int write_bin_log(THD *thd, bool clear_error, } -/* - Write to binary log with optional adding "IF EXISTS" +/** + Write to binary log the query event whose text is taken from thd->query(). - The query is taken from thd->query() + @param thd Thread handle. + @param clear_error Whether to clear out any error from + execution context and binlog event. + @param is_trans Whether the query changed transactional data. + @param add_if_exists True indicates the binary logging of the query + should be done with "if exists" option. + @param commit_alter Whether the query should be binlogged as + Commit Alter. + + @return false on Success + @return true otherwise */ int write_bin_log_with_if_exists(THD *thd, bool clear_error, - bool is_trans, bool add_if_exists) + bool is_trans, bool add_if_exists, + bool commit_alter) { int result; ulonglong save_option_bits= thd->variables.option_bits; if (add_if_exists) thd->variables.option_bits|= OPTION_IF_EXISTS; + if (commit_alter) + thd->set_binlog_flags_for_alter(Gtid_log_event::FL_COMMIT_ALTER_E1); + result= write_bin_log(thd, clear_error, thd->query(), thd->query_length(), is_trans); + if (commit_alter) + { + thd->set_binlog_flags_for_alter(0); + thd->set_binlog_start_alter_seq_no(0); + } thd->variables.option_bits= save_option_bits; return result; } @@ -2185,7 +2326,7 @@ void promote_first_timestamp_column(List<Create_field> *column_definitions) static bool key_cmp(const Key_part_spec &a, const Key_part_spec &b) { - return a.length == b.length && + return a.length == b.length && a.asc == b.asc && !lex_string_cmp(system_charset_info, &a.field_name, &b.field_name); } @@ -2765,8 +2906,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, */ if (sql_field->stored_in_db()) record_offset+= sql_field->pack_length; - if (sql_field->flags & VERS_SYSTEM_FIELD) - continue; } /* Update virtual fields' offset and give error if All fields are invisible */ @@ -3120,14 +3259,14 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (!sql_field || (sql_field->invisible > INVISIBLE_USER && !column->generated)) { - my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str); + my_error(ER_KEY_COLUMN_DOES_NOT_EXIST, MYF(0), column->field_name.str); DBUG_RETURN(TRUE); } if (sql_field->invisible > INVISIBLE_USER && !(sql_field->flags & VERS_SYSTEM_FIELD) && - !key->invisible && DBUG_EVALUATE_IF("test_invisible_index", 0, 1)) + !key->invisible && !DBUG_IF("test_invisible_index")) { - my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str); + my_error(ER_KEY_COLUMN_DOES_NOT_EXIST, MYF(0), column->field_name.str); DBUG_RETURN(TRUE); } while ((dup_column= cols2++) != column) @@ -3233,6 +3372,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, key_part_info->fieldnr= field; key_part_info->offset= (uint16) sql_field->offset; key_part_info->key_type=sql_field->pack_flag; + key_part_info->key_part_flag= column->asc ? 0 : HA_REVERSE_SORT; uint key_part_length= sql_field->type_handler()-> calc_key_length(*sql_field); @@ -4201,7 +4341,6 @@ err: @retval -1 table existed but IF NOT EXISTS was used */ -static int create_table_impl(THD *thd, DDL_LOG_STATE *ddl_log_state_create, DDL_LOG_STATE *ddl_log_state_rm, @@ -6066,9 +6205,8 @@ remove_key: if (!part_elem) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, - ER_DROP_PARTITION_NON_EXISTENT, - ER_THD(thd, ER_DROP_PARTITION_NON_EXISTENT), - "DROP"); + ER_PARTITION_DOES_NOT_EXIST, + ER_THD(thd, ER_PARTITION_DOES_NOT_EXIST)); names_it.remove(); } } @@ -6258,6 +6396,14 @@ Compare_keys compare_keys_but_name(const KEY *table_key, const KEY *new_key, return Compare_keys::NotEqual; } + /* + Check the descending flag for index field. + */ + if ((new_part->key_part_flag ^ key_part->key_part_flag) & HA_REVERSE_SORT) + { + return Compare_keys::NotEqual; + } + auto compare= table->file->compare_key_parts( *table->field[key_part->fieldnr - 1], new_field, *key_part, *new_part); result= merge(result, compare); @@ -6894,10 +7040,8 @@ static void update_altered_table(const Alter_inplace_info &ha_alter_info, @retval false success */ -bool mysql_compare_tables(TABLE *table, - Alter_info *alter_info, - HA_CREATE_INFO *create_info, - bool *metadata_equal) +bool mysql_compare_tables(TABLE *table, Alter_info *alter_info, + HA_CREATE_INFO *create_info, bool *metadata_equal) { DBUG_ENTER("mysql_compare_tables"); @@ -6922,15 +7066,14 @@ bool mysql_compare_tables(TABLE *table, Alter_info tmp_alter_info(*alter_info, thd->mem_root); uint db_options= 0; /* not used */ KEY *key_info_buffer= NULL; - LEX_CSTRING db= { table->s->db.str, table->s->db.length }; - LEX_CSTRING table_name= { table->s->db.str, table->s->table_name.length }; /* Create the prepared information. */ int create_table_mode= table->s->tmp_table == NO_TMP_TABLE ? C_ORDINARY_CREATE : C_ALTER_TABLE; if (mysql_prepare_create_table(thd, create_info, &tmp_alter_info, &db_options, table->file, &key_info_buffer, - &key_count, create_table_mode, db, table_name)) + &key_count, create_table_mode, + table->s->db, table->s->table_name)) DBUG_RETURN(1); /* Some very basic checks. */ @@ -7025,7 +7168,8 @@ bool mysql_compare_tables(TABLE *table, are equal. Comparing field numbers is sufficient. */ if ((table_part->length != new_part->length) || - (table_part->fieldnr - 1 != new_part->fieldnr)) + (table_part->fieldnr - 1 != new_part->fieldnr) || + ((table_part->key_part_flag ^ new_part->key_part_flag) & HA_REVERSE_SORT)) DBUG_RETURN(false); } } @@ -7210,6 +7354,86 @@ static bool notify_tabledef_changed(TABLE_LIST *table_list) DBUG_RETURN(0); } +/** + The function is invoked in error branches of ALTER processing. + Write Rollback alter in case of partial_alter is true else + call process_master_state. + + @param thd Thread handle. + @param[in/out] + start_alter_id Start Alter identifier or zero, + it is reset to zero. + @param[in/out] + partial_alter When is true at the function enter + that indicates Start Alter phase completed; + it then is reset to false. + @param if_exists True indicates the binary logging of the query + should be done with "if exists" option. + + @return false on Success + @return true otherwise +*/ +static bool +write_bin_log_start_alter_rollback(THD *thd, uint64 &start_alter_id, + bool &partial_alter, bool if_exists) +{ +#if defined(HAVE_REPLICATION) + if (start_alter_id) + { + start_alter_info *info= thd->rgi_slave->sa_info; + Master_info *mi= thd->rgi_slave->rli->mi; + + if (info->sa_seq_no == 0) + { + /* + Error occurred before SA got to processing incl its binlogging. + So it's a failure to apply and thus no need to wait for master's + completion result. + */ + return true; + } + mysql_mutex_lock(&mi->start_alter_lock); + if (info->direct_commit_alter) + { + DBUG_ASSERT(info->state == start_alter_state::ROLLBACK_ALTER); + + /* + SA may end up in the rollback state through FTWRL that breaks + SA's waiting for a master decision. + Then it completes "officially", and `direct_commit_alter` true status + will affect the future of CA to re-execute the whole query. + */ + info->state= start_alter_state::COMPLETED; + if (info->direct_commit_alter) + mysql_cond_broadcast(&info->start_alter_cond); + mysql_mutex_unlock(&mi->start_alter_lock); + + return true; // not really an error to be handled by caller specifically + } + mysql_mutex_unlock(&mi->start_alter_lock); + /* + We have to call wait for master here because in main calculation + we can error out before calling wait for master + (for example if copy_data_between_tables fails) + */ + if (info->state == start_alter_state::REGISTERED) + wait_for_master(thd); + if(process_master_state(thd, 1, start_alter_id, if_exists)) + return true; + } + else +#endif + if (partial_alter) // Write only if SA written + { + // Send the rollback message + Write_log_with_flags wlwf(thd, Gtid_log_event::FL_ROLLBACK_ALTER_E1); + if(write_bin_log_with_if_exists(thd, false, false, if_exists, false)) + return true; + partial_alter= false; + } + return false; +} + /** Perform in-place alter table. @@ -7223,9 +7447,17 @@ static bool notify_tabledef_changed(TABLE_LIST *table_list) used during different phases. @param target_mdl_request Metadata request/lock on the target table name. @param alter_ctx ALTER TABLE runtime context. - - @retval true Error - @retval false Success + @param partial_alter Is set to true to return the fact of the first + "START ALTER" binlogging phase is done. + @param[in/out] + start_alter_id Gtid seq_no of START ALTER or zero otherwise; + it may get changed to return to the caller. + @param if_exists True indicates the binary logging of the query + should be done with "if exists" option. + + @retval >=1 Error{ 1= ROLLBACK recieved from master , 2= error + in alter so no ROLLBACK in binlog } + @retval 0 Success @note If mysql_alter_table does not need to copy the table, it is @@ -7248,13 +7480,16 @@ static bool mysql_inplace_alter_table(THD *thd, MDL_request *target_mdl_request, DDL_LOG_STATE *ddl_log_state, TRIGGER_RENAME_PARAM *trigger_param, - Alter_table_ctx *alter_ctx) + Alter_table_ctx *alter_ctx, + bool &partial_alter, + uint64 &start_alter_id, bool if_exists) { Open_table_context ot_ctx(thd, MYSQL_OPEN_REOPEN | MYSQL_OPEN_IGNORE_KILLED); handlerton *db_type= table->s->db_type(); Alter_info *alter_info= ha_alter_info->alter_info; bool reopen_tables= false; bool res, commit_succeded_with_error= 0; + const enum_alter_inplace_result inplace_supported= ha_alter_info->inplace_supported; DBUG_ENTER("mysql_inplace_alter_table"); @@ -7334,10 +7569,31 @@ static bool mysql_inplace_alter_table(THD *thd, thd->variables.lock_wait_timeout)) goto cleanup; + DBUG_ASSERT(table->s->tmp_table == NO_TMP_TABLE || start_alter_id == 0); + + if (table->s->tmp_table == NO_TMP_TABLE) + { + if (write_bin_log_start_alter(thd, partial_alter, start_alter_id, + if_exists)) + goto cleanup; + } + else if (start_alter_id) + { + DBUG_ASSERT(thd->rgi_slave); + + my_error(ER_INCONSISTENT_SLAVE_TEMP_TABLE, MYF(0), thd->query(), + table_list->db.str, table_list->table_name.str); + goto cleanup; + } + + DBUG_EXECUTE_IF("start_alter_kill_after_binlog", { + DBUG_SUICIDE(); + }); + + // It's now safe to take the table level lock. if (lock_tables(thd, table_list, alter_ctx->tables_opened, 0)) goto cleanup; - DEBUG_SYNC(thd, "alter_table_inplace_after_lock_upgrade"); THD_STAGE_INFO(thd, stage_alter_inplace_prepare); @@ -7417,14 +7673,23 @@ static bool mysql_inplace_alter_table(THD *thd, DEBUG_SYNC(thd, "alter_table_inplace_after_lock_downgrade"); THD_STAGE_INFO(thd, stage_alter_inplace); + DBUG_EXECUTE_IF("start_alter_delay_master", { + debug_sync_set_action(thd, + STRING_WITH_LEN("now wait_for alter_cont NO_CLEAR_EVENT")); + }); /* We can abort alter table for any table type */ thd->abort_on_warning= !ha_alter_info->ignore && thd->is_strict_mode(); res= table->file->ha_inplace_alter_table(altered_table, ha_alter_info); thd->abort_on_warning= false; + + if (start_alter_id && wait_for_master(thd)) + goto rollback; + if (res) goto rollback; + DEBUG_SYNC(thd, "alter_table_inplace_before_lock_upgrade"); // Upgrade to EXCLUSIVE before commit. if (wait_while_table_is_used(thd, table, HA_EXTRA_PREPARE_FOR_RENAME)) @@ -7533,7 +7798,7 @@ static bool mysql_inplace_alter_table(THD *thd, thd->is_error()) { // Since changes were done in-place, we can't revert them. - DBUG_RETURN(true); + goto err; } debug_crash_here("ddl_log_alter_after_rename_frm"); @@ -7550,7 +7815,7 @@ static bool mysql_inplace_alter_table(THD *thd, If the rename fails we will still have a working table with the old name, but with other changes applied. */ - DBUG_RETURN(true); + goto err; } debug_crash_here("ddl_log_alter_before_rename_triggers"); if (Table_triggers_list::change_table_name(thd, trigger_param, @@ -7595,6 +7860,8 @@ static bool mysql_inplace_alter_table(THD *thd, if (thd->locked_tables_list.reopen_tables(thd, false)) thd->locked_tables_list.unlink_all_closed_tables(thd, NULL, 0); } + +err: DBUG_RETURN(true); } @@ -7637,6 +7904,7 @@ void append_drop_column(THD *thd, String *str, Field *field) } +#ifdef WITH_PARTITION_STORAGE_ENGINE static inline void rename_field_in_list(Create_field *field, List<const char> *field_list) { @@ -7649,6 +7917,7 @@ void rename_field_in_list(Create_field *field, List<const char> *field_list) it.replace(field->field_name.str); } } +#endif /** @@ -7802,8 +8071,9 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, thd->calloc(sizeof(void*) * table->s->keys)) == NULL) DBUG_RETURN(1); - create_info->option_list= merge_engine_table_options(table->s->option_list, - create_info->option_list, thd->mem_root); + if (merge_engine_options(table->s->option_list, create_info->option_list, + &create_info->option_list, thd->mem_root)) + DBUG_RETURN(1); /* First collect all fields from table which isn't in drop_list @@ -8374,9 +8644,10 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, key_part_length= 0; // Use whole field } key_part_length /= kfield->charset()->mbmaxlen; - key_parts.push_back(new (thd->mem_root) Key_part_spec(&cfield->field_name, - key_part_length, true), - thd->mem_root); + Key_part_spec *kps= new (thd->mem_root) Key_part_spec(&cfield->field_name, + key_part_length, true); + kps->asc= !(key_part->key_part_flag & HA_REVERSE_SORT); + key_parts.push_back(kps, thd->mem_root); if (!(cfield->invisible == INVISIBLE_SYSTEM && cfield->vers_sys_field())) user_keyparts= true; } @@ -8435,7 +8706,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, key_type= Key::UNIQUE; if (dropped_key_part) { - my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), dropped_key_part); + my_error(ER_KEY_COLUMN_DOES_NOT_EXIST, MYF(0), dropped_key_part); if (long_hash_key) { key_info->algorithm= HA_KEY_ALG_LONG_HASH; @@ -9361,6 +9632,153 @@ static bool log_and_ok(THD *thd) return(0); } +/* + Wait for master to send result of Alter table. + Returns + true when Rollback is decided + false otherwise +*/ +static bool wait_for_master(THD *thd) +{ +#ifdef HAVE_REPLICATION + start_alter_info* info= thd->rgi_slave->sa_info; + Master_info *mi= thd->rgi_slave->rli->mi; + + DBUG_ASSERT(info); + DBUG_ASSERT(info->state != start_alter_state::INVALID); + DBUG_ASSERT(mi); + + mysql_mutex_lock(&mi->start_alter_lock); + + DBUG_ASSERT(!info->direct_commit_alter || + info->state == start_alter_state::ROLLBACK_ALTER); + + while (info->state == start_alter_state::REGISTERED) + { + mysql_cond_wait(&info->start_alter_cond, &mi->start_alter_lock); + } + if (info->state == start_alter_state::ROLLBACK_ALTER) + { + /* + SA thread will not give error , We will set the error in info->error + and then RA worker will output the error + We can modify the info->error without taking mutex, because CA worker + will be waiting on ::COMPLETED wait condition + */ + if(thd->is_error()) + { + info->error= thd->get_stmt_da()->sql_errno(); + thd->clear_error(); + thd->reset_killed(); + } + } + mysql_mutex_unlock(&mi->start_alter_lock); + + return info->state == start_alter_state::ROLLBACK_ALTER; +#else + return 0; +#endif +} + +#ifdef HAVE_REPLICATION +/** + In this function, we are going to change info->state to ::COMPLETED. + This means we are messaging CA/RA worker that we have binlogged, so our + here is finished. + + @param thd Thread handle + @param start_alter_state ALTER replicaton execution context + @param mi Master_info of the replication source +*/ +static void alter_committed(THD *thd, start_alter_info* info, Master_info *mi) +{ + start_alter_state tmp= info->state; + mysql_mutex_lock(&mi->start_alter_lock); + info->state= start_alter_state::COMPLETED; + mysql_cond_broadcast(&info->start_alter_cond); + mysql_mutex_unlock(&mi->start_alter_lock); + if (tmp == start_alter_state::ROLLBACK_ALTER) + { + thd->clear_error(); + thd->reset_killed(); + } +} +#endif + +/** + process_master_state:- process the info->state recieved from master + We will comapre master state with alter_result + In the case of ROLLBACK_ALTER alter_result > 0 + In the case of COMMIT_ALTER alter_result == 0 + if the condition is not satisfied we will report error and + Return 1. Make sure wait_for_master is called before calling this function + This function should be called only at the write binlog time of commit/ + rollback alter. We will alter_committed if everything is fine. + + @param thd Thread handle. + @param alter_result Result of execution. + @param[in/out] + start_alter_id Start Alter identifier or zero, + it is reset to zero. + @param if_exists True indicates the binary logging of the query + should be done with "if exists" option. + @retval 1 error + @retval 0 Ok +*/ +static int process_master_state(THD *thd, int alter_result, + uint64 &start_alter_id, bool if_exists) +{ +#ifdef HAVE_REPLICATION + start_alter_info *info= thd->rgi_slave->sa_info; + bool partial_alter= false; + + if (info->state == start_alter_state::INVALID) + { + /* the caller has not yet called SA logging nor wait for master decision */ + if (!write_bin_log_start_alter(thd, partial_alter, start_alter_id, + if_exists)) + wait_for_master(thd); + + DBUG_ASSERT(!partial_alter); + } + + /* this function shouldn't be called twice */ + DBUG_ASSERT(start_alter_id); + + start_alter_id= 0; + if ((info->state == start_alter_state::ROLLBACK_ALTER && alter_result >= 0) + || (info->state == start_alter_state::COMMIT_ALTER && !alter_result)) + { + alter_committed(thd, info, thd->rgi_slave->rli->mi); + return 0; + } + else + { + thd->is_slave_error= 1; + return 1; + } +#else + return 0; +#endif +} + +/* + Returns a (global unique) identifier of START Alter when slave applier + executes mysql_alter_table(). + In non-slave context it is zero. +*/ +static uint64 get_start_alter_id(THD *thd) +{ + DBUG_ASSERT(!(thd->rgi_slave && + (thd->rgi_slave->gtid_ev_flags_extra & + Gtid_log_event::FL_START_ALTER_E1)) || + !thd->rgi_slave->direct_commit_alter); + return + thd->rgi_slave && + (thd->rgi_slave->gtid_ev_flags_extra & Gtid_log_event::FL_START_ALTER_E1) ? + thd->variables.gtid_seq_no : 0; +} + /** Alter table @@ -9419,6 +9837,14 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, bool partition_changed= false; bool fast_alter_partition= false; #endif + bool partial_alter= false; + /* + start_alter_id is the gtid seq no of the START Alter - the 1st part + of two-phase loggable ALTER. The variable is meaningful only + on slave execution. + */ + uint64 start_alter_id= get_start_alter_id(thd); + /* Create .FRM for new version of table with a temporary name. We don't log the statement, it will be logged later. @@ -9557,6 +9983,7 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, } table= table_list->table; + bool is_reg_table= table->s->tmp_table == NO_TMP_TABLE; #ifdef WITH_WSREP if (WSREP(thd) && @@ -9713,7 +10140,8 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, Table maybe does not exist, but we got an exclusive lock on the name, now we can safely try to find out for sure. */ - if (ha_table_exists(thd, &alter_ctx.new_db, &alter_ctx.new_name)) + if (!(alter_info->partition_flags & ALTER_PARTITION_CONVERT_IN) && + ha_table_exists(thd, &alter_ctx.new_db, &alter_ctx.new_name)) { /* Table will be closed in do_command() */ my_error(ER_TABLE_EXISTS_ERROR, MYF(0), alter_ctx.new_alias.str); @@ -9968,6 +10396,8 @@ do_continue:; { DBUG_RETURN(true); } + if (parse_engine_part_options(thd, table)) + DBUG_RETURN(true); } /* If the old table had partitions and we are doing ALTER TABLE ... @@ -10050,10 +10480,8 @@ do_continue:; } // In-place execution of ALTER TABLE for partitioning. - DBUG_RETURN(fast_alter_partition_table(thd, table, alter_info, - create_info, table_list, - &alter_ctx.db, - &alter_ctx.table_name)); + DBUG_RETURN(fast_alter_partition_table(thd, table, alter_info, &alter_ctx, + create_info, table_list)); } #endif @@ -10355,7 +10783,8 @@ do_continue:; &ha_alter_info, &target_mdl_request, &ddl_log_state, &trigger_param, - &alter_ctx); + &alter_ctx, partial_alter, + start_alter_id, if_exists); thd->count_cuted_fields= org_count_cuted_fields; inplace_alter_table_committed= ha_alter_info.inplace_alter_table_committed; inplace_alter_table_committed_argument= @@ -10410,6 +10839,25 @@ do_continue:; else thd->close_unused_temporary_table_instances(table_list); + if (table->s->tmp_table == NO_TMP_TABLE) + { + if (write_bin_log_start_alter(thd, partial_alter, start_alter_id, + if_exists)) + goto err_new_table_cleanup; + } + else if (start_alter_id) + { + DBUG_ASSERT(thd->rgi_slave); + + my_error(ER_INCONSISTENT_SLAVE_TEMP_TABLE, MYF(0), thd->query(), + table_list->db.str, table_list->table_name.str); + goto err_new_table_cleanup; + } + + DBUG_EXECUTE_IF("start_alter_delay_master", { + debug_sync_set_action(thd, + STRING_WITH_LEN("now wait_for alter_cont NO_CLEAR_EVENT")); + }); // It's now safe to take the table level lock. if (lock_tables(thd, table_list, alter_ctx.tables_opened, MYSQL_LOCK_USE_MALLOC)) @@ -10522,6 +10970,13 @@ do_continue:; } thd->count_cuted_fields= CHECK_FIELD_IGNORE; + if (start_alter_id) + { + DBUG_ASSERT(thd->slave_thread); + + if (wait_for_master(thd)) + goto err_new_table_cleanup; + } if (table->s->tmp_table != NO_TMP_TABLE) { /* Release lock if this is a transactional temporary table */ @@ -10565,6 +11020,9 @@ do_continue:; binlog_commit(thd, true); } + DBUG_ASSERT(!start_alter_id); // no 2 phase logging for + DBUG_ASSERT(!partial_alter); // temporary table alter + /* We don't replicate alter table statement on temporary tables */ if (!thd->is_current_stmt_binlog_format_row() && table_creation_was_logged && @@ -10803,12 +11261,26 @@ end_inplace: DBUG_ASSERT(!(mysql_bin_log.is_open() && thd->is_current_stmt_binlog_format_row() && (create_info->tmp_table()))); - if (!binlog_as_create_select) + + if(start_alter_id) + { + if (!is_reg_table) + { + my_error(ER_INCONSISTENT_SLAVE_TEMP_TABLE, MYF(0), thd->query(), + table_list->db.str, table_list->table_name.str); + DBUG_RETURN(true); + } + + if (process_master_state(thd, 0, start_alter_id, if_exists)) + DBUG_RETURN(true); + } + else if (!binlog_as_create_select) { int tmp_error; thd->binlog_xid= thd->query_id; ddl_log_update_xid(&ddl_log_state, thd->binlog_xid); - tmp_error= write_bin_log_with_if_exists(thd, true, false, log_if_exists); + tmp_error= write_bin_log_with_if_exists(thd, true, false, log_if_exists, + partial_alter); thd->binlog_xid= 0; if (tmp_error) goto err_cleanup; @@ -10904,6 +11376,10 @@ err_cleanup: /* Signal to storage engine that ddl log is committed */ (*inplace_alter_table_committed)(inplace_alter_table_committed_argument); } + DEBUG_SYNC(thd, "alter_table_after_temp_table_drop"); + if (partial_alter || start_alter_id) + write_bin_log_start_alter_rollback(thd, start_alter_id, partial_alter, + if_exists); DBUG_RETURN(true); err_with_mdl: @@ -11065,6 +11541,8 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, if (!(*ptr)->vcol_info) { bitmap_set_bit(from->read_set, def->field->field_index); + if ((*ptr)->check_assignability_from(def->field, ignore)) + goto err; (copy_end++)->set(*ptr,def->field,0); } } @@ -11152,7 +11630,7 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, if (ignore && !alter_ctx->fk_error_if_delete_row) to->file->extra(HA_EXTRA_IGNORE_DUP_KEY); - thd->get_stmt_da()->reset_current_row_for_warning(); + thd->get_stmt_da()->reset_current_row_for_warning(1); restore_record(to, s->default_values); // Create empty record to->reset_default_fields(); @@ -11620,16 +12098,11 @@ bool check_engine(THD *thd, const char *db_name, if (create_info->tmp_table() && ha_check_storage_engine_flag(*new_engine, HTON_TEMPORARY_NOT_SUPPORTED)) { - if (create_info->used_fields & HA_CREATE_USED_ENGINE) - { - my_error(ER_ILLEGAL_HA_CREATE_OPTION, MYF(0), - hton_name(*new_engine)->str, "TEMPORARY"); - *new_engine= 0; - DBUG_RETURN(true); - } - *new_engine= myisam_hton; + my_error(ER_ILLEGAL_HA_CREATE_OPTION, MYF(0), + hton_name(*new_engine)->str, "TEMPORARY"); + *new_engine= 0; + DBUG_RETURN(true); } - lex_string_set(&create_info->new_storage_engine_name, ha_resolve_storage_engine_name(*new_engine)); DBUG_RETURN(false); diff --git a/sql/sql_table.h b/sql/sql_table.h index e93c277f7e3..0f9a73c848a 100644 --- a/sql/sql_table.h +++ b/sql/sql_table.h @@ -20,6 +20,10 @@ #include <my_sys.h> // pthread_mutex_t #include "m_string.h" // LEX_CUSTRING +#define ERROR_INJECT(code) \ + ((DBUG_IF("crash_" code) && (DBUG_SUICIDE(), 0)) || \ + (DBUG_IF("fail_" code) && (my_error(ER_UNKNOWN_ERROR, MYF(0)), 1))) + class Alter_info; class Alter_table_ctx; class Column_definition; @@ -53,6 +57,8 @@ enum enum_explain_filename_mode #define WFRM_WRITE_SHADOW 1 #define WFRM_INSTALL_SHADOW 2 #define WFRM_KEEP_SHARE 4 +#define WFRM_WRITE_CONVERTED_TO 8 +#define WFRM_BACKUP_ORIGINAL 16 /* Flags for conversion functions. */ static const uint FN_FROM_IS_TMP= 1 << 0; @@ -77,7 +83,8 @@ bool check_mysql50_prefix(const char *name); uint build_table_filename(char *buff, size_t bufflen, const char *db, const char *table, const char *ext, uint flags); uint build_table_shadow_filename(char *buff, size_t bufflen, - ALTER_PARTITION_PARAM_TYPE *lpt); + ALTER_PARTITION_PARAM_TYPE *lpt, + bool backup= false); void build_lower_case_table_filename(char *buff, size_t bufflen, const LEX_CSTRING *db, const LEX_CSTRING *table, @@ -206,7 +213,8 @@ int write_bin_log(THD *thd, bool clear_error, char const *query, ulong query_length, bool is_trans= FALSE); int write_bin_log_with_if_exists(THD *thd, bool clear_error, - bool is_trans, bool add_if_exists); + bool is_trans, bool add_if_exists, + bool commit_alter= false); void promote_first_timestamp_column(List<Create_field> *column_definitions); diff --git a/sql/sql_tablespace.cc b/sql/sql_tablespace.cc deleted file mode 100644 index bfbaf185243..00000000000 --- a/sql/sql_tablespace.cc +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ - -/* drop and alter of tablespaces */ - -#include "mariadb.h" -#include "sql_priv.h" -#include "unireg.h" -#include "sql_tablespace.h" -#include "sql_table.h" // write_bin_log -#include "sql_class.h" // THD - -int mysql_alter_tablespace(THD *thd, st_alter_tablespace *ts_info) -{ - int error= HA_ADMIN_NOT_IMPLEMENTED; - handlerton *hton= ts_info->storage_engine; - - DBUG_ENTER("mysql_alter_tablespace"); - /* - If the user haven't defined an engine, this will fallback to using the - default storage engine. - */ - if (hton == NULL) - { - hton= ha_default_handlerton(thd); - if (ts_info->storage_engine != 0) - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_WARN_USING_OTHER_HANDLER, - ER_THD(thd, ER_WARN_USING_OTHER_HANDLER), - hton_name(hton)->str, - ts_info->tablespace_name ? ts_info->tablespace_name - : ts_info->logfile_group_name); - } - - if (hton->alter_tablespace) - { - if (unlikely((error= hton->alter_tablespace(hton, thd, ts_info)))) - { - if (error == 1) - DBUG_RETURN(1); - - if (error == HA_ADMIN_NOT_IMPLEMENTED) - my_error(ER_CHECK_NOT_IMPLEMENTED, MYF(0), ""); - else - my_error(error, MYF(0)); - - DBUG_RETURN(error); - } - } - else - { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_ILLEGAL_HA_CREATE_OPTION, - ER_THD(thd, ER_ILLEGAL_HA_CREATE_OPTION), - hton_name(hton)->str, - "TABLESPACE or LOGFILE GROUP"); - } - error= write_bin_log(thd, FALSE, thd->query(), thd->query_length()); - DBUG_RETURN(error); -} diff --git a/sql/sql_tablespace.h b/sql/sql_tablespace.h deleted file mode 100644 index 0760935edfc..00000000000 --- a/sql/sql_tablespace.h +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ - -#ifndef SQL_TABLESPACE_INCLUDED -#define SQL_TABLESPACE_INCLUDED - -class THD; -class st_alter_tablespace; - -int mysql_alter_tablespace(THD* thd, st_alter_tablespace *ts_info); - -#endif /* SQL_TABLESPACE_INCLUDED */ diff --git a/sql/sql_test.cc b/sql/sql_test.cc index bca6c21f668..479d4289406 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -264,7 +264,7 @@ static void print_keyuse(KEYUSE *keyuse) void print_keyuse_array(DYNAMIC_ARRAY *keyuse_array) { DBUG_LOCK_FILE; - fprintf(DBUG_FILE, "KEYUSE array (%d elements)\n", keyuse_array->elements); + fprintf(DBUG_FILE, "KEYUSE array (%zu elements)\n", keyuse_array->elements); for(uint i=0; i < keyuse_array->elements; i++) print_keyuse((KEYUSE*)dynamic_array_ptr(keyuse_array, i)); DBUG_UNLOCK_FILE; diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc index bd6e09d3ab9..d4fd4738873 100644 --- a/sql/sql_tvc.cc +++ b/sql/sql_tvc.cc @@ -422,7 +422,9 @@ bool table_value_constr::exec(SELECT_LEX *sl) DBUG_ENTER("table_value_constr::exec"); List_iterator_fast<List_item> li(lists_of_values); List_item *elem; + THD *cur_thd= sl->parent_lex->thd; ha_rows send_records= 0; + int rc=0; if (select_options & SELECT_DESCRIBE) DBUG_RETURN(false); @@ -438,12 +440,10 @@ bool table_value_constr::exec(SELECT_LEX *sl) while ((elem= li++)) { - THD *cur_thd= sl->parent_lex->thd; + cur_thd->get_stmt_da()->inc_current_row_for_warning(); if (send_records >= sl->master_unit()->lim.get_select_limit()) break; - int rc= - result->send_data_with_check(*elem, sl->master_unit(), send_records); - cur_thd->get_stmt_da()->inc_current_row_for_warning(); + rc= result->send_data_with_check(*elem, sl->master_unit(), send_records); if (!rc) send_records++; else if (rc > 0) diff --git a/sql/sql_type.cc b/sql/sql_type.cc index a09d2648757..f3c0a8404db 100644 --- a/sql/sql_type.cc +++ b/sql/sql_type.cc @@ -1746,7 +1746,7 @@ Type_handler_time_common::type_handler_for_native_format() const const Type_handler *Type_handler_typelib::type_handler_for_item_field() const { - return &type_handler_string; + return &type_handler_varchar; } diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 4c45822a802..7dfde55ec7c 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -271,7 +271,7 @@ static void prepare_record_for_error_message(int error, TABLE *table) DBUG_VOID_RETURN; /* Create unique_map with all fields used by that index. */ - my_bitmap_init(&unique_map, unique_map_buf, table->s->fields, FALSE); + my_bitmap_init(&unique_map, unique_map_buf, table->s->fields); table->mark_index_columns(keynr, &unique_map); /* Subtract read_set and write_set. */ @@ -523,6 +523,10 @@ int mysql_update(THD *thd, DBUG_RETURN(1); /* purecov: inspected */ } + if (table_list->table->check_assignability_explicit_fields(fields, values, + ignore)) + DBUG_RETURN(true); + if (check_unique_table(thd, table_list)) DBUG_RETURN(TRUE); @@ -1003,6 +1007,7 @@ update_begin: THD_STAGE_INFO(thd, stage_updating); fix_rownum_pointers(thd, thd->lex->current_select, &updated_or_same); + thd->get_stmt_da()->reset_current_row_for_warning(1); while (!(error=info.read_record()) && !thd->killed) { explain->tracker.on_record_read(); @@ -2087,7 +2092,9 @@ int multi_update::prepare(List<Item> ¬_used_values, */ int error= setup_fields(thd, Ref_ptr_array(), - *values, MARK_COLUMNS_READ, 0, NULL, 0); + *values, MARK_COLUMNS_READ, 0, NULL, 0) || + TABLE::check_assignability_explicit_fields(*fields, *values, + ignore); ti.rewind(); while ((table_ref= ti++)) diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 65aec5feb35..b6ab30cc86d 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -291,6 +291,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() class With_element_head *with_element_head; class With_clause *with_clause; class Virtual_column_info *virtual_column; + engine_option_value *engine_option_value_ptr; handlerton *db_type; st_select_lex *select_lex; @@ -776,6 +777,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %token <kwd> CATALOG_NAME_SYM /* SQL-2003-N */ %token <kwd> CHAIN_SYM /* SQL-2003-N */ %token <kwd> CHANGED +%token <kwd> CHANNEL_SYM %token <kwd> CHARSET %token <kwd> CHECKPOINT_SYM %token <kwd> CHECKSUM_SYM @@ -1477,7 +1479,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); field_options last_field_options %type <ulonglong_number> - ulonglong_num real_ulonglong_num size_number + ulonglong_num real_ulonglong_num %type <longlong_number> longlong_num @@ -1763,8 +1765,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type <num> sp_handler_type sp_hcond_list %type <spcondvalue> sp_cond sp_hcond sqlstate signal_value opt_signal_value %type <spname> sp_name -%type <spvar> sp_param_name sp_param_name_and_type -%type <spvar> sp_param_name_and_type_anchored +%type <spvar> sp_param_name sp_param_name_and_mode sp_param +%type <spvar> sp_param_anchored %type <for_loop> sp_for_loop_index_and_bounds %type <for_loop_bounds> sp_for_loop_bounds %type <trim> trim_operands @@ -1818,6 +1820,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type <vers_range_unit> opt_history_unit %type <vers_history_point> history_point %type <vers_column_versioning> with_or_without_system +%type <engine_option_value_ptr> engine_defined_option; %ifdef MARIADB %type <NONE> sp_tail_standalone @@ -2097,6 +2100,7 @@ change: Lex->sql_command = SQLCOM_CHANGE_MASTER; } master_defs + optional_for_channel {} ; @@ -2328,6 +2332,34 @@ connection_name: } ; +optional_for_channel: + /* empty */ + { + /*do nothing */ + } + | for_channel + + ; + +for_channel: + FOR_SYM CHANNEL_SYM TEXT_STRING_sys + { + if (Lex->mi.connection_name.str != NULL) + { + my_yyabort_error((ER_WRONG_ARGUMENTS, MYF(0), "CONNECTION_NAME AND FOR CHANNEL CAN NOT BE SPECIFIED AT THE SAME TIME)")); + } + else + { + Lex->mi.connection_name= $3; +#ifdef HAVE_REPLICATION + if (unlikely(check_master_connection_name(&$3))) + my_yyabort_error((ER_WRONG_ARGUMENTS, MYF(0), "MASTER_CONNECTION_NAME")); +#endif + } + + } + ; + /* create a table */ create: @@ -2572,14 +2604,6 @@ create: $1 | $3))) MYSQL_YYABORT; } - | CREATE LOGFILE_SYM GROUP_SYM logfile_group_info - { - Lex->alter_tablespace_info->ts_cmd_type= CREATE_LOGFILE_GROUP; - } - | CREATE TABLESPACE tablespace_info - { - Lex->alter_tablespace_info->ts_cmd_type= CREATE_TABLESPACE; - } | create_or_replace { Lex->set_command(SQLCOM_CREATE_SERVER, $1); } server_def { } @@ -3070,8 +3094,8 @@ sp_fdparam_list: ; sp_fdparams: - sp_fdparams ',' sp_param_name_and_type - | sp_param_name_and_type + sp_fdparams ',' sp_param + | sp_param ; sp_param_name: @@ -3082,20 +3106,6 @@ sp_param_name: } ; -sp_param_name_and_type: - sp_param_name field_type - { - if (unlikely(Lex->sp_param_fill_definition($$= $1, $2))) - MYSQL_YYABORT; - } - | sp_param_name ROW_SYM row_type_body - { - if (unlikely(Lex->sphead->spvar_fill_row(thd, $$= $1, $3))) - MYSQL_YYABORT; - } - | sp_param_name_and_type_anchored - ; - /* Stored PROCEDURE parameter declaration list */ sp_pdparam_list: /* Empty */ @@ -3103,8 +3113,8 @@ sp_pdparam_list: ; sp_pdparams: - sp_pdparams ',' sp_pdparam - | sp_pdparam + sp_pdparams ',' sp_param + | sp_param ; sp_parameter_type: @@ -3496,6 +3506,8 @@ signal_condition_information_item_name: { $$= DIAG_MESSAGE_TEXT; } | MYSQL_ERRNO_SYM { $$= DIAG_MYSQL_ERRNO; } + | ROW_NUMBER_SYM + { $$= DIAG_ROW_NUMBER; } ; resignal_stmt: @@ -3576,6 +3588,11 @@ simple_target_specification: } | '@' ident_or_text { + if (!$2.length) + { + thd->parse_error(); + MYSQL_YYABORT; + } $$= new (thd->mem_root) Item_func_get_user_var(thd, &$2); if (unlikely($$ == NULL)) MYSQL_YYABORT; @@ -3652,6 +3669,8 @@ condition_information_item_name: { $$= Condition_information_item::MYSQL_ERRNO; } | RETURNED_SQLSTATE_SYM { $$= Condition_information_item::RETURNED_SQLSTATE; } + | ROW_NUMBER_SYM + { $$= Condition_information_item::ROW_NUMBER; } ; sp_decl_ident: @@ -4351,350 +4370,6 @@ trg_event: | DELETE_SYM { Lex->trg_chistics.event= TRG_EVENT_DELETE; } ; -/* - This part of the parser contains common code for all TABLESPACE - commands. - CREATE TABLESPACE name ... - ALTER TABLESPACE name CHANGE DATAFILE ... - ALTER TABLESPACE name ADD DATAFILE ... - ALTER TABLESPACE name access_mode - CREATE LOGFILE GROUP_SYM name ... - ALTER LOGFILE GROUP_SYM name ADD UNDOFILE .. - ALTER LOGFILE GROUP_SYM name ADD REDOFILE .. - DROP TABLESPACE name - DROP LOGFILE GROUP_SYM name -*/ -change_tablespace_access: - tablespace_name - ts_access_mode - ; - -change_tablespace_info: - tablespace_name - CHANGE ts_datafile - change_ts_option_list - ; - -tablespace_info: - tablespace_name - ADD ts_datafile - opt_logfile_group_name - tablespace_option_list - ; - -opt_logfile_group_name: - /* empty */ {} - | USE_SYM LOGFILE_SYM GROUP_SYM ident - { - LEX *lex= Lex; - lex->alter_tablespace_info->logfile_group_name= $4.str; - } - ; - -alter_tablespace_info: - tablespace_name - ADD ts_datafile - alter_tablespace_option_list - { - Lex->alter_tablespace_info->ts_alter_tablespace_type= ALTER_TABLESPACE_ADD_FILE; - } - | tablespace_name - DROP ts_datafile - alter_tablespace_option_list - { - Lex->alter_tablespace_info->ts_alter_tablespace_type= ALTER_TABLESPACE_DROP_FILE; - } - ; - -logfile_group_info: - logfile_group_name - add_log_file - logfile_group_option_list - ; - -alter_logfile_group_info: - logfile_group_name - add_log_file - alter_logfile_group_option_list - ; - -add_log_file: - ADD lg_undofile - | ADD lg_redofile - ; - -change_ts_option_list: - /* empty */ {} - change_ts_options - ; - -change_ts_options: - change_ts_option - | change_ts_options change_ts_option - | change_ts_options ',' change_ts_option - ; - -change_ts_option: - opt_ts_initial_size - | opt_ts_autoextend_size - | opt_ts_max_size - ; - -tablespace_option_list: - tablespace_options - ; - -tablespace_options: - tablespace_option - | tablespace_options tablespace_option - | tablespace_options ',' tablespace_option - ; - -tablespace_option: - opt_ts_initial_size - | opt_ts_autoextend_size - | opt_ts_max_size - | opt_ts_extent_size - | opt_ts_nodegroup - | opt_ts_engine - | ts_wait - | opt_ts_comment - ; - -alter_tablespace_option_list: - alter_tablespace_options - ; - -alter_tablespace_options: - alter_tablespace_option - | alter_tablespace_options alter_tablespace_option - | alter_tablespace_options ',' alter_tablespace_option - ; - -alter_tablespace_option: - opt_ts_initial_size - | opt_ts_autoextend_size - | opt_ts_max_size - | opt_ts_engine - | ts_wait - ; - -logfile_group_option_list: - logfile_group_options - ; - -logfile_group_options: - logfile_group_option - | logfile_group_options logfile_group_option - | logfile_group_options ',' logfile_group_option - ; - -logfile_group_option: - opt_ts_initial_size - | opt_ts_undo_buffer_size - | opt_ts_redo_buffer_size - | opt_ts_nodegroup - | opt_ts_engine - | ts_wait - | opt_ts_comment - ; - -alter_logfile_group_option_list: - alter_logfile_group_options - ; - -alter_logfile_group_options: - alter_logfile_group_option - | alter_logfile_group_options alter_logfile_group_option - | alter_logfile_group_options ',' alter_logfile_group_option - ; - -alter_logfile_group_option: - opt_ts_initial_size - | opt_ts_engine - | ts_wait - ; - - -ts_datafile: - DATAFILE_SYM TEXT_STRING_sys - { - LEX *lex= Lex; - lex->alter_tablespace_info->data_file_name= $2.str; - } - ; - -lg_undofile: - UNDOFILE_SYM TEXT_STRING_sys - { - LEX *lex= Lex; - lex->alter_tablespace_info->undo_file_name= $2.str; - } - ; - -lg_redofile: - REDOFILE_SYM TEXT_STRING_sys - { - LEX *lex= Lex; - lex->alter_tablespace_info->redo_file_name= $2.str; - } - ; - -tablespace_name: - ident - { - LEX *lex= Lex; - lex->alter_tablespace_info= (new (thd->mem_root) - st_alter_tablespace()); - if (unlikely(lex->alter_tablespace_info == NULL)) - MYSQL_YYABORT; - lex->alter_tablespace_info->tablespace_name= $1.str; - lex->sql_command= SQLCOM_ALTER_TABLESPACE; - } - ; - -logfile_group_name: - ident - { - LEX *lex= Lex; - lex->alter_tablespace_info= (new (thd->mem_root) - st_alter_tablespace()); - if (unlikely(lex->alter_tablespace_info == NULL)) - MYSQL_YYABORT; - lex->alter_tablespace_info->logfile_group_name= $1.str; - lex->sql_command= SQLCOM_ALTER_TABLESPACE; - } - ; - -ts_access_mode: - READ_ONLY_SYM - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_access_mode= TS_READ_ONLY; - } - | READ_WRITE_SYM - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_access_mode= TS_READ_WRITE; - } - | NOT_SYM ACCESSIBLE_SYM - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_access_mode= TS_NOT_ACCESSIBLE; - } - ; - -opt_ts_initial_size: - INITIAL_SIZE_SYM opt_equal size_number - { - LEX *lex= Lex; - lex->alter_tablespace_info->initial_size= $3; - } - ; - -opt_ts_autoextend_size: - AUTOEXTEND_SIZE_SYM opt_equal size_number - { - LEX *lex= Lex; - lex->alter_tablespace_info->autoextend_size= $3; - } - ; - -opt_ts_max_size: - MAX_SIZE_SYM opt_equal size_number - { - LEX *lex= Lex; - lex->alter_tablespace_info->max_size= $3; - } - ; - -opt_ts_extent_size: - EXTENT_SIZE_SYM opt_equal size_number - { - LEX *lex= Lex; - lex->alter_tablespace_info->extent_size= $3; - } - ; - -opt_ts_undo_buffer_size: - UNDO_BUFFER_SIZE_SYM opt_equal size_number - { - LEX *lex= Lex; - lex->alter_tablespace_info->undo_buffer_size= $3; - } - ; - -opt_ts_redo_buffer_size: - REDO_BUFFER_SIZE_SYM opt_equal size_number - { - LEX *lex= Lex; - lex->alter_tablespace_info->redo_buffer_size= $3; - } - ; - -opt_ts_nodegroup: - NODEGROUP_SYM opt_equal real_ulong_num - { - LEX *lex= Lex; - if (unlikely(lex->alter_tablespace_info->nodegroup_id != UNDEF_NODEGROUP)) - my_yyabort_error((ER_FILEGROUP_OPTION_ONLY_ONCE,MYF(0),"NODEGROUP")); - lex->alter_tablespace_info->nodegroup_id= $3; - } - ; - -opt_ts_comment: - COMMENT_SYM opt_equal TEXT_STRING_sys - { - LEX *lex= Lex; - if (unlikely(lex->alter_tablespace_info->ts_comment != NULL)) - my_yyabort_error((ER_FILEGROUP_OPTION_ONLY_ONCE,MYF(0),"COMMENT")); - lex->alter_tablespace_info->ts_comment= $3.str; - } - ; - -opt_ts_engine: - opt_storage ENGINE_SYM opt_equal storage_engines - { - LEX *lex= Lex; - if (unlikely(lex->alter_tablespace_info->storage_engine != NULL)) - my_yyabort_error((ER_FILEGROUP_OPTION_ONLY_ONCE, MYF(0), - "STORAGE ENGINE")); - lex->alter_tablespace_info->storage_engine= $4; - } - ; - -opt_ts_wait: - /* empty */ - | ts_wait - ; - -ts_wait: - WAIT_SYM - { - LEX *lex= Lex; - lex->alter_tablespace_info->wait_until_completed= TRUE; - } - | NO_WAIT_SYM - { - LEX *lex= Lex; - if (unlikely(!(lex->alter_tablespace_info->wait_until_completed))) - my_yyabort_error((ER_FILEGROUP_OPTION_ONLY_ONCE,MYF(0),"NO_WAIT")); - lex->alter_tablespace_info->wait_until_completed= FALSE; - } - ; - -size_number: - real_ulonglong_num { $$= $1;} - | IDENT_sys - { - if ($1.to_size_number(&$$)) - MYSQL_YYABORT; - } - ; - -/* - End tablespace part -*/ create_body: create_field_list_parens @@ -5068,8 +4743,13 @@ part_def_list: | part_def_list ',' part_definition {} ; +opt_partition: + /* empty */ + | PARTITION_SYM + ; + part_definition: - PARTITION_SYM + opt_partition { partition_info *part_info= Lex->part_info; partition_element *p_elem= new (thd->mem_root) partition_element(); @@ -5150,13 +4830,17 @@ opt_part_values: part_values_in {} | CURRENT_SYM { +#ifdef WITH_PARTITION_STORAGE_ENGINE if (Lex->part_values_current(thd)) MYSQL_YYABORT; +#endif } | HISTORY_SYM { +#ifdef WITH_PARTITION_STORAGE_ENGINE if (Lex->part_values_history(thd)) MYSQL_YYABORT; +#endif } | DEFAULT { @@ -5391,7 +5075,7 @@ sub_part_definition: part_info->use_default_num_subpartitions= FALSE; part_info->count_curr_subparts++; } - sub_name opt_part_options {} + sub_name opt_subpart_options {} ; sub_name: @@ -5405,17 +5089,36 @@ sub_name: opt_part_options: /* empty */ {} - | opt_part_option_list {} + | part_option_list {} ; -opt_part_option_list: - opt_part_option_list opt_part_option {} - | opt_part_option {} +part_option_list: + part_option_list part_option {} + | part_option {} ; -opt_part_option: +part_option: + server_part_option {} + | engine_defined_option + { + $1->link(&Lex->part_info->curr_part_elem->option_list, + &Lex->option_list_last); + } + ; + +opt_subpart_options: + /* empty */ {} + | subpart_option_list {} + ; + +subpart_option_list: + subpart_option_list server_part_option {} + | server_part_option {} + ; + +server_part_option: TABLESPACE opt_equal ident_or_text - { Lex->part_info->curr_part_elem->tablespace_name= $3.str; } + { /* Compatibility with MySQL */ } | opt_storage ENGINE_SYM opt_equal storage_engines { partition_info *part_info= Lex->part_info; @@ -5755,7 +5458,7 @@ create_table_option: Lex->create_info.used_fields|= HA_CREATE_USED_INDEXDIR; } | TABLESPACE ident - {Lex->create_info.tablespace= $2.str;} + { /* Compatiblity with MySQL */ } | STORAGE_SYM DISK_SYM {Lex->create_info.storage_media= HA_SM_DISK;} | STORAGE_SYM MEMORY_SYM @@ -5776,42 +5479,43 @@ create_table_option: Lex->create_info.used_fields|= HA_CREATE_USED_TRANSACTIONAL; Lex->create_info.transactional= $3; } - | IDENT_sys equal TEXT_STRING_sys + | engine_defined_option + { + $1->link(&Lex->create_info.option_list, &Lex->option_list_last); + } + | SEQUENCE_SYM opt_equal choice + { + Lex->create_info.used_fields|= HA_CREATE_USED_SEQUENCE; + Lex->create_info.sequence= ($3 == HA_CHOICE_YES); + } + | versioning_option + ; + +engine_defined_option: + IDENT_sys equal TEXT_STRING_sys { if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH)) my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str)); - (void) new (thd->mem_root) - engine_option_value($1, $3, true, - &Lex->create_info.option_list, - &Lex->option_list_last); + $$= new (thd->mem_root) engine_option_value($1, $3, true); + MYSQL_YYABORT_UNLESS($$); } | IDENT_sys equal ident { if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH)) my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str)); - (void) new (thd->mem_root) - engine_option_value($1, $3, false, - &Lex->create_info.option_list, - &Lex->option_list_last); + $$= new (thd->mem_root) engine_option_value($1, $3, false); + MYSQL_YYABORT_UNLESS($$); } | IDENT_sys equal real_ulonglong_num { - (void) new (thd->mem_root) - engine_option_value($1, $3, &Lex->create_info.option_list, - &Lex->option_list_last, thd->mem_root); + $$= new (thd->mem_root) engine_option_value($1, $3, thd->mem_root); + MYSQL_YYABORT_UNLESS($$); } | IDENT_sys equal DEFAULT { - (void) new (thd->mem_root) - engine_option_value($1, &Lex->create_info.option_list, - &Lex->option_list_last); + $$= new (thd->mem_root) engine_option_value($1); + MYSQL_YYABORT_UNLESS($$); } - | SEQUENCE_SYM opt_equal choice - { - Lex->create_info.used_fields|= HA_CREATE_USED_SEQUENCE; - Lex->create_info.sequence= ($3 == HA_CHOICE_YES); - } - | versioning_option ; opt_versioning_option: @@ -5824,7 +5528,7 @@ versioning_option: { if (unlikely(Lex->create_info.options & HA_LEX_CREATE_TMP_TABLE)) { - if (DBUG_EVALUATE_IF("sysvers_force", 0, 1)) + if (!DBUG_IF("sysvers_force")) { my_error(ER_VERS_NOT_SUPPORTED, MYF(0), "CREATE TEMPORARY TABLE"); MYSQL_YYABORT; @@ -6705,35 +6409,9 @@ asrow_attribute: serial_attribute: asrow_attribute - | IDENT_sys equal TEXT_STRING_sys + | engine_defined_option { - if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH)) - my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str)); - (void) new (thd->mem_root) - engine_option_value($1, $3, true, - &Lex->last_field->option_list, - &Lex->option_list_last); - } - | IDENT_sys equal ident - { - if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH)) - my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str)); - (void) new (thd->mem_root) - engine_option_value($1, $3, false, - &Lex->last_field->option_list, - &Lex->option_list_last); - } - | IDENT_sys equal real_ulonglong_num - { - (void) new (thd->mem_root) - engine_option_value($1, $3, &Lex->last_field->option_list, - &Lex->option_list_last, thd->mem_root); - } - | IDENT_sys equal DEFAULT - { - (void) new (thd->mem_root) - engine_option_value($1, &Lex->last_field->option_list, - &Lex->option_list_last); + $1->link(&Lex->last_field->option_list, &Lex->option_list_last); } | with_or_without_system VERSIONING_SYM { @@ -7139,33 +6817,9 @@ all_key_opt: { Lex->last_key->key_create_info.is_ignored= $1; } - | IDENT_sys equal TEXT_STRING_sys - { - if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH)) - my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str)); - (void) new (thd->mem_root) - engine_option_value($1, $3, true, &Lex->option_list, - &Lex->option_list_last); - } - | IDENT_sys equal ident + | engine_defined_option { - if (unlikely($3.length > ENGINE_OPTION_MAX_LENGTH)) - my_yyabort_error((ER_VALUE_TOO_LONG, MYF(0), $1.str)); - (void) new (thd->mem_root) - engine_option_value($1, $3, false, &Lex->option_list, - &Lex->option_list_last); - } - | IDENT_sys equal real_ulonglong_num - { - (void) new (thd->mem_root) - engine_option_value($1, $3, &Lex->option_list, - &Lex->option_list_last, thd->mem_root); - } - | IDENT_sys equal DEFAULT - { - (void) new (thd->mem_root) - engine_option_value($1, &Lex->option_list, - &Lex->option_list_last); + $1->link(&Lex->option_list, &Lex->option_list_last); } ; @@ -7203,10 +6857,12 @@ ignorability: key_list: key_list ',' key_part order_dir { + $3->asc= $4; Lex->last_key->columns.push_back($3, thd->mem_root); } | key_part order_dir { + $1->asc= $2; Lex->last_key->columns.push_back($1, thd->mem_root); } ; @@ -7410,26 +7066,6 @@ alter: Lex->pop_select(); //main select } - | ALTER TABLESPACE alter_tablespace_info - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_cmd_type= ALTER_TABLESPACE; - } - | ALTER LOGFILE_SYM GROUP_SYM alter_logfile_group_info - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_cmd_type= ALTER_LOGFILE_GROUP; - } - | ALTER TABLESPACE change_tablespace_info - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_cmd_type= CHANGE_FILE_TABLESPACE; - } - | ALTER TABLESPACE change_tablespace_access - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_cmd_type= ALTER_ACCESS_MODE_TABLESPACE; - } | ALTER SERVER_SYM ident_or_text { LEX *lex= Lex; @@ -7663,6 +7299,54 @@ alter_commands: if (Lex->stmt_alter_table_exchange_partition($6)) MYSQL_YYABORT; } + | CONVERT_SYM PARTITION_SYM alt_part_name_item + TO_SYM TABLE_SYM table_ident have_partitioning + { + LEX *lex= Lex; + if (Lex->stmt_alter_table($6)) + MYSQL_YYABORT; + lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_alter_table(); + if (unlikely(lex->m_sql_cmd == NULL)) + MYSQL_YYABORT; + lex->alter_info.partition_flags|= ALTER_PARTITION_CONVERT_OUT; + } + | CONVERT_SYM TABLE_SYM table_ident + { + LEX *lex= Lex; + if (!lex->first_select_lex()->add_table_to_list(thd, $3, nullptr, 0, + TL_READ_NO_INSERT, + MDL_SHARED_NO_WRITE)) + MYSQL_YYABORT; + + /* + This will appear as (new_db, new_name) in alter_ctx. + new_db will be IX-locked and new_name X-locked. + */ + lex->first_select_lex()->db= $3->db; + lex->name= $3->table; + if (lex->first_select_lex()->db.str == NULL && + lex->copy_db_to(&lex->first_select_lex()->db)) + MYSQL_YYABORT; + + lex->part_info= new (thd->mem_root) partition_info(); + if (unlikely(!lex->part_info)) + MYSQL_YYABORT; + + lex->part_info->num_parts= 1; + /* + OR-ed with ALTER_PARTITION_ADD because too many checks of + ALTER_PARTITION_ADD required. + */ + lex->alter_info.partition_flags|= ALTER_PARTITION_ADD | + ALTER_PARTITION_CONVERT_IN; + } + TO_SYM PARTITION_SYM part_definition + { + LEX *lex= Lex; + lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_alter_table(); + if (unlikely(lex->m_sql_cmd == NULL)) + MYSQL_YYABORT; + } ; remove_partitioning: @@ -7911,17 +7595,9 @@ alter_list_item: } | RENAME opt_to table_ident { - LEX *lex=Lex; - lex->first_select_lex()->db= $3->db; - if (lex->first_select_lex()->db.str == NULL && - lex->copy_db_to(&lex->first_select_lex()->db)) + if (Lex->stmt_alter_table($3)) MYSQL_YYABORT; - if (unlikely(check_table_name($3->table.str,$3->table.length, - FALSE)) || - ($3->db.str && unlikely(check_db_name((LEX_STRING*) &$3->db)))) - my_yyabort_error((ER_WRONG_TABLE_NAME, MYF(0), $3->table.str)); - lex->name= $3->table; - lex->alter_info.flags|= ALTER_RENAME; + Lex->alter_info.flags|= ALTER_RENAME; } | RENAME COLUMN_SYM opt_if_exists_table_element ident TO_SYM ident { @@ -8088,7 +7764,7 @@ opt_to: ; slave: - START_SYM SLAVE optional_connection_name slave_thread_opts + START_SYM SLAVE optional_connection_name slave_thread_opts optional_for_channel { LEX *lex=Lex; lex->sql_command = SQLCOM_SLAVE_START; @@ -8105,7 +7781,7 @@ slave: /* If you change this code don't forget to update STOP SLAVE too */ } {} - | STOP_SYM SLAVE optional_connection_name slave_thread_opts + | STOP_SYM SLAVE optional_connection_name slave_thread_opts optional_for_channel { LEX *lex=Lex; lex->sql_command = SQLCOM_SLAVE_STOP; @@ -11346,6 +11022,11 @@ variable_aux: ident_or_text SET_VAR expr { Item_func_set_user_var *item; + if (!$1.length) + { + thd->parse_error(); + MYSQL_YYABORT; + } $$= item= new (thd->mem_root) Item_func_set_user_var(thd, &$1, $3); if (unlikely($$ == NULL)) MYSQL_YYABORT; @@ -11355,6 +11036,11 @@ variable_aux: } | ident_or_text { + if (!$1.length) + { + thd->parse_error(); + MYSQL_YYABORT; + } $$= new (thd->mem_root) Item_func_get_user_var(thd, &$1); if (unlikely($$ == NULL)) MYSQL_YYABORT; @@ -13008,6 +12694,12 @@ select_var_ident: select_outvar select_outvar: '@' ident_or_text { + if (!$2.length) + { + thd->parse_error(); + MYSQL_YYABORT; + } + $$ = Lex->result ? new (thd->mem_root) my_var_user(&$2) : NULL; } | ident_or_text @@ -13156,16 +12848,6 @@ drop: lex->set_command(SQLCOM_DROP_TRIGGER, $3); lex->spname= $4; } - | DROP TABLESPACE tablespace_name opt_ts_engine opt_ts_wait - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_cmd_type= DROP_TABLESPACE; - } - | DROP LOGFILE_SYM GROUP_SYM logfile_group_name opt_ts_engine opt_ts_wait - { - LEX *lex= Lex; - lex->alter_tablespace_info->ts_cmd_type= DROP_LOGFILE_GROUP; - } | DROP SERVER_SYM opt_if_exists ident_or_text { Lex->set_command(SQLCOM_DROP_SERVER, $3); @@ -13270,6 +12952,8 @@ insert: { Lex->sql_command= SQLCOM_INSERT; Lex->duplicates= DUP_ERROR; + thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); + thd->get_stmt_da()->reset_current_row_for_warning(1); } insert_start insert_lock_option opt_ignore opt_into insert_table { @@ -13279,6 +12963,7 @@ insert: stmt_end { Lex->mark_first_table_as_inserting(); + thd->get_stmt_da()->reset_current_row_for_warning(0); } ; @@ -13287,6 +12972,8 @@ replace: { Lex->sql_command = SQLCOM_REPLACE; Lex->duplicates= DUP_REPLACE; + thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); + thd->get_stmt_da()->reset_current_row_for_warning(1); } insert_start replace_lock_option opt_into insert_table { @@ -13296,6 +12983,7 @@ replace: stmt_end { Lex->mark_first_table_as_inserting(); + thd->get_stmt_da()->reset_current_row_for_warning(0); } ; @@ -13447,6 +13135,7 @@ no_braces: opt_values ')' { LEX *lex=Lex; + thd->get_stmt_da()->inc_current_row_for_warning(); if (unlikely(lex->many_values.push_back(lex->insert_list, thd->mem_root))) MYSQL_YYABORT; @@ -13462,6 +13151,7 @@ no_braces_with_names: opt_values_with_names ')' { LEX *lex=Lex; + thd->get_stmt_da()->inc_current_row_for_warning(); if (unlikely(lex->many_values.push_back(lex->insert_list, thd->mem_root))) MYSQL_YYABORT; @@ -14041,7 +13731,8 @@ show_param: LEX *lex= Lex; lex->sql_command= SQLCOM_SHOW_RELAYLOG_EVENTS; } - opt_global_limit_clause + opt_global_limit_clause optional_for_channel + { } | keys_or_index from_or_in table_ident opt_db opt_where_clause { LEX *lex= Lex; @@ -14186,16 +13877,7 @@ show_param: MYSQL_YYABORT; Lex->sql_command = SQLCOM_SHOW_SLAVE_STAT; } - | SLAVE STATUS_SYM - { - LEX *lex= thd->lex; - lex->mi.connection_name= null_clex_str; - if (!(lex->m_sql_cmd= new (thd->mem_root) - Sql_cmd_show_slave_status())) - MYSQL_YYABORT; - lex->sql_command = SQLCOM_SHOW_SLAVE_STAT; - } - | SLAVE connection_name STATUS_SYM + | SLAVE optional_connection_name STATUS_SYM optional_for_channel { if (!(Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_show_slave_status())) @@ -14580,7 +14262,7 @@ flush_option: { Lex->type|= REFRESH_SLOW_LOG; } | BINARY LOGS_SYM opt_delete_gtid_domain { Lex->type|= REFRESH_BINARY_LOG; } - | RELAY LOGS_SYM optional_connection_name + | RELAY LOGS_SYM optional_connection_name optional_for_channel { LEX *lex= Lex; if (unlikely(lex->type & REFRESH_RELAY_LOG)) @@ -14731,7 +14413,8 @@ reset_options: reset_option: SLAVE { Lex->type|= REFRESH_SLAVE; } optional_connection_name - slave_reset_options { } + slave_reset_options optional_for_channel + { } | MASTER_SYM { Lex->type|= REFRESH_MASTER; @@ -15021,6 +14704,12 @@ field_or_var: simple_ident_nospvar {$$= $1;} | '@' ident_or_text { + if (!$2.length) + { + thd->parse_error(); + MYSQL_YYABORT; + } + $$= new (thd->mem_root) Item_user_var_as_out_param(thd, &$2); if (unlikely($$ == NULL)) MYSQL_YYABORT; @@ -16029,6 +15718,7 @@ keyword_sp_var_and_label: | CASCADED | CATALOG_NAME_SYM | CHAIN_SYM + | CHANNEL_SYM | CHANGED | CIPHER_SYM | CLIENT_SYM @@ -16836,6 +16526,12 @@ option_value_no_option_type: } | '@' ident_or_text equal { + if (!$2.length) + { + thd->parse_error(); + MYSQL_YYABORT; + } + if (sp_create_assignment_lex(thd, $1.str)) MYSQL_YYABORT; } @@ -18410,11 +18106,6 @@ sp_opt_default: | DEFAULT expr { $$ = $2; } ; -sp_pdparam: - sp_parameter_type sp_param_name_and_type { $2->mode=$1; } - | sp_param_name_and_type { $1->mode= sp_variable::MODE_IN; } - ; - sp_decl_variable_list_anchored: sp_decl_idents_init_vars TYPE_SYM OF_SYM optionally_qualified_column_ident @@ -18434,26 +18125,49 @@ sp_decl_variable_list_anchored: } ; -sp_param_name_and_type_anchored: - sp_param_name TYPE_SYM OF_SYM ident '.' ident +sp_param_name_and_mode: + sp_parameter_type sp_param_name + { + $2->mode= $1; + $$= $2; + } + | sp_param_name + ; + +sp_param: + sp_param_name_and_mode field_type + { + if (unlikely(Lex->sp_param_fill_definition($$= $1, $2))) + MYSQL_YYABORT; + } + | sp_param_name_and_mode ROW_SYM row_type_body + { + if (unlikely(Lex->sphead->spvar_fill_row(thd, $$= $1, $3))) + MYSQL_YYABORT; + } + | sp_param_anchored + ; + +sp_param_anchored: + sp_param_name_and_mode TYPE_SYM OF_SYM ident '.' ident { if (unlikely(Lex->sphead->spvar_fill_type_reference(thd, $$= $1, $4, $6))) MYSQL_YYABORT; } - | sp_param_name TYPE_SYM OF_SYM ident '.' ident '.' ident + | sp_param_name_and_mode TYPE_SYM OF_SYM ident '.' ident '.' ident { if (unlikely(Lex->sphead->spvar_fill_type_reference(thd, $$= $1, $4, $6, $8))) MYSQL_YYABORT; } - | sp_param_name ROW_SYM TYPE_SYM OF_SYM ident + | sp_param_name_and_mode ROW_SYM TYPE_SYM OF_SYM ident { if (unlikely(Lex->sphead->spvar_fill_table_rowtype_reference(thd, $$= $1, $5))) MYSQL_YYABORT; } - | sp_param_name ROW_SYM TYPE_SYM OF_SYM ident '.' ident + | sp_param_name_and_mode ROW_SYM TYPE_SYM OF_SYM ident '.' ident { if (unlikely(Lex->sphead->spvar_fill_table_rowtype_reference(thd, $$= $1, $5, $7))) MYSQL_YYABORT; @@ -18856,46 +18570,6 @@ sp_opt_inout: | IN_SYM OUT_SYM { $$= sp_variable::MODE_INOUT; } ; -sp_pdparam: - sp_param_name sp_opt_inout field_type - { - $1->mode= $2; - if (unlikely(Lex->sp_param_fill_definition($1, $3))) - MYSQL_YYABORT; - } - | sp_param_name sp_opt_inout sp_decl_ident '.' ident PERCENT_ORACLE_SYM TYPE_SYM - { - $1->mode= $2; - if (unlikely(Lex->sphead->spvar_fill_type_reference(thd, $1, $3, $5))) - MYSQL_YYABORT; - } - | sp_param_name sp_opt_inout sp_decl_ident '.' ident '.' ident PERCENT_ORACLE_SYM TYPE_SYM - { - $1->mode= $2; - if (unlikely(Lex->sphead->spvar_fill_type_reference(thd, $1, $3, $5, $7))) - MYSQL_YYABORT; - } - | sp_param_name sp_opt_inout sp_decl_ident PERCENT_ORACLE_SYM ROWTYPE_ORACLE_SYM - { - $1->mode= $2; - if (unlikely(Lex->sphead->spvar_fill_table_rowtype_reference(thd, $1, $3))) - MYSQL_YYABORT; - } - | sp_param_name sp_opt_inout sp_decl_ident '.' ident PERCENT_ORACLE_SYM ROWTYPE_ORACLE_SYM - { - $1->mode= $2; - if (unlikely(Lex->sphead->spvar_fill_table_rowtype_reference(thd, $1, $3, $5))) - MYSQL_YYABORT; - } - | sp_param_name sp_opt_inout ROW_SYM row_type_body - { - $1->mode= $2; - if (unlikely(Lex->sphead->spvar_fill_row(thd, $1, $4))) - MYSQL_YYABORT; - } - ; - - sp_proc_stmts1_implicit_block: { Lex->sp_block_init(thd); @@ -19329,23 +19003,45 @@ sp_decl_variable_list_anchored: } ; -sp_param_name_and_type_anchored: - sp_param_name sp_decl_ident '.' ident PERCENT_ORACLE_SYM TYPE_SYM +sp_param_name_and_mode: + sp_param_name sp_opt_inout + { + $1->mode= $2; + $$= $1; + } + ; + +sp_param: + sp_param_name_and_mode field_type + { + if (unlikely(Lex->sp_param_fill_definition($$= $1, $2))) + MYSQL_YYABORT; + } + | sp_param_name_and_mode ROW_SYM row_type_body + { + if (unlikely(Lex->sphead->spvar_fill_row(thd, $$= $1, $3))) + MYSQL_YYABORT; + } + | sp_param_anchored + ; + +sp_param_anchored: + sp_param_name_and_mode sp_decl_ident '.' ident PERCENT_ORACLE_SYM TYPE_SYM { if (unlikely(Lex->sphead->spvar_fill_type_reference(thd, $$= $1, $2, $4))) MYSQL_YYABORT; } - | sp_param_name sp_decl_ident '.' ident '.' ident PERCENT_ORACLE_SYM TYPE_SYM + | sp_param_name_and_mode sp_decl_ident '.' ident '.' ident PERCENT_ORACLE_SYM TYPE_SYM { if (unlikely(Lex->sphead->spvar_fill_type_reference(thd, $$= $1, $2, $4, $6))) MYSQL_YYABORT; } - | sp_param_name sp_decl_ident PERCENT_ORACLE_SYM ROWTYPE_ORACLE_SYM + | sp_param_name_and_mode sp_decl_ident PERCENT_ORACLE_SYM ROWTYPE_ORACLE_SYM { if (unlikely(Lex->sphead->spvar_fill_table_rowtype_reference(thd, $$= $1, $2))) MYSQL_YYABORT; } - | sp_param_name sp_decl_ident '.' ident PERCENT_ORACLE_SYM ROWTYPE_ORACLE_SYM + | sp_param_name_and_mode sp_decl_ident '.' ident PERCENT_ORACLE_SYM ROWTYPE_ORACLE_SYM { if (unlikely(Lex->sphead->spvar_fill_table_rowtype_reference(thd, $$= $1, $2, $4))) MYSQL_YYABORT; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 29cd5809ea1..a149bd23f33 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -740,6 +740,23 @@ static Sys_var_charptr_fscs Sys_character_sets_dir( READ_ONLY GLOBAL_VAR(charsets_dir), CMD_LINE(REQUIRED_ARG), DEFAULT(0)); +static bool check_engine_supports_temporary(sys_var *self, THD *thd, set_var *var) +{ + plugin_ref plugin= var->save_result.plugin; + if (!plugin) + return false; + DBUG_ASSERT(plugin); + handlerton *hton= plugin_hton(plugin); + DBUG_ASSERT(hton); + if (ha_check_storage_engine_flag(hton, HTON_TEMPORARY_NOT_SUPPORTED)) + { + my_error(ER_ILLEGAL_HA_CREATE_OPTION, MYF(0), hton_name(hton)->str, + "TEMPORARY"); + return true; + } + return false; +} + static bool check_not_null(sys_var *self, THD *thd, set_var *var) { return var->value && var->value->is_null(); @@ -2395,6 +2412,12 @@ static Sys_var_bit Sys_skip_parallel_replication( SESSION_ONLY(option_bits), NO_CMD_LINE, OPTION_RPL_SKIP_PARALLEL, DEFAULT(FALSE)); +static Sys_var_mybool Sys_binlog_alter_two_phase( + "binlog_alter_two_phase", + "When set, split ALTER at binary logging into 2 statements: " + "START ALTER and COMMIT/ROLLBACK ALTER", + SESSION_VAR(binlog_alter_two_phase), CMD_LINE(OPT_ARG), + DEFAULT(FALSE)); static bool check_gtid_ignore_duplicates(sys_var *self, THD *thd, set_var *var) @@ -4264,7 +4287,8 @@ static Sys_var_plugin Sys_storage_engine( static Sys_var_plugin Sys_default_tmp_storage_engine( "default_tmp_storage_engine", "The default storage engine for user-created temporary tables", SESSION_VAR(tmp_table_plugin), NO_CMD_LINE, - MYSQL_STORAGE_ENGINE_PLUGIN, DEFAULT(&default_tmp_storage_engine)); + MYSQL_STORAGE_ENGINE_PLUGIN, DEFAULT(&default_tmp_storage_engine), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_engine_supports_temporary)); static Sys_var_plugin Sys_enforce_storage_engine( "enforce_storage_engine", "Force the use of a storage engine for new tables", @@ -4470,10 +4494,7 @@ static bool fix_sql_log_bin_after_update(sys_var *self, THD *thd, { DBUG_ASSERT(type == OPT_SESSION); - if (thd->variables.sql_log_bin) - thd->variables.option_bits |= OPTION_BIN_LOG; - else - thd->variables.option_bits &= ~OPTION_BIN_LOG; + thd->set_binlog_bit(); return FALSE; } @@ -4903,7 +4924,9 @@ static Sys_var_mybool Sys_keep_files_on_create( "keep_files_on_create", "Don't overwrite stale .MYD and .MYI even if no directory is specified", SESSION_VAR(keep_files_on_create), CMD_LINE(OPT_ARG), - DEFAULT(FALSE)); + DEFAULT(FALSE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(0), + DEPRECATED("")); // since 10.8.0 static char *license; static Sys_var_charptr Sys_license( @@ -6040,16 +6063,6 @@ static Sys_var_mybool Sys_wsrep_desync ( ON_CHECK(wsrep_desync_check), ON_UPDATE(wsrep_desync_update)); -static Sys_var_mybool Sys_wsrep_strict_ddl ( - "wsrep_strict_ddl", - "If set, reject DDL on affected tables not supporting Galera replication", - GLOBAL_VAR(wsrep_strict_ddl), - CMD_LINE(OPT_ARG), DEFAULT(FALSE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, - ON_CHECK(0), - ON_UPDATE(wsrep_strict_ddl_update), - DEPRECATED("'@@wsrep_mode=STRICT_REPLICATION'")); // since 10.6.0 - static const char *wsrep_reject_queries_names[]= { "NONE", "ALL", "ALL_KILL", NullS }; static Sys_var_enum Sys_wsrep_reject_queries( "wsrep_reject_queries", "Variable to set to reject queries", @@ -6070,13 +6083,6 @@ static Sys_var_mybool Sys_wsrep_recover_datadir( READ_ONLY GLOBAL_VAR(wsrep_recovery), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); -static Sys_var_mybool Sys_wsrep_replicate_myisam( - "wsrep_replicate_myisam", "To enable myisam replication", - GLOBAL_VAR(wsrep_replicate_myisam), CMD_LINE(OPT_ARG), DEFAULT(FALSE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), - ON_UPDATE(wsrep_replicate_myisam_update), - DEPRECATED("'@@wsrep_mode=REPLICATE_MYISAM'")); // since 10.6.0 - static Sys_var_mybool Sys_wsrep_log_conflicts( "wsrep_log_conflicts", "To log multi-master conflicts", GLOBAL_VAR(wsrep_log_conflicts), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); @@ -6520,7 +6526,8 @@ static Sys_var_enum Sys_histogram_type( "Specifies type of the histograms created by ANALYZE. " "Possible values are: " "SINGLE_PREC_HB - single precision height-balanced, " - "DOUBLE_PREC_HB - double precision height-balanced.", + "DOUBLE_PREC_HB - double precision height-balanced, " + "JSON_HB - height-balanced, stored as JSON.", SESSION_VAR(histogram_type), CMD_LINE(REQUIRED_ARG), histogram_types, DEFAULT(1)); diff --git a/sql/table.cc b/sql/table.cc index 1a4875bde81..6d207bc847b 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2652,11 +2652,9 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, { auto field_type= handler->real_field_type(); - if (DBUG_EVALUATE_IF("error_vers_wrong_type", 1, 0)) - field_type= MYSQL_TYPE_BLOB; + DBUG_EXECUTE_IF("error_vers_wrong_type", field_type= MYSQL_TYPE_BLOB;); - switch (field_type) - { + switch (field_type) { case MYSQL_TYPE_TIMESTAMP2: break; case MYSQL_TYPE_LONGLONG: @@ -3340,7 +3338,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->column_bitmap_size * bitmap_count))) goto err; - my_bitmap_init(&share->all_set, bitmaps, share->fields, FALSE); + my_bitmap_init(&share->all_set, bitmaps, share->fields); bitmap_set_all(&share->all_set); if (share->check_set) { @@ -3351,7 +3349,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, my_bitmap_init(share->check_set, (my_bitmap_map*) ((uchar*) bitmaps + share->column_bitmap_size), - share->fields, FALSE); + share->fields); bitmap_clear_all(share->check_set); } @@ -4314,6 +4312,8 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, thd->restore_active_arena(&part_func_arena, &backup_arena); goto partititon_err; } + if (parse_engine_part_options(thd, outparam)) + goto err; outparam->part_info->is_auto_partitioned= share->auto_partitioned; DBUG_PRINT("info", ("autopartitioned: %u", share->auto_partitioned)); /* @@ -4370,26 +4370,26 @@ partititon_err: goto err; my_bitmap_init(&outparam->def_read_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); bitmaps+= bitmap_size; my_bitmap_init(&outparam->def_write_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); bitmaps+= bitmap_size; my_bitmap_init(&outparam->has_value_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); bitmaps+= bitmap_size; my_bitmap_init(&outparam->tmp_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); bitmaps+= bitmap_size; my_bitmap_init(&outparam->eq_join_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); bitmaps+= bitmap_size; my_bitmap_init(&outparam->cond_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); bitmaps+= bitmap_size; my_bitmap_init(&outparam->def_rpl_write_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); + (my_bitmap_map*) bitmaps, share->fields); outparam->default_column_bitmaps(); outparam->cond_selectivity= 1.0; @@ -9269,6 +9269,62 @@ bool TABLE::validate_default_values_of_unset_fields(THD *thd) const } +/* + Check assignment compatibility of a value list against an explicitly + specified field list, e.g. + INSERT INTO t1 (a,b) VALUES (1,2); +*/ +bool TABLE::check_assignability_explicit_fields(List<Item> fields, + List<Item> values, + bool ignore) +{ + DBUG_ENTER("TABLE::check_assignability_explicit_fields"); + DBUG_ASSERT(fields.elements == values.elements); + + List_iterator<Item> fi(fields); + List_iterator<Item> vi(values); + Item *f, *value; + while ((f= fi++) && (value= vi++)) + { + Item_field *item_field= f->field_for_view_update(); + if (!item_field) + { + /* + A non-updatable field of a view found. + This scenario is caught later and an error is raised. + We could eventually move error reporting here. For now just continue. + */ + continue; + } + if (value->check_assignability_to(item_field->field, ignore)) + DBUG_RETURN(true); + } + DBUG_RETURN(false); +} + + +/* + Check assignment compatibility for a value list against + all visible fields of the table, e.g. + INSERT INTO t1 VALUES (1,2); +*/ +bool TABLE::check_assignability_all_visible_fields(List<Item> &values, + bool ignore) const +{ + DBUG_ENTER("TABLE::check_assignability_all_visible_fields"); + DBUG_ASSERT(s->visible_fields == values.elements); + + List_iterator<Item> vi(values); + for (uint i= 0; i < s->fields; i++) + { + if (!field[i]->invisible && + (vi++)->check_assignability_to(field[i], ignore)) + DBUG_RETURN(true); + } + DBUG_RETURN(false); +} + + bool TABLE::insert_all_rows_into_tmp_table(THD *thd, TABLE *tmp_table, TMP_TABLE_PARAM *tmp_table_param, @@ -9953,6 +10009,9 @@ bool TR_table::update(ulonglong start_id, ulonglong end_id) int error= table->file->ha_write_row(table->record[0]); if (unlikely(error)) table->file->print_error(error, MYF(0)); + /* extra() is used to apply the bulk insert operation + on mysql/transaction_registry table */ + table->file->extra(HA_EXTRA_IGNORE_INSERT); return error; } diff --git a/sql/table.h b/sql/table.h index 436e0cb717a..83abc107971 100644 --- a/sql/table.h +++ b/sql/table.h @@ -694,16 +694,21 @@ class TABLE_STATISTICS_CB public: MEM_ROOT mem_root; /* MEM_ROOT to allocate statistical data for the table */ Table_statistics *table_stats; /* Structure to access the statistical data */ - ulong total_hist_size; /* Total size of all histograms */ + + /* + Whether the table has histograms. + (If the table has none, histograms_are_ready() can finish sooner) + */ + bool have_histograms; bool histograms_are_ready() const { - return !total_hist_size || hist_state.is_ready(); + return !have_histograms || hist_state.is_ready(); } bool start_histograms_load() { - return total_hist_size && hist_state.start_load(); + return have_histograms && hist_state.start_load(); } void end_histograms_load() { hist_state.end_load(); } @@ -1725,6 +1730,30 @@ public: Field **field_to_fill(); bool validate_default_values_of_unset_fields(THD *thd) const; + // Check if the value list is assignable to the explicit field list + static bool check_assignability_explicit_fields(List<Item> fields, + List<Item> values, + bool ignore); + // Check if the value list is assignable to all visible fields + bool check_assignability_all_visible_fields(List<Item> &values, + bool ignore) const; + /* + Check if the value list is assignable to: + - The explicit field list if fields.elements > 0, e.g. + INSERT INTO t1 (a,b) VALUES (1,2); + - All visible fields, if fields.elements==0, e.g. + INSERT INTO t1 VALUES (1,2); + */ + bool check_assignability_opt_fields(List<Item> fields, + List<Item> values, + bool ignore) const + { + DBUG_ASSERT(values.elements); + return fields.elements ? + check_assignability_explicit_fields(fields, values, ignore) : + check_assignability_all_visible_fields(values, ignore); + } + bool insert_all_rows_into_tmp_table(THD *thd, TABLE *tmp_table, TMP_TABLE_PARAM *tmp_table_param, diff --git a/sql/temporary_tables.cc b/sql/temporary_tables.cc index 8555aa1a7f5..3ba6a772549 100644 --- a/sql/temporary_tables.cc +++ b/sql/temporary_tables.cc @@ -340,7 +340,8 @@ bool THD::open_temporary_table(TABLE_LIST *tl) */ DBUG_ASSERT(!tl->derived); DBUG_ASSERT(!tl->schema_table); - DBUG_ASSERT(has_temporary_tables()); + DBUG_ASSERT(has_temporary_tables() || + (rgi_slave && rgi_slave->is_parallel_exec)); if (tl->open_type == OT_BASE_ONLY) { @@ -878,12 +879,20 @@ void THD::restore_tmp_table_share(TMP_TABLE_SHARE *share) bool THD::has_temporary_tables() { DBUG_ENTER("THD::has_temporary_tables"); - bool result= + bool result; #ifdef HAVE_REPLICATION - rgi_slave ? (rgi_slave->rli->save_temporary_tables && - !rgi_slave->rli->save_temporary_tables->is_empty()) : + if (rgi_slave) + { + mysql_mutex_lock(&rgi_slave->rli->data_lock); + result= rgi_slave->rli->save_temporary_tables && + !rgi_slave->rli->save_temporary_tables->is_empty(); + mysql_mutex_unlock(&rgi_slave->rli->data_lock); + } + else #endif - has_thd_temporary_tables(); + { + result= has_thd_temporary_tables(); + } DBUG_RETURN(result); } diff --git a/sql/thread_pool_info.cc b/sql/thread_pool_info.cc index 90ac6871784..e3ffd160a11 100644 --- a/sql/thread_pool_info.cc +++ b/sql/thread_pool_info.cc @@ -14,9 +14,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/ #include <mysql_version.h> -#include <mysql/plugin.h> #include <my_global.h> +#include <mysql/plugin.h> #include <sql_class.h> #include <sql_i_s.h> #include <mysql/plugin.h> diff --git a/sql/tztime.cc b/sql/tztime.cc index 1c482718cc8..8ddb9f0e30e 100644 --- a/sql/tztime.cc +++ b/sql/tztime.cc @@ -2004,7 +2004,11 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables) #endif /* ttid is increasing because we are reading using index */ - DBUG_ASSERT(ttid >= tmp_tz_info.typecnt); + if (ttid < tmp_tz_info.typecnt) + { + sql_print_error("mysql.time_zone_transition_type table is incorrectly defined or corrupted"); + goto end; + } tmp_tz_info.typecnt= ttid + 1; @@ -2517,7 +2521,7 @@ scan_tz_dir(char * name_end, uint symlink_recursion_level, uint verbose) { MY_DIR *cur_dir; char *name_end_tmp; - uint i; + size_t i; /* Sort directory data, to pass mtr tests on different platforms. */ if (!(cur_dir= my_dir(fullname, MYF(MY_WANT_STAT|MY_WANT_SORT)))) diff --git a/sql/uniques.cc b/sql/uniques.cc index a0cebe3e4dd..572d80f0b64 100644 --- a/sql/uniques.cc +++ b/sql/uniques.cc @@ -709,7 +709,7 @@ bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size, { IO_CACHE *outfile= &sort.io_cache; Merge_chunk *file_ptr= (Merge_chunk*) file_ptrs.buffer; - uint maxbuffer= file_ptrs.elements - 1; + uint maxbuffer= (uint)file_ptrs.elements - 1; my_off_t save_pos; bool error= 1; Sort_param sort_param; diff --git a/sql/unireg.cc b/sql/unireg.cc index 7f89f4e78b1..e6f52cd953e 100644 --- a/sql/unireg.cc +++ b/sql/unireg.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. - Copyright (c) 2009, 2020, MariaDB Corporation. + Copyright (c) 2009, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -685,6 +685,13 @@ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo, DBUG_PRINT("loop", ("flags: %lu key_parts: %d key_part: %p", key->flags, key->user_defined_key_parts, key->key_part)); + + /* For SPATIAL, FULLTEXT and HASH indexes (anything other than B-tree), + ignore the ASC/DESC attribute of columns. */ + const uchar ha_reverse_sort= + key->algorithm > HA_KEY_ALG_BTREE || key->flags & (HA_FULLTEXT|HA_SPATIAL) + ? 0 : HA_REVERSE_SORT; + for (key_part=key->key_part,key_part_end=key_part+key->user_defined_key_parts ; key_part != key_part_end ; key_part++) @@ -697,7 +704,8 @@ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo, int2store(pos,key_part->fieldnr+1+FIELD_NAME_USED); offset= (uint) (key_part->offset+data_offset+1); int2store(pos+2, offset); - pos[4]=0; // Sort order + key_part->key_part_flag &= ha_reverse_sort; + pos[4]= (uchar)(key_part->key_part_flag); int2store(pos+5,key_part->key_type); int2store(pos+7,key_part->length); pos+=9; diff --git a/sql/upgrade_conf_file.cc b/sql/upgrade_conf_file.cc index 543df7b9bdf..0d7bc603468 100644 --- a/sql/upgrade_conf_file.cc +++ b/sql/upgrade_conf_file.cc @@ -25,6 +25,8 @@ Note : the list below only includes the default-compiled server and none of the loadable plugins. */ +#include <my_global.h> +#include <my_sys.h> #include <windows.h> #include <initializer_list> #include <stdlib.h> @@ -158,51 +160,159 @@ static int cmp_strings(const void* a, const void *b) return strcmp((const char *)a, *(const char **)b); } -/** - Convert file from a previous version, by removing -*/ -int upgrade_config_file(const char *myini_path) + +#define MY_INI_SECTION_SIZE 32 * 1024 + 3 + +static bool is_utf8_str(const char *s) +{ + MY_STRCOPY_STATUS status; + const struct charset_info_st *cs= &my_charset_utf8mb4_bin; + size_t len= strlen(s); + if (!len) + return true; + cs->cset->well_formed_char_length(cs, s, s + len, len, &status); + return status.m_well_formed_error_pos == nullptr; +} + + +static UINT get_system_acp() { -#define MY_INI_SECTION_SIZE 32*1024 +3 + static DWORD system_acp; + if (system_acp) + return system_acp; + + char str_cp[10]; + int cch= GetLocaleInfo(GetSystemDefaultLCID(), LOCALE_IDEFAULTANSICODEPAGE, + str_cp, sizeof(str_cp)); + + system_acp= cch > 0 ? atoi(str_cp) : 1252; + + return system_acp; +} + + +static char *ansi_to_utf8(const char *s) +{ +#define MAX_STR_LEN MY_INI_SECTION_SIZE + static wchar_t utf16_buf[MAX_STR_LEN]; + static char utf8_buf[MAX_STR_LEN]; + if (MultiByteToWideChar(get_system_acp(), 0, s, -1, utf16_buf, MAX_STR_LEN)) + { + if (WideCharToMultiByte(CP_UTF8, 0, utf16_buf, -1, utf8_buf, MAX_STR_LEN, + 0, 0)) + return utf8_buf; + } + return 0; +} + +int fix_section(const char *myini_path, const char *section_name, + bool is_server) +{ + if (!is_server && GetACP() != CP_UTF8) + return 0; + static char section_data[MY_INI_SECTION_SIZE]; - for (const char *section_name : { "mysqld","server","mariadb" }) + DWORD size= GetPrivateProfileSection(section_name, section_data, + MY_INI_SECTION_SIZE, myini_path); + if (size == MY_INI_SECTION_SIZE - 2) { - DWORD size = GetPrivateProfileSection(section_name, section_data, MY_INI_SECTION_SIZE, myini_path); - if (size == MY_INI_SECTION_SIZE - 2) - { - return -1; - } + return -1; + } - for (char *keyval = section_data; *keyval; keyval += strlen(keyval) + 1) + for (char *keyval= section_data; *keyval; keyval += strlen(keyval)+1) + { + char varname[256]; + char *value; + char *key_end= strchr(keyval, '='); + if (!key_end) + key_end= keyval + strlen(keyval); + + if (key_end - keyval > sizeof(varname)) + continue; + + value= key_end + 1; + if (GetACP() == CP_UTF8 && !is_utf8_str(value)) { - char varname[256]; - char *key_end = strchr(keyval, '='); - if (!key_end) - key_end = keyval+ strlen(keyval); - - if (key_end - keyval > sizeof(varname)) - continue; - // copy and normalize (convert dash to underscore) to variable names - for (char *p = keyval, *q = varname;; p++,q++) + /*Convert a value, if it is not already UTF-8*/ + char *new_val= ansi_to_utf8(value); + if (new_val) { - if (p == key_end) - { - *q = 0; - break; - } - *q = (*p == '-') ? '_' : *p; + *key_end= 0; + fprintf(stdout, "Fixing variable '%s' charset, value=%s\n", keyval, + new_val); + WritePrivateProfileString(section_name, keyval, new_val, myini_path); + *key_end= '='; } - const char *v = (const char *)bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]), - sizeof(char *), cmp_strings); + } + if (!is_server) + continue; - if (v) + // Check if variable should be removed from config. + // First, copy and normalize (convert dash to underscore) to variable + // names + for (char *p= keyval, *q= varname;; p++, q++) + { + if (p == key_end) { - fprintf(stdout, "Removing variable '%s' from config file\n", varname); - // delete variable - *key_end = 0; - WritePrivateProfileString(section_name, keyval, 0, myini_path); + *q= 0; + break; } + *q= (*p == '-') ? '_' : *p; + } + const char *v= (const char *) bsearch(varname, removed_variables, sizeof(removed_variables) / sizeof(removed_variables[0]), + sizeof(char *), cmp_strings); + + if (v) + { + fprintf(stdout, "Removing variable '%s' from config file\n", varname); + // delete variable + *key_end= 0; + WritePrivateProfileString(section_name, keyval, 0, myini_path); } } return 0; } + +static bool is_mariadb_section(const char *name, bool *is_server) +{ + if (strncmp(name, "mysql", 5) + && strncmp(name, "mariadb", 7) + && strcmp(name, "client") + && strcmp(name, "client-server") + && strcmp(name, "server")) + { + return false; + } + + for (const char *section_name : {"mysqld", "server", "mariadb"}) + if (*is_server= !strcmp(section_name, name)) + break; + + return true; +} + + +/** + Convert file from a previous version, by removing obsolete variables + Also, fix values to be UTF8, if MariaDB is running in utf8 mode +*/ +int upgrade_config_file(const char *myini_path) +{ + static char all_sections[MY_INI_SECTION_SIZE]; + int sz= GetPrivateProfileSectionNamesA(all_sections, MY_INI_SECTION_SIZE, + myini_path); + if (!sz) + return 0; + if (sz > MY_INI_SECTION_SIZE - 2) + { + fprintf(stderr, "Too many sections in config file\n"); + return -1; + } + for (char *section= all_sections; *section; section+= strlen(section) + 1) + { + bool is_server_section; + if (is_mariadb_section(section, &is_server_section)) + fix_section(myini_path, section, is_server_section); + } + return 0; +} diff --git a/sql/winmain.cc b/sql/winmain.cc index 7def0aed531..2ed43130fa7 100644 --- a/sql/winmain.cc +++ b/sql/winmain.cc @@ -55,6 +55,7 @@ #include <windows.h> #include <string> #include <cassert> +#include <winservice.h> static SERVICE_STATUS svc_status{SERVICE_WIN32_OWN_PROCESS}; static SERVICE_STATUS_HANDLE svc_status_handle; diff --git a/sql/winservice.c b/sql/winservice.c index a11087e5cd5..d4e3bb0944d 100644 --- a/sql/winservice.c +++ b/sql/winservice.c @@ -134,6 +134,20 @@ static void get_datadir_from_ini(const char *ini, char *service_name, char *data } +static int fix_and_check_datadir(mysqld_service_properties *props) +{ + normalize_path(props->datadir, MAX_PATH); + /* Check if datadir really exists */ + if (GetFileAttributes(props->datadir) != INVALID_FILE_ATTRIBUTES) + return 0; + /* + It is possible, that datadir contains some unconvertable character. + We just pretend not to know what's the data directory + */ + props->datadir[0]= 0; + return 0; +} + /* Retrieve some properties from windows mysqld service binary path. We're interested in ini file location and datadir, and also in version of @@ -284,16 +298,9 @@ int get_mysql_service_properties(const wchar_t *bin_path, } } - if (props->datadir[0]) - { - normalize_path(props->datadir, MAX_PATH); - /* Check if datadir really exists */ - if (GetFileAttributes(props->datadir) == INVALID_FILE_ATTRIBUTES) - goto end; - } - else + if (props->datadir[0] == 0 || fix_and_check_datadir(props)) { - /* There is no datadir in ini file, bail out.*/ + /* There is no datadir in ini file, or non-existing dir, bail out.*/ goto end; } diff --git a/sql/winservice.h b/sql/winservice.h index f9ab3eda332..aa0528be5ee 100644 --- a/sql/winservice.h +++ b/sql/winservice.h @@ -17,11 +17,22 @@ /* Extract properties of a windows service binary path */ +#pragma once + #ifdef __cplusplus extern "C" { #endif -#include <windows.h> +#include <windows.h> +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4995) +#endif +#include <winsvc.h> +#ifdef _MSC_VER +#pragma warning(pop) +#endif + typedef struct mysqld_service_properties_st { char mysqld_exe[MAX_PATH]; @@ -32,9 +43,170 @@ typedef struct mysqld_service_properties_st int version_patch; } mysqld_service_properties; -extern int get_mysql_service_properties(const wchar_t *bin_path, +extern int get_mysql_service_properties(const wchar_t *bin_path, mysqld_service_properties *props); + +#if !defined(UNICODE) +/* + The following wrappers workaround Windows bugs + with CreateService/OpenService with ANSI codepage UTF8. + + Apparently, these function in ANSI mode, for this codepage only + do *not* behave as expected (as-if string parameters were + converted to UTF16 and "wide" function were called) +*/ +#include <malloc.h> +static inline wchar_t* awstrdup(const char *str) +{ + if (!str) + return NULL; + size_t len= strlen(str) + 1; + wchar_t *wstr= (wchar_t *) malloc(sizeof(wchar_t)*len); + if (MultiByteToWideChar(GetACP(), 0, str, (int)len, wstr, (int)len) == 0) + { + free(wstr); + return NULL; + } + return wstr; +} + +#define AWSTRDUP(dest, src) \ + dest= awstrdup(src); \ + if (src && !dest) \ + { \ + ok= FALSE; \ + last_error = ERROR_OUTOFMEMORY; \ + goto end; \ + } + +static inline SC_HANDLE my_OpenService(SC_HANDLE hSCManager, LPCSTR lpServiceName, DWORD dwDesiredAccess) +{ + wchar_t *w_ServiceName= NULL; + BOOL ok=TRUE; + DWORD last_error=0; + SC_HANDLE sch=NULL; + + AWSTRDUP(w_ServiceName, lpServiceName); + sch= OpenServiceW(hSCManager, w_ServiceName, dwDesiredAccess); + if (!sch) + { + ok= FALSE; + last_error= GetLastError(); + } + +end: + free(w_ServiceName); + if (!ok) + SetLastError(last_error); + return sch; +} + +static inline SC_HANDLE my_CreateService(SC_HANDLE hSCManager, + LPCSTR lpServiceName, LPCSTR lpDisplayName, + DWORD dwDesiredAccess, DWORD dwServiceType, + DWORD dwStartType, DWORD dwErrorControl, + LPCSTR lpBinaryPathName, LPCSTR lpLoadOrderGroup, + LPDWORD lpdwTagId, LPCSTR lpDependencies, + LPCSTR lpServiceStartName, LPCSTR lpPassword) +{ + wchar_t *w_ServiceName= NULL; + wchar_t *w_DisplayName= NULL; + wchar_t *w_BinaryPathName= NULL; + wchar_t *w_LoadOrderGroup= NULL; + wchar_t *w_Dependencies= NULL; + wchar_t *w_ServiceStartName= NULL; + wchar_t *w_Password= NULL; + SC_HANDLE sch = NULL; + DWORD last_error=0; + BOOL ok= TRUE; + + AWSTRDUP(w_ServiceName,lpServiceName); + AWSTRDUP(w_DisplayName,lpDisplayName); + AWSTRDUP(w_BinaryPathName, lpBinaryPathName); + AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup); + AWSTRDUP(w_Dependencies, lpDependencies); + AWSTRDUP(w_ServiceStartName, lpServiceStartName); + AWSTRDUP(w_Password, lpPassword); + + sch= CreateServiceW( + hSCManager, w_ServiceName, w_DisplayName, dwDesiredAccess, dwServiceType, + dwStartType, dwErrorControl, w_BinaryPathName, w_LoadOrderGroup, + lpdwTagId, w_Dependencies, w_ServiceStartName, w_Password); + if(!sch) + { + ok= FALSE; + last_error= GetLastError(); + } + +end: + free(w_ServiceName); + free(w_DisplayName); + free(w_BinaryPathName); + free(w_LoadOrderGroup); + free(w_Dependencies); + free(w_ServiceStartName); + free(w_Password); + + if (!ok) + SetLastError(last_error); + return sch; +} + +static inline BOOL my_ChangeServiceConfig(SC_HANDLE hService, DWORD dwServiceType, + DWORD dwStartType, DWORD dwErrorControl, + LPCSTR lpBinaryPathName, LPCSTR lpLoadOrderGroup, + LPDWORD lpdwTagId, LPCSTR lpDependencies, + LPCSTR lpServiceStartName, LPCSTR lpPassword, + LPCSTR lpDisplayName) +{ + wchar_t *w_DisplayName= NULL; + wchar_t *w_BinaryPathName= NULL; + wchar_t *w_LoadOrderGroup= NULL; + wchar_t *w_Dependencies= NULL; + wchar_t *w_ServiceStartName= NULL; + wchar_t *w_Password= NULL; + DWORD last_error=0; + BOOL ok= TRUE; + + AWSTRDUP(w_DisplayName, lpDisplayName); + AWSTRDUP(w_BinaryPathName, lpBinaryPathName); + AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup); + AWSTRDUP(w_Dependencies, lpDependencies); + AWSTRDUP(w_ServiceStartName, lpServiceStartName); + AWSTRDUP(w_Password, lpPassword); + + ok= ChangeServiceConfigW( + hService, dwServiceType, dwStartType, dwErrorControl, w_BinaryPathName, + w_LoadOrderGroup, lpdwTagId, w_Dependencies, w_ServiceStartName, + w_Password, w_DisplayName); + if (!ok) + { + last_error= GetLastError(); + } + +end: + free(w_DisplayName); + free(w_BinaryPathName); + free(w_LoadOrderGroup); + free(w_Dependencies); + free(w_ServiceStartName); + free(w_Password); + + if (last_error) + SetLastError(last_error); + return ok; +} +#undef AWSTRDUP + +#undef OpenService +#define OpenService my_OpenService +#undef ChangeServiceConfig +#define ChangeServiceConfig my_ChangeServiceConfig +#undef CreateService +#define CreateService my_CreateService +#endif + #ifdef __cplusplus } #endif diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index e0db00a3de3..cff31ab4484 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -90,7 +90,6 @@ my_bool wsrep_drupal_282555_workaround; // Retry autoinc insert after du my_bool wsrep_certify_nonPK; // Certify, even when no primary key ulong wsrep_certification_rules = WSREP_CERTIFICATION_RULES_STRICT; my_bool wsrep_recovery; // Recovery -my_bool wsrep_replicate_myisam; // Enable MyISAM replication my_bool wsrep_log_conflicts; my_bool wsrep_load_data_splitting= 0; // Commit load data every 10K intervals my_bool wsrep_slave_UK_checks; // Slave thread does UK checks @@ -100,9 +99,6 @@ my_bool wsrep_restart_slave; // Should mysql slave thread be my_bool wsrep_desync; // De(re)synchronize the node from the // cluster ulonglong wsrep_mode; -my_bool wsrep_strict_ddl; // Deprecated: Reject DDL to - // effected tables not - // supporting Galera replication bool wsrep_service_started; // If Galera was initialized long wsrep_slave_threads; // No. of slave appliers threads ulong wsrep_retry_autocommit; // Retry aborted autocommit trx @@ -1713,6 +1709,7 @@ wsrep_append_fk_parent_table(THD* thd, TABLE_LIST* tables, wsrep::key_array* key { bool fail= false; TABLE_LIST *table; + TABLE_LIST *table_last_in_list; for (table= tables; table; table= table->next_local) { @@ -1728,6 +1725,12 @@ wsrep_append_fk_parent_table(THD* thd, TABLE_LIST* tables, wsrep::key_array* key uint counter; MDL_savepoint mdl_savepoint= thd->mdl_context.mdl_savepoint(); + for (table_last_in_list= tables;;table_last_in_list= table_last_in_list->next_local) { + if (!table_last_in_list->next_local) { + break; + } + } + if (open_tables(thd, &tables, &counter, MYSQL_OPEN_FORCE_SHARED_HIGH_PRIO_MDL)) { WSREP_DEBUG("Unable to open table for FK checks for %s", wsrep_thd_query(thd)); @@ -1758,11 +1761,19 @@ exit: /* close the table and release MDL locks */ close_thread_tables(thd); thd->mdl_context.rollback_to_savepoint(mdl_savepoint); + bool invalidate_next_global= false; for (table= tables; table; table= table->next_local) { table->table= NULL; - table->next_global= NULL; table->mdl_request.ticket= NULL; + // We should invalidate `next_global` only for entries that are added + // in this function + if (table == table_last_in_list) { + invalidate_next_global= true; + } + if (invalidate_next_global) { + table->next_global= NULL; + } } return fail; @@ -2671,6 +2682,8 @@ static int wsrep_TOI_begin(THD *thd, const char *db, const char *table, } thd_proc_info(thd, "acquiring total order isolation"); + WSREP_DEBUG("wsrep_TOI_begin for %s", wsrep_thd_query(thd)); + THD_STAGE_INFO(thd, stage_waiting_isolation); wsrep::client_state& cs(thd->wsrep_cs()); @@ -3019,39 +3032,49 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, const MDL_ticket *ticket, const MDL_key *key) { - /* Fallback to the non-wsrep behaviour */ - if (!WSREP_ON) return; - THD *request_thd= requestor_ctx->get_thd(); THD *granted_thd= ticket->get_ctx()->get_thd(); + /* Fallback to the non-wsrep behaviour */ + if (!WSREP(request_thd)) return; + const char* schema= key->db_name(); int schema_len= key->db_name_length(); mysql_mutex_lock(&request_thd->LOCK_thd_data); - if (wsrep_thd_is_toi(request_thd) || - wsrep_thd_is_applying(request_thd)) { + if (wsrep_thd_is_toi(request_thd) || + wsrep_thd_is_applying(request_thd)) + { + WSREP_DEBUG("wsrep_handle_mdl_conflict request TOI/APPLY for %s", + wsrep_thd_query(request_thd)); + THD_STAGE_INFO(request_thd, stage_waiting_isolation); mysql_mutex_unlock(&request_thd->LOCK_thd_data); WSREP_MDL_LOG(DEBUG, "MDL conflict ", schema, schema_len, request_thd, granted_thd); ticket->wsrep_report(wsrep_debug); mysql_mutex_lock(&granted_thd->LOCK_thd_data); + if (wsrep_thd_is_toi(granted_thd) || wsrep_thd_is_applying(granted_thd)) { if (wsrep_thd_is_aborting(granted_thd)) { - WSREP_DEBUG("BF thread waiting for SR in aborting state"); + WSREP_DEBUG("BF thread waiting for SR in aborting state for %s", + wsrep_thd_query(request_thd)); + THD_STAGE_INFO(request_thd, stage_waiting_isolation); ticket->wsrep_report(wsrep_debug); mysql_mutex_unlock(&granted_thd->LOCK_thd_data); } else if (wsrep_thd_is_SR(granted_thd) && !wsrep_thd_is_SR(request_thd)) { - WSREP_MDL_LOG(INFO, "MDL conflict, DDL vs SR", + WSREP_MDL_LOG(INFO, "MDL conflict, DDL vs SR", schema, schema_len, request_thd, granted_thd); mysql_mutex_unlock(&granted_thd->LOCK_thd_data); + WSREP_DEBUG("wsrep_handle_mdl_conflict DDL vs SR for %s", + wsrep_thd_query(request_thd)); + THD_STAGE_INFO(request_thd, stage_waiting_isolation); wsrep_abort_thd(request_thd, granted_thd, 1); } else @@ -3066,7 +3089,9 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, else if (granted_thd->lex->sql_command == SQLCOM_FLUSH || granted_thd->mdl_context.has_explicit_locks()) { - WSREP_DEBUG("BF thread waiting for FLUSH"); + WSREP_DEBUG("BF thread waiting for FLUSH for %s", + wsrep_thd_query(request_thd)); + THD_STAGE_INFO(request_thd, stage_waiting_ddl); ticket->wsrep_report(wsrep_debug); mysql_mutex_unlock(&granted_thd->LOCK_thd_data); if (granted_thd->current_backup_stage != BACKUP_FINISHED && @@ -3077,8 +3102,10 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, } else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE) { - WSREP_DEBUG("DROP caused BF abort, conf %s", - wsrep_thd_transaction_state_str(granted_thd)); + WSREP_DEBUG("DROP caused BF abort, conf %s for %s", + wsrep_thd_transaction_state_str(granted_thd), + wsrep_thd_query(request_thd)); + THD_STAGE_INFO(request_thd, stage_waiting_isolation); ticket->wsrep_report(wsrep_debug); mysql_mutex_unlock(&granted_thd->LOCK_thd_data); wsrep_abort_thd(request_thd, granted_thd, 1); @@ -3087,7 +3114,11 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, { WSREP_MDL_LOG(DEBUG, "MDL conflict-> BF abort", schema, schema_len, request_thd, granted_thd); + WSREP_DEBUG("wsrep_handle_mdl_conflict -> BF abort for %s", + wsrep_thd_query(request_thd)); + THD_STAGE_INFO(request_thd, stage_waiting_isolation); ticket->wsrep_report(wsrep_debug); + if (granted_thd->wsrep_trx().active()) { mysql_mutex_unlock(&granted_thd->LOCK_thd_data); @@ -3100,14 +3131,16 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, thd is BF, BF abort and wait. */ mysql_mutex_unlock(&granted_thd->LOCK_thd_data); + if (wsrep_thd_is_BF(request_thd, FALSE)) { ha_abort_transaction(request_thd, granted_thd, TRUE); } else { - WSREP_MDL_LOG(INFO, "MDL unknown BF-BF conflict", schema, schema_len, - request_thd, granted_thd); + WSREP_MDL_LOG(INFO, "MDL unknown BF-BF conflict", + schema, schema_len, + request_thd, granted_thd); ticket->wsrep_report(true); unireg_abort(1); } diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index c9e08d77b6c..d14e5995771 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -72,7 +72,6 @@ extern long int wsrep_protocol_version; extern my_bool wsrep_desync; extern ulong wsrep_reject_queries; extern my_bool wsrep_recovery; -extern my_bool wsrep_replicate_myisam; extern my_bool wsrep_log_conflicts; extern ulong wsrep_mysql_replication_bundle; extern my_bool wsrep_load_data_splitting; @@ -91,7 +90,6 @@ extern bool wsrep_gtid_mode; extern uint32 wsrep_gtid_domain_id; extern std::atomic <bool > wsrep_thread_create_failed; extern ulonglong wsrep_mode; -extern my_bool wsrep_strict_ddl; enum enum_wsrep_reject_types { WSREP_REJECT_NONE, /* nothing rejected */ diff --git a/sql/wsrep_trans_observer.h b/sql/wsrep_trans_observer.h index 8f998244ee6..42ba56fa2c2 100644 --- a/sql/wsrep_trans_observer.h +++ b/sql/wsrep_trans_observer.h @@ -289,12 +289,14 @@ static inline int wsrep_before_commit(THD* thd, bool all) WSREP_DEBUG("wsrep_before_commit: %d, %lld", wsrep_is_real(thd, all), (long long)wsrep_thd_trx_seqno(thd)); + THD_STAGE_INFO(thd, stage_waiting_certification); int ret= 0; DBUG_ASSERT(wsrep_run_commit_hook(thd, all)); + if ((ret= thd->wsrep_cs().before_commit()) == 0) { DBUG_ASSERT(!thd->wsrep_trx().ws_meta().gtid().is_undefined()); - if (!thd->variables.gtid_seq_no && + if (!thd->variables.gtid_seq_no && (thd->wsrep_trx().ws_meta().flags() & wsrep::provider::flag::commit)) { uint64 seqno= 0; diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 5ec32d63626..0de1b034953 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -854,10 +854,11 @@ bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var) return true; } } else { + THD_STAGE_INFO(thd, stage_waiting_flow); ret= Wsrep_server_state::instance().provider().resync(); if (ret != WSREP_OK) { WSREP_WARN ("SET resync failed %d for schema: %s, query: %s", ret, - thd->get_db(), thd->query()); + thd->get_db(), wsrep_thd_query(thd)); my_error (ER_CANNOT_USER, MYF(0), "'resync'", thd->query()); return true; } @@ -1118,24 +1119,3 @@ bool wsrep_gtid_domain_id_update(sys_var* self, THD *thd, enum_var_type) return false; } -bool wsrep_strict_ddl_update(sys_var *self, THD* thd, enum_var_type var_type) -{ - // In case user still sets wsrep_strict_ddl we set new - // option to wsrep_mode - if (wsrep_strict_ddl) - wsrep_mode|= WSREP_MODE_STRICT_REPLICATION; - else - wsrep_mode&= (~WSREP_MODE_STRICT_REPLICATION); - return false; -} - -bool wsrep_replicate_myisam_update(sys_var *self, THD* thd, enum_var_type var_type) -{ - // In case user still sets wsrep_replicate_myisam we set new - // option to wsrep_mode - if (wsrep_replicate_myisam) - wsrep_mode|= WSREP_MODE_REPLICATE_MYISAM; - else - wsrep_mode&= (~WSREP_MODE_REPLICATE_MYISAM); - return false; -} diff --git a/sql/wsrep_var.h b/sql/wsrep_var.h index 7908e873795..0f811d70928 100644 --- a/sql/wsrep_var.h +++ b/sql/wsrep_var.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2013 Codership Oy <info@codership.com> +/* Copyright (C) 2013-2021 Codership Oy <info@codership.com> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -109,8 +109,6 @@ extern bool wsrep_gtid_seq_no_check CHECK_ARGS; extern bool wsrep_gtid_domain_id_update UPDATE_ARGS; extern bool wsrep_mode_check CHECK_ARGS; -extern bool wsrep_strict_ddl_update UPDATE_ARGS; -extern bool wsrep_replicate_myisam_update UPDATE_ARGS; #else /* WITH_WSREP */ #define wsrep_provider_init(X) |