From 1adc559370cc53ca69e225739a942287eba1b974 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 3 Jan 2020 22:54:28 +0530 Subject: Making group_by test stable --- mysql-test/r/group_by.result | 6 +++--- mysql-test/t/group_by.test | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result index 9e50ba12db3..d135c196874 100644 --- a/mysql-test/r/group_by.result +++ b/mysql-test/r/group_by.result @@ -2796,15 +2796,15 @@ drop table t1; # MDEV-20922: Adding an order by changes the query results # CREATE TABLE t1(a int, b int); -INSERT INTO t1 values (1, 100), (2, 200), (3, 100), (4, 200); +INSERT INTO t1 values (1, 100), (2, 200), (3, 100), (4, 200), (5, 200); create view v1 as select a, b+1 as x from t1; SELECT x, COUNT(DISTINCT a) AS y FROM v1 GROUP BY x ORDER BY y; x y 101 2 -201 2 +201 3 SELECT b+1 AS x, COUNT(DISTINCT a) AS y FROM t1 GROUP BY x ORDER BY y; x y 101 2 -201 2 +201 3 drop view v1; drop table t1; diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test index 71e29d55303..4cb4b32e8b8 100644 --- a/mysql-test/t/group_by.test +++ b/mysql-test/t/group_by.test @@ -1920,7 +1920,7 @@ drop table t1; --echo # CREATE TABLE t1(a int, b int); -INSERT INTO t1 values (1, 100), (2, 200), (3, 100), (4, 200); +INSERT INTO t1 values (1, 100), (2, 200), (3, 100), (4, 200), (5, 200); create view v1 as select a, b+1 as x from t1; -- cgit v1.2.1 From fd899b3bbd2c7107f708ed46ba3798c48e2cfadd Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Mon, 23 Dec 2019 01:18:39 +0800 Subject: Lets add another intrusive double linked list! Features: * STL-like interface * Fast modification: no branches on insertion or deletion * Fast iteration: one pointer dereference and one pointer comparison * Your class can be a part of several lists Modeled after std::list but currently has fewer methods (not complete yet) For even more performance it's possible to customize list with templates so it won't have size counter variable or won't NULLify unlinked node. How existing lists differ? No existing lists support STL-like interface. I_List: * slower iteration (one more branch on iteration) * element can't be a part of two lists simultaneously I_P_List: * slower modification (branches, except for the fastest push_back() case) * slower iteration (one more branch on iteration) UT_LIST_BASE_NODE_T: * slower modification (branches) Three UT_LISTs were replaced: two in fil_system_t and one in dyn_buf_t. --- include/intrusive_list.h | 188 +++++++++++++++++++++++++++++++++++++ storage/innobase/fil/fil0crypt.cc | 2 +- storage/innobase/fil/fil0fil.cc | 144 ++++++++++++++-------------- storage/innobase/include/dyn0buf.h | 96 +++++++++---------- storage/innobase/include/fil0fil.h | 26 +++-- 5 files changed, 315 insertions(+), 141 deletions(-) create mode 100644 include/intrusive_list.h diff --git a/include/intrusive_list.h b/include/intrusive_list.h new file mode 100644 index 00000000000..23534596b93 --- /dev/null +++ b/include/intrusive_list.h @@ -0,0 +1,188 @@ +/* + Copyright (c) 2019, MariaDB + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +#pragma once + +#include +#include + +namespace intrusive +{ + +// Derive your class from this struct to insert to a linked list. +template struct list_node +{ + list_node(list_node *next= NULL, list_node *prev= NULL) + : next(next), prev(prev) + { + } + + list_node *next; + list_node *prev; +}; + +// Modelled after std::list +template class list +{ +public: + typedef list_node ListNode; + class Iterator; + + // All containers in C++ should define these types to implement generic + // container interface. + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type &reference; + typedef const value_type &const_reference; + typedef T *pointer; + typedef const T *const_pointer; + typedef Iterator iterator; + typedef const Iterator const_iterator; + typedef std::reverse_iterator reverse_iterator; + typedef std::reverse_iterator const_reverse_iterator; + + class Iterator + { + public: + // All iterators in C++ should define these types to implement generic + // iterator interface. + typedef std::bidirectional_iterator_tag iterator_category; + typedef T value_type; + typedef std::ptrdiff_t difference_type; + typedef T *pointer; + typedef T &reference; + + Iterator(ListNode *node) : node_(node) {} + + Iterator &operator++() + { + node_= node_->next; + return *this; + } + Iterator operator++(int) + { + Iterator tmp(*this); + operator++(); + return tmp; + } + + Iterator &operator--() + { + node_= node_->prev; + return *this; + } + Iterator operator--(int) + { + Iterator tmp(*this); + operator--(); + return tmp; + } + + reference operator*() { return *static_cast(node_); } + pointer operator->() { return static_cast(node_); } + + bool operator==(const Iterator &rhs) { return node_ == rhs.node_; } + bool operator!=(const Iterator &rhs) { return !(*this == rhs); } + + private: + ListNode *node_; + + friend class list; + }; + + list() : sentinel_(&sentinel_, &sentinel_), size_(0) {} + + reference front() { return *begin(); } + reference back() { return *--end(); } + + iterator begin() { return iterator(sentinel_.next); } + const_iterator begin() const + { + return iterator(const_cast(sentinel_.next)); + } + iterator end() { return iterator(&sentinel_); } + const_iterator end() const + { + return iterator(const_cast(&sentinel_)); + } + + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return reverse_iterator(begin()); } + + bool empty() const { return size_ == 0; } + size_type size() const { return size_; } + + void clear() + { + sentinel_.next= &sentinel_; + sentinel_.prev= &sentinel_; + size_= 0; + } + + iterator insert(iterator pos, reference value) + { + ListNode *curr= pos.node_; + ListNode *prev= pos.node_->prev; + + prev->next= &value; + curr->prev= &value; + + static_cast(value).prev= prev; + static_cast(value).next= curr; + + ++size_; + return iterator(&value); + } + + iterator erase(iterator pos) + { + ListNode *prev= pos.node_->prev; + ListNode *next= pos.node_->next; + + prev->next= next; + next->prev= prev; + + // This is not required for list functioning. But maybe it'll prevent bugs + // and ease debugging. + ListNode *curr= pos.node_; + curr->prev= NULL; + curr->next= NULL; + + --size_; + return next; + } + + void push_back(reference value) { insert(end(), value); } + void pop_back() { erase(end()); } + + void push_front(reference value) { insert(begin(), value); } + void pop_front() { erase(begin()); } + + // STL version is O(n) but this is O(1) because an element can't be inserted + // several times in the same intrusive list. + void remove(reference value) { erase(iterator(&value)); } + +private: + ListNode sentinel_; + size_type size_; +}; + +} // namespace intrusive diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index d1aca92ef62..2211370ada1 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2346,7 +2346,7 @@ static void fil_crypt_rotation_list_fill() } } - UT_LIST_ADD_LAST(fil_system->rotation_list, space); + fil_system->rotation_list.push_back(*space); } } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index e37cb963c28..64c2834a635 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -898,9 +898,7 @@ skip_flush: if (space->is_in_unflushed_spaces() && fil_space_is_flushed(space)) { - UT_LIST_REMOVE( - fil_system->unflushed_spaces, - space); + fil_system->unflushed_spaces.remove(*space); } } @@ -1201,7 +1199,7 @@ fil_node_close_to_free( } else if (space->is_in_unflushed_spaces() && fil_space_is_flushed(space)) { - UT_LIST_REMOVE(fil_system->unflushed_spaces, space); + fil_system->unflushed_spaces.remove(*space); } fil_node_close_file(node); @@ -1232,12 +1230,12 @@ fil_space_detach( ut_ad(!fil_buffering_disabled(space)); - UT_LIST_REMOVE(fil_system->unflushed_spaces, space); + fil_system->unflushed_spaces.remove(*space); } if (space->is_in_rotation_list()) { - UT_LIST_REMOVE(fil_system->rotation_list, space); + fil_system->rotation_list.remove(*space); } UT_LIST_REMOVE(fil_system->space_list, space); @@ -1463,7 +1461,7 @@ fil_space_create( srv_encrypt_tables)) { /* Key rotation is not enabled, need to inform background encryption threads. */ - UT_LIST_ADD_LAST(fil_system->rotation_list, space); + fil_system->rotation_list.push_back(*space); mutex_exit(&fil_system->mutex); mutex_enter(&fil_crypt_threads_mutex); os_event_set(fil_crypt_threads_event); @@ -1799,8 +1797,7 @@ fil_init( ut_a(hash_size > 0); ut_a(max_n_open > 0); - fil_system = static_cast( - ut_zalloc_nokey(sizeof(*fil_system))); + fil_system = new fil_system_t(); mutex_create(LATCH_ID_FIL_SYSTEM, &fil_system->mutex); @@ -1809,9 +1806,6 @@ fil_init( UT_LIST_INIT(fil_system->LRU, &fil_node_t::LRU); UT_LIST_INIT(fil_system->space_list, &fil_space_t::space_list); - UT_LIST_INIT(fil_system->rotation_list, &fil_space_t::rotation_list); - UT_LIST_INIT(fil_system->unflushed_spaces, - &fil_space_t::unflushed_spaces); UT_LIST_INIT(fil_system->named_spaces, &fil_space_t::named_spaces); fil_system->max_n_open = max_n_open; @@ -4768,8 +4762,8 @@ fil_node_complete_io(fil_node_t* node, const IORequest& type) if (!node->space->is_in_unflushed_spaces()) { - UT_LIST_ADD_FIRST(fil_system->unflushed_spaces, - node->space); + fil_system->unflushed_spaces.push_front( + *node->space); } } } @@ -5237,7 +5231,6 @@ void fil_flush_file_spaces( fil_type_t purpose) { - fil_space_t* space; ulint* space_ids; ulint n_space_ids; @@ -5245,30 +5238,25 @@ fil_flush_file_spaces( mutex_enter(&fil_system->mutex); - n_space_ids = UT_LIST_GET_LEN(fil_system->unflushed_spaces); + n_space_ids = fil_system->unflushed_spaces.size(); if (n_space_ids == 0) { mutex_exit(&fil_system->mutex); return; } - /* Assemble a list of space ids to flush. Previously, we - traversed fil_system->unflushed_spaces and called UT_LIST_GET_NEXT() - on a space that was just removed from the list by fil_flush(). - Thus, the space could be dropped and the memory overwritten. */ space_ids = static_cast( ut_malloc_nokey(n_space_ids * sizeof(*space_ids))); n_space_ids = 0; - for (space = UT_LIST_GET_FIRST(fil_system->unflushed_spaces); - space; - space = UT_LIST_GET_NEXT(unflushed_spaces, space)) { - - if (space->purpose == purpose - && !space->is_stopping()) { + for (intrusive::list::iterator it + = fil_system->unflushed_spaces.begin(), + end = fil_system->unflushed_spaces.end(); + it != end; ++it) { - space_ids[n_space_ids++] = space->id; + if (it->purpose == purpose && !it->is_stopping()) { + space_ids[n_space_ids++] = it->id; } } @@ -5420,12 +5408,12 @@ fil_close(void) hash_table_free(fil_system->name_hash); ut_a(UT_LIST_GET_LEN(fil_system->LRU) == 0); - ut_a(UT_LIST_GET_LEN(fil_system->unflushed_spaces) == 0); + ut_a(fil_system->unflushed_spaces.size() == 0); ut_a(UT_LIST_GET_LEN(fil_system->space_list) == 0); mutex_free(&fil_system->mutex); - ut_free(fil_system); + delete fil_system; fil_system = NULL; fil_space_crypt_cleanup(); @@ -5990,8 +5978,8 @@ fil_space_remove_from_keyrotation(fil_space_t* space) ut_ad(space); if (space->n_pending_ops == 0 && space->is_in_rotation_list()) { - ut_a(UT_LIST_GET_LEN(fil_system->rotation_list) > 0); - UT_LIST_REMOVE(fil_system->rotation_list, space); + ut_a(!fil_system->rotation_list.empty()); + fil_system->rotation_list.remove(*space); } } @@ -6008,55 +5996,56 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last */ -fil_space_t* -fil_system_t::keyrotate_next( - fil_space_t* prev_space, - bool recheck, - uint key_version) +fil_space_t *fil_system_t::keyrotate_next(fil_space_t *prev_space, + bool recheck, uint key_version) { - mutex_enter(&fil_system->mutex); + mutex_enter(&fil_system->mutex); - /* If one of the encryption threads already started the encryption - of the table then don't remove the unencrypted spaces from - rotation list + /* If one of the encryption threads already started the encryption + of the table then don't remove the unencrypted spaces from rotation list - If there is a change in innodb_encrypt_tables variables value then - don't remove the last processed tablespace from the rotation list. */ - const bool remove = ((!recheck || prev_space->crypt_data) - && (!key_version == !srv_encrypt_tables)); + If there is a change in innodb_encrypt_tables variables value then + don't remove the last processed tablespace from the rotation list. */ + const bool remove= (!recheck || prev_space->crypt_data) && + !key_version == !srv_encrypt_tables; + intrusive::list::iterator it= + prev_space == NULL ? fil_system->rotation_list.end() : prev_space; - fil_space_t* space = prev_space; + if (it == fil_system->rotation_list.end()) + { + it= fil_system->rotation_list.begin(); + } + else + { + ut_ad(prev_space->n_pending_ops > 0); - if (prev_space == NULL) { - space = UT_LIST_GET_FIRST(fil_system->rotation_list); + /* Move on to the next fil_space_t */ + prev_space->n_pending_ops--; - /* We can trust that space is not NULL because we - checked list length above */ - } else { - ut_ad(space->n_pending_ops > 0); + ++it; - /* Move on to the next fil_space_t */ - space->n_pending_ops--; + while (it != fil_system->rotation_list.end() && + (UT_LIST_GET_LEN(it->chain) == 0 || it->is_stopping())) + { + ++it; + } - space = UT_LIST_GET_NEXT(rotation_list, space); + if (remove) + { + fil_space_remove_from_keyrotation(prev_space); + } + } - while (space != NULL - && (UT_LIST_GET_LEN(space->chain) == 0 - || space->is_stopping())) { - space = UT_LIST_GET_NEXT(rotation_list, space); - } + fil_space_t *space= it == fil_system->rotation_list.end() ? NULL : &*it; - if (remove) { - fil_space_remove_from_keyrotation(prev_space); - } - } + if (space) + { + space->n_pending_ops++; + } - if (space != NULL) { - space->n_pending_ops++; - } + mutex_exit(&fil_system->mutex); - mutex_exit(&fil_system->mutex); - return(space); + return space; } /** Determine the block size of the data file. @@ -6134,18 +6123,21 @@ fil_space_set_punch_hole( /** Checks that this tablespace in a list of unflushed tablespaces. @return true if in a list */ -bool fil_space_t::is_in_unflushed_spaces() const { - ut_ad(mutex_own(&fil_system->mutex)); +bool fil_space_t::is_in_unflushed_spaces() const +{ + ut_ad(mutex_own(&fil_system->mutex)); - return fil_system->unflushed_spaces.start == this - || unflushed_spaces.next || unflushed_spaces.prev; + return static_cast *>( + this) + ->next; } /** Checks that this tablespace needs key rotation. @return true if in a rotation list */ -bool fil_space_t::is_in_rotation_list() const { - ut_ad(mutex_own(&fil_system->mutex)); +bool fil_space_t::is_in_rotation_list() const +{ + ut_ad(mutex_own(&fil_system->mutex)); - return fil_system->rotation_list.start == this || rotation_list.next - || rotation_list.prev; + return static_cast *>(this) + ->next; } diff --git a/storage/innobase/include/dyn0buf.h b/storage/innobase/include/dyn0buf.h index a7a8e1dea1e..311f6518943 100644 --- a/storage/innobase/include/dyn0buf.h +++ b/storage/innobase/include/dyn0buf.h @@ -29,7 +29,8 @@ Created 2013-03-16 Sunny Bains #include "mem0mem.h" #include "dyn0types.h" -#include "ut0lst.h" +#include "intrusive_list.h" + /** Class that manages dynamic buffers. It uses a UT_LIST of dyn_buf_t::block_t instances. We don't use STL containers in @@ -42,12 +43,7 @@ template class dyn_buf_t { public: - class block_t; - - typedef UT_LIST_NODE_T(block_t) block_node_t; - typedef UT_LIST_BASE_NODE_T(block_t) block_list_t; - - class block_t { + class block_t : public intrusive::list_node<> { public: block_t() @@ -157,16 +153,13 @@ public: /** SIZE - sizeof(m_node) + sizeof(m_used) */ enum { MAX_DATA_SIZE = SIZE - - sizeof(block_node_t) + - sizeof(intrusive::list_node<>) + sizeof(ib_uint32_t) }; /** Storage */ byte m_data[MAX_DATA_SIZE]; - /** Doubly linked list node. */ - block_node_t m_node; - /** number of data bytes used in this block; DYN_BLOCK_FULL_FLAG is set when the block becomes full */ ib_uint32_t m_used; @@ -174,6 +167,8 @@ public: friend class dyn_buf_t; }; + typedef intrusive::list list_t; + enum { MAX_DATA_SIZE = block_t::MAX_DATA_SIZE}; /** Default constructor */ @@ -182,7 +177,6 @@ public: m_heap(), m_size() { - UT_LIST_INIT(m_list, &block_t::m_node); push_back(&m_first_block); } @@ -200,11 +194,11 @@ public: m_heap = NULL; /* Initialise the list and add the first block. */ - UT_LIST_INIT(m_list, &block_t::m_node); - push_back(&m_first_block); + m_list.clear(); + m_list.push_back(m_first_block); } else { m_first_block.init(); - ut_ad(UT_LIST_GET_LEN(m_list) == 1); + ut_ad(m_list.size() == 1); } m_size = 0; @@ -236,7 +230,7 @@ public: @param ptr end of used space */ void close(const byte* ptr) { - ut_ad(UT_LIST_GET_LEN(m_list) > 0); + ut_ad(!m_list.empty()); block_t* block = back(); m_size -= block->used(); @@ -324,11 +318,10 @@ public: #ifdef UNIV_DEBUG ulint total_size = 0; - for (const block_t* block = UT_LIST_GET_FIRST(m_list); - block != NULL; - block = UT_LIST_GET_NEXT(m_node, block)) { - - total_size += block->used(); + for (typename list_t::iterator it = m_list.begin(), + end = m_list.end(); + it != end; ++it) { + total_size += it->used(); } ut_ad(total_size == m_size); @@ -342,12 +335,12 @@ public: template bool for_each_block(Functor& functor) const { - for (const block_t* block = UT_LIST_GET_FIRST(m_list); - block != NULL; - block = UT_LIST_GET_NEXT(m_node, block)) { + for (typename list_t::iterator it = m_list.begin(), + end = m_list.end(); + it != end; ++it) { - if (!functor(block)) { - return(false); + if (!functor(&*it)) { + return false; } } @@ -360,12 +353,12 @@ public: template bool for_each_block_in_reverse(Functor& functor) const { - for (block_t* block = UT_LIST_GET_LAST(m_list); - block != NULL; - block = UT_LIST_GET_PREV(m_node, block)) { + for (typename list_t::reverse_iterator it = m_list.rbegin(), + end = m_list.rend(); + it != end; ++it) { - if (!functor(block)) { - return(false); + if (!functor(&*it)) { + return false; } } @@ -378,12 +371,12 @@ public: template bool for_each_block_in_reverse(const Functor& functor) const { - for (block_t* block = UT_LIST_GET_LAST(m_list); - block != NULL; - block = UT_LIST_GET_PREV(m_node, block)) { + for (typename list_t::reverse_iterator it = m_list.rbegin(), + end = m_list.rend(); + it != end; ++it) { - if (!functor(block)) { - return(false); + if (!functor(&*it)) { + return false; } } @@ -395,8 +388,7 @@ public: block_t* front() MY_ATTRIBUTE((warn_unused_result)) { - ut_ad(UT_LIST_GET_LEN(m_list) > 0); - return(UT_LIST_GET_FIRST(m_list)); + return &m_list.front(); } /** @@ -417,14 +409,13 @@ private: void push_back(block_t* block) { block->init(); - - UT_LIST_ADD_LAST(m_list, block); + m_list.push_back(*block); } /** @return the last block in the list */ block_t* back() { - return(UT_LIST_GET_LAST(m_list)); + return &m_list.back(); } /* @@ -447,25 +438,22 @@ private: @return the block containing the pos. */ block_t* find(ulint& pos) { - block_t* block; + ut_ad(!m_list.empty()); - ut_ad(UT_LIST_GET_LEN(m_list) > 0); + for (typename list_t::iterator it = m_list.begin(), + end = m_list.end(); + it != end; ++it) { - for (block = UT_LIST_GET_FIRST(m_list); - block != NULL; - block = UT_LIST_GET_NEXT(m_node, block)) { + if (pos < it->used()) { + ut_ad(it->used() >= pos); - if (pos < block->used()) { - break; + return &*it; } - pos -= block->used(); + pos -= it->used(); } - ut_ad(block != NULL); - ut_ad(block->used() >= pos); - - return(block); + return NULL; } /** @@ -491,7 +479,7 @@ private: mem_heap_t* m_heap; /** Allocated blocks */ - block_list_t m_list; + list_t m_list; /** Total size used by all blocks */ ulint m_size; diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 386f574a59f..5484477d913 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -33,9 +33,13 @@ Created 10/25/1995 Heikki Tuuri #include "dict0types.h" #include "page0size.h" #include "ibuf0types.h" +#include "intrusive_list.h" #include +struct unflushed_spaces_tag_t; +struct rotation_list_tag_t; + // Forward declaration extern ibool srv_use_doublewrite_buf; extern struct buf_dblwr_t* buf_dblwr; @@ -78,7 +82,9 @@ fil_type_is_data( struct fil_node_t; /** Tablespace or log data space */ -struct fil_space_t { +struct fil_space_t : intrusive::list_node, + intrusive::list_node +{ ulint id; /*!< space id */ hash_node_t hash; /*!< hash chain node */ char* name; /*!< Tablespace name */ @@ -156,9 +162,6 @@ struct fil_space_t { ulint n_pending_ios; rw_lock_t latch; /*!< latch protecting the file space storage allocation */ - UT_LIST_NODE_T(fil_space_t) unflushed_spaces; - /*!< list of spaces with at least one unflushed - file we have written to */ UT_LIST_NODE_T(fil_space_t) named_spaces; /*!< list of spaces for which MLOG_FILE_NAME records have been issued */ @@ -167,8 +170,6 @@ struct fil_space_t { bool is_in_unflushed_spaces() const; UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - /** other tablespaces needing key rotation */ - UT_LIST_NODE_T(fil_space_t) rotation_list; /** Checks that this tablespace needs key rotation. @return true if in a rotation list */ bool is_in_rotation_list() const; @@ -484,6 +485,11 @@ fil_space_get( data space) is stored here; below we talk about tablespaces, but also the ib_logfiles form a 'space' and it is handled here */ struct fil_system_t { + fil_system_t() + : n_open(0), max_assigned_id(0), space_id_reuse_warned(false) + { + } + ib_mutex_t mutex; /*!< The mutex protecting the cache */ hash_table_t* spaces; /*!< The hash table of spaces in the system; they are hashed on the space @@ -501,8 +507,8 @@ struct fil_system_t { not put to this list: they are opened after the startup, and kept open until shutdown */ - UT_LIST_BASE_NODE_T(fil_space_t) unflushed_spaces; - /*!< base node for the list of those + intrusive::list unflushed_spaces; + /*!< list of those tablespaces whose files contain unflushed writes; those spaces have at least one file node where @@ -524,11 +530,11 @@ struct fil_system_t { record has been written since the latest redo log checkpoint. Protected only by log_sys->mutex. */ - UT_LIST_BASE_NODE_T(fil_space_t) rotation_list; + intrusive::list rotation_list; /*!< list of all file spaces needing key rotation.*/ - ibool space_id_reuse_warned; + bool space_id_reuse_warned; /* !< TRUE if fil_space_create() has issued a warning about potential space_id reuse */ -- cgit v1.2.1 From 5824e9f8df35e24bada99ea1409301f9e166421f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 2 Jan 2020 08:06:23 +0200 Subject: MDEV-13569: wsrep_info.plugin failed in buildbot with "no nodes coming from prim view Modify configuration so that all nodes are part of galera cluster i.e. wsrep_on=ON. Add missing wait conditions. test changes only. --- mysql-test/suite/wsrep/r/variables.result | 9 +++++++++ mysql-test/suite/wsrep/t/variables.test | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result index 99e30788488..3abc861f3d0 100644 --- a/mysql-test/suite/wsrep/r/variables.result +++ b/mysql-test/suite/wsrep/r/variables.result @@ -111,6 +111,9 @@ SELECT @@global.wsrep_slave_threads; SELECT @@global.wsrep_cluster_address; @@global.wsrep_cluster_address +SELECT @@global.wsrep_on; +@@global.wsrep_on +1 SHOW STATUS LIKE 'threads_connected'; Variable_name Value Threads_connected 1 @@ -124,6 +127,9 @@ libgalera_smm.so SELECT @@global.wsrep_cluster_address; @@global.wsrep_cluster_address +SELECT @@global.wsrep_on; +@@global.wsrep_on +1 SHOW STATUS LIKE 'threads_connected'; Variable_name Value Threads_connected 1 @@ -150,6 +156,9 @@ libgalera_smm.so SELECT @@global.wsrep_cluster_address; @@global.wsrep_cluster_address gcomm:// +SELECT @@global.wsrep_on; +@@global.wsrep_on +1 SHOW STATUS LIKE 'threads_connected'; Variable_name Value Threads_connected 1 diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index e01ba3e3a8b..5336fae17b6 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -84,6 +84,7 @@ eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER'; SELECT @@global.wsrep_provider; SELECT @@global.wsrep_slave_threads; SELECT @@global.wsrep_cluster_address; +SELECT @@global.wsrep_on; SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; --echo @@ -95,6 +96,7 @@ eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER'; --replace_regex /.*libgalera_smm.*/libgalera_smm.so/ SELECT @@global.wsrep_provider; SELECT @@global.wsrep_cluster_address; +SELECT @@global.wsrep_on; SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; --echo @@ -107,6 +109,8 @@ SET GLOBAL wsrep_cluster_address= 'gcomm://'; --let $wait_timeout=600 --let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count'; --source include/wait_condition.inc +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count'; +--source include/wait_condition.inc SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count'; SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count'; @@ -115,6 +119,7 @@ SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME --replace_regex /.*libgalera_smm.*/libgalera_smm.so/ SELECT @@global.wsrep_provider; SELECT @@global.wsrep_cluster_address; +SELECT @@global.wsrep_on; SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; --echo @@ -137,6 +142,7 @@ SHOW STATUS LIKE 'threads_connected'; # set wsrep_on=0; set wsrep_on=1; +--source include/wait_until_connected_again.inc create user test@localhost; connect con1,localhost,test; set auto_increment_increment=10; -- cgit v1.2.1 From 82187a1221467c7d193fca60a11a020ab4228e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 7 Jan 2020 10:43:22 +0200 Subject: MDEV-21429 TRUNCATE and OPTIMIZE are being refused due to "row size too large" By default (innodb_strict_mode=ON), InnoDB attempts to guarantee at DDL time that any INSERT to the table can succeed. MDEV-19292 recently revised the "row size too large" check in InnoDB. The check still is somewhat inaccurate; that should be addressed in MDEV-20194. Note: If a table contains multiple long string columns so that each column is part of a column prefix index, then an UPDATE that attempts to modify all those columns at once may fail, because the undo log record might not fit in a single undo log page (of innodb_page_size). In the worst case, the undo log record would grow by about 3KiB of for each updated column. The DDL-time check (since the InnoDB Plugin for MySQL 5.1) is optional in the sense that when the maximum B-tree record size or undo log record size would be exceeded, the DML operation will fail and the transaction will be properly rolled back. create_table_info_t::row_size_is_acceptable(): Add the parameter 'bool strict' so that innodb_strict_mode=ON can be overridden during TRUNCATE, OPTIMIZE and ALTER TABLE...FORCE (when the storage format is not changing). create_table_info_t::create_table(): Perform a sloppy check for TRUNCATE TABLE (create_fk=false). prepare_inplace_alter_table_dict(): Perform a sloppy check for simple operations. trx_is_strict(): Remove. The function became unused in commit 98694ab0cbaf623c6ad67dd45d6f90c5c6214fd1 (MDEV-20949). --- .../innodb/r/row_size_error_log_warnings_3.result | 26 +++++++++++++- .../innodb/t/row_size_error_log_warnings_3.test | 10 +++++- storage/innobase/handler/ha_innodb.cc | 40 +++++++--------------- storage/innobase/handler/ha_innodb.h | 8 +++-- storage/innobase/handler/handler0alter.cc | 23 +++++++++---- storage/innobase/include/trx0trx.h | 9 +---- 6 files changed, 69 insertions(+), 47 deletions(-) diff --git a/mysql-test/suite/innodb/r/row_size_error_log_warnings_3.result b/mysql-test/suite/innodb/r/row_size_error_log_warnings_3.result index c175f8ee915..fba209dd2b5 100644 --- a/mysql-test/suite/innodb/r/row_size_error_log_warnings_3.result +++ b/mysql-test/suite/innodb/r/row_size_error_log_warnings_3.result @@ -16,6 +16,30 @@ col_1 TEXT ) ENGINE=INNODB ROW_FORMAT=COMPACT; Warnings: Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. +TRUNCATE TABLE t1; +Warnings: +Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize note Table does not support optimize, doing recreate + analyze instead +test.t1 optimize status OK +Warnings: +Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. +ALTER TABLE t1 FORCE; +Warnings: +Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. +SET innodb_strict_mode = ON; +TRUNCATE TABLE t1; +Warnings: +Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize note Table does not support optimize, doing recreate + analyze instead +test.t1 optimize status OK +Warnings: +Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. +ALTER TABLE t1 FORCE; +Warnings: +Warning 139 Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. DROP TABLE t1; SET @@global.log_warnings = 2; -SET innodb_strict_mode = 1; diff --git a/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test b/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test index 35b86cc4c46..af6c15e0e6b 100644 --- a/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test +++ b/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test @@ -18,7 +18,15 @@ CREATE TABLE t1 ( ,col_10 TEXT ,col_11 TEXT ) ENGINE=INNODB ROW_FORMAT=COMPACT; +--enable_warnings +TRUNCATE TABLE t1; +OPTIMIZE TABLE t1; +ALTER TABLE t1 FORCE; +SET innodb_strict_mode = ON; +TRUNCATE TABLE t1; +OPTIMIZE TABLE t1; +ALTER TABLE t1 FORCE; DROP TABLE t1; +--disable_warnings SET @@global.log_warnings = 2; -SET innodb_strict_mode = 1; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 1e1583804eb..0338e2b682a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2019, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -3504,17 +3504,6 @@ trx_is_interrupted( return(trx && trx->mysql_thd && thd_kill_level(trx->mysql_thd)); } -/**********************************************************************//** -Determines if the currently running transaction is in strict mode. -@return TRUE if strict */ -ibool -trx_is_strict( -/*==========*/ - trx_t* trx) /*!< in: transaction */ -{ - return(trx && trx->mysql_thd && THDVAR(trx->mysql_thd, strict_mode)); -} - /**************************************************************//** Resets some fields of a m_prebuilt struct. The template is used in fast retrieval of just those column values MySQL needs in its processing. */ @@ -12742,7 +12731,10 @@ int create_table_info_t::create_table(bool create_fk) DICT_ERR_IGNORE_NONE); ut_ad(innobase_table); - const bool is_acceptable = row_size_is_acceptable(*innobase_table); + /* In TRUNCATE TABLE, we will merely warn about the maximum + row size being too large. */ + const bool is_acceptable = row_size_is_acceptable(*innobase_table, + create_fk); dict_table_close(innobase_table, true, false); @@ -12755,18 +12747,12 @@ int create_table_info_t::create_table(bool create_fk) } bool create_table_info_t::row_size_is_acceptable( - const dict_table_t &table) const + const dict_table_t &table, bool strict) const { for (dict_index_t *index= dict_table_get_first_index(&table); index; index= dict_table_get_next_index(index)) - { - - if (!row_size_is_acceptable(*index)) - { + if (!row_size_is_acceptable(*index, strict)) return false; - } - } - return true; } @@ -12944,7 +12930,7 @@ static void ib_warn_row_too_big(THD *thd, const dict_table_t *table) } bool create_table_info_t::row_size_is_acceptable( - const dict_index_t &index) const + const dict_index_t &index, bool strict) const { if ((index.type & DICT_FTS) || index.table->is_system_db) { @@ -12953,7 +12939,7 @@ bool create_table_info_t::row_size_is_acceptable( return true; } - const bool strict= THDVAR(m_thd, strict_mode); + const bool innodb_strict_mode= THDVAR(m_thd, strict_mode); dict_index_t::record_size_info_t info= index.record_size_info(); if (info.row_is_too_big()) @@ -12964,9 +12950,9 @@ bool create_table_info_t::row_size_is_acceptable( const size_t idx= info.get_first_overrun_field_index(); const dict_field_t *field= dict_index_get_nth_field(&index, idx); - if (strict || global_system_variables.log_warnings > 2) + if (innodb_strict_mode || global_system_variables.log_warnings > 2) { - ib::error_or_warn(strict) + ib::error_or_warn(strict && innodb_strict_mode) << "Cannot add field " << field->name << " in table " << index.table->name << " because after adding it, the row size is " << info.get_overrun_size() @@ -12974,10 +12960,8 @@ bool create_table_info_t::row_size_is_acceptable( << info.max_leaf_size << " bytes) for a record on index leaf page."; } - if (strict) - { + if (strict && innodb_strict_mode) return false; - } ib_warn_row_too_big(m_thd, index.table); } diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 312da6451f0..1de26e03607 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -680,9 +680,11 @@ public: void allocate_trx(); /** Checks that every index have sane size. Depends on strict mode */ - bool row_size_is_acceptable(const dict_table_t& table) const; + bool row_size_is_acceptable(const dict_table_t& table, + bool strict) const; /** Checks that given index have sane size. Depends on strict mode */ - bool row_size_is_acceptable(const dict_index_t& index) const; + bool row_size_is_acceptable(const dict_index_t& index, + bool strict) const; /** Determines InnoDB table flags. If strict_mode=OFF, this will adjust the flags to what should be assumed. diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 4cc1250b719..64b1a806e44 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2019, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -101,19 +101,23 @@ static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_FOREIGN_OPERATIONS = Alter_inplace_info::DROP_FOREIGN_KEY | Alter_inplace_info::ADD_FOREIGN_KEY; -/** Operations that InnoDB cares about and can perform without rebuild */ -static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ALTER_NOREBUILD +/** Operations that InnoDB cares about and can perform without validation */ +static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ALTER_NOVALIDATE = INNOBASE_ONLINE_CREATE | INNOBASE_FOREIGN_OPERATIONS | Alter_inplace_info::DROP_INDEX | Alter_inplace_info::DROP_UNIQUE_INDEX | Alter_inplace_info::ALTER_COLUMN_NAME - | Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH //| Alter_inplace_info::ALTER_INDEX_COMMENT - | Alter_inplace_info::ADD_VIRTUAL_COLUMN | Alter_inplace_info::DROP_VIRTUAL_COLUMN | Alter_inplace_info::ALTER_VIRTUAL_COLUMN_ORDER; +/** Operations that InnoDB cares about and can perform without rebuild */ +static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ALTER_NOREBUILD + = INNOBASE_ALTER_NOVALIDATE + | Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH + | Alter_inplace_info::ADD_VIRTUAL_COLUMN; + struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx { /** Dummy query graph */ @@ -4844,7 +4848,14 @@ index_created: goto error_handling; } - if (!info.row_size_is_acceptable(*ctx->add_index[a])) { + /* For ALTER TABLE...FORCE or OPTIMIZE TABLE, we may + only issue warnings, because there will be no schema change. */ + if (!info.row_size_is_acceptable( + *ctx->add_index[a], + !!(ha_alter_info->handler_flags + & ~(INNOBASE_INPLACE_IGNORE + | INNOBASE_ALTER_NOVALIDATE + | Alter_inplace_info::RECREATE_TABLE)))) { error = DB_TOO_BIG_RECORD; goto error_handling; } diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 1d71920d6c5..7e46304b6ee 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2019, MariaDB Corporation. +Copyright (c) 2015, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -432,13 +432,6 @@ ibool trx_is_interrupted( /*===============*/ const trx_t* trx); /*!< in: transaction */ -/**********************************************************************//** -Determines if the currently running transaction is in strict mode. -@return TRUE if strict */ -ibool -trx_is_strict( -/*==========*/ - trx_t* trx); /*!< in: transaction */ /*******************************************************************//** Calculates the "weight" of a transaction. The weight of one transaction -- cgit v1.2.1