diff options
658 files changed, 16222 insertions, 11770 deletions
diff --git a/.gitignore b/.gitignore index 433d14db7e9..f1c300a89c6 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ extra/jemalloc/build/ extra/jemalloc/tmp/ extra/my_print_defaults extra/mysql_waitpid +extra/mysqld_safe_helper extra/perror extra/replace extra/resolve_stack_dump @@ -213,6 +214,7 @@ support-files/mysql.spec support-files/mysqld_multi.server support-files/wsrep.cnf support-files/wsrep_notify +support-files/SELinux/mariadb.pp tags tests/async_queries tests/bug25714 diff --git a/CMakeLists.txt b/CMakeLists.txt index 0adbb85d431..df4ac556726 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ -# Copyright (c) 2006, 2014, Oracle and/or its affiliates. -# Copyright (c) 2008, 2014, Monty Program Ab +# Copyright (c) 2006, 2017, Oracle and/or its affiliates. +# Copyright (c) 2008, 2017, MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -241,12 +241,9 @@ IF (WITH_ASAN) ENDIF() ENDIF() - -OPTION(ENABLE_DEBUG_SYNC "Enable debug sync (debug builds only)" ON) -IF(ENABLE_DEBUG_SYNC) - SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC") - SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC") -ENDIF() +# Always enable debug sync for debug builds. +SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC") +SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC") OPTION(ENABLE_GCOV "Enable gcov (debug, Linux builds only)" OFF) IF (ENABLE_GCOV AND NOT WIN32 AND NOT APPLE) @@ -303,7 +300,7 @@ ENDIF() # Set commonly used variables IF(WIN32) - SET(DEFAULT_MYSQL_HOME "C:/MariaDB${MYSQL_BASE_VERSION}") + SET(DEFAULT_MYSQL_HOME "C:/Program Files/MariaDB ${MYSQL_BASE_VERSION}") SET(SHAREDIR share) ELSE() SET(DEFAULT_MYSQL_HOME ${CMAKE_INSTALL_PREFIX}) @@ -348,6 +345,9 @@ SET(CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON) # Common defines and includes ADD_DEFINITIONS(-DHAVE_CONFIG_H) +IF(_FILE_OFFSET_BITS) + ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=${_FILE_OFFSET_BITS}) +ENDIF() INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/include) # Add bundled or system zlib. @@ -480,8 +480,7 @@ ADD_CUSTOM_TARGET(INFO_BIN ALL WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) -INSTALL_DOCUMENTATION(README CREDITS COPYING COPYING.LESSER EXCEPTIONS-CLIENT - COMPONENT Readme) +INSTALL_DOCUMENTATION(README CREDITS COPYING EXCEPTIONS-CLIENT COMPONENT Readme) # MDEV-6526 these files are not installed anymore #INSTALL_DOCUMENTATION(${CMAKE_BINARY_DIR}/Docs/INFO_SRC diff --git a/COPYING.LESSER b/COPYING.LESSER deleted file mode 100644 index c2c87d69256..00000000000 --- a/COPYING.LESSER +++ /dev/null @@ -1,516 +0,0 @@ - - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some -specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You -can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations -below. - - When we speak of free software, we are referring to freedom of use, -not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge -for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of -it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for -you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them -with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that -there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know -that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be -introduced by others. -^L - Finally, software patents pose a constant threat to the existence of -any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a -restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be -consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the -ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and -is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those -libraries into non-free programs. - - When a program is linked with a library, whether statically or using -a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary -General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General -Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it -does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less -of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it -becomes -a de-facto standard. To achieve this, non-free programs must be -allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this -case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free -programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in -non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating -system. - - Although the Lesser General Public License is Less protective of the -users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run -that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, whereas the latter must -be combined with the library in order to run. -^L - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other -program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control -compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. -^L - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. -^L - 6. As an exception to the Sections above, you may also combine or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, - rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if - the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. -^L - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties with -this License. -^L - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply, and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License -may add an explicit geographical distribution limitation excluding those -countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. -^L - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS -^L - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms -of the ordinary General Public License). - - To apply these terms, attach the following notices to the library. -It is safest to attach them to the start of each source file to most -effectively convey the exclusion of warranty; and each file should -have at least the "copyright" line and a pointer to where the full -notice is found. - - - <one line to give the library's name and a brief idea of what it -does.> - Copyright (C) <year> <name of author> - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA - -Also add information on how to contact you by electronic and paper -mail. - -You should also get your employer (if you work as a programmer) or -your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James -Random Hacker. - - <signature of Ty Coon>, 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! - - @@ -3,16 +3,18 @@ organization registered in the USA. The current main sponsors of the MariaDB Foundation are: -Booking.com http://www.booking.com (2013 - 2016) -Development Bank of Singapore http://dbs.com (2016) -MariaDB Corporation https://www.mariadb.com (2013 - 2016) -Visma http://visma.com (2015 - 2016) -Acronis http://acronis.com (2016) -Nexedi https://www.nexedi.com (2016) -Automattic https://automattic.com (2014 - 2016) -Tencent Game DBA http://tencentdba.com/about (2016) -Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2016) -Virtuozzo https://virtuozzo.com (2016) +Alibaba Cloud https://intl.aliyun.com (2017) +Booking.com https://www.booking.com (2013 - 2017) +Development Bank of Singapore https://dbs.com (2016 - 2017) +MariaDB Corporation https://www.mariadb.com (2013 - 2017) +Visma https://visma.com (2015 - 2017) +Acronis http://acronis.com (2016 - 2017) +Nexedi https://www.nexedi.com (2016 - 2017) +Automattic https://automattic.com (2014 - 2017) +Tencent Game DBA http://tencentdba.com/about (2016 - 2017) +Tencent TDSQL http://tdsql.org/ (2016 - 2017) +Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2017) +Virtuozzo https://virtuozzo.com (2016 - 2017) For a full list of sponsors, see https://mariadb.org/about/supporters/ @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=29 +MYSQL_VERSION_PATCH=31 diff --git a/client/client_priv.h b/client/client_priv.h index 3905f323156..52f73b51ed5 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -117,4 +117,3 @@ enum options_client Name of the performance schema database. */ #define PERFORMANCE_SCHEMA_DB_NAME "performance_schema" - diff --git a/client/mysql.cc b/client/mysql.cc index 4b20e4d98cb..a9310a87d8e 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2014, Oracle and/or its affiliates. - Copyright (c) 2009, 2016, MariaDB + Copyright (c) 2009, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c index 737f9c9c8d7..f2d94871f07 100644 --- a/client/mysql_upgrade.c +++ b/client/mysql_upgrade.c @@ -1,6 +1,6 @@ /* Copyright (c) 2006, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1140,4 +1140,3 @@ end: my_end(my_end_arg); exit(0); } - diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index 56d2784a1f5..bab94d6e5ee 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2014, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -38,8 +38,8 @@ char ex_var_names[MAX_MYSQL_VAR][FN_REFLEN]; ulonglong last_values[MAX_MYSQL_VAR]; static int interval=0; static my_bool option_force=0,interrupted=0,new_line=0, - opt_compress=0, opt_relative=0, opt_verbose=0, opt_vertical=0, - tty_password= 0, opt_nobeep; + opt_compress= 0, opt_local= 0, opt_relative= 0, opt_verbose= 0, + opt_vertical= 0, tty_password= 0, opt_nobeep; static my_bool debug_info_flag= 0, debug_check_flag= 0; static uint tcp_port = 0, option_wait = 0, option_silent=0, nr_iterations; static uint opt_count_iterations= 0, my_end_arg; @@ -102,9 +102,12 @@ enum commands { ADMIN_PING, ADMIN_EXTENDED_STATUS, ADMIN_FLUSH_STATUS, ADMIN_FLUSH_PRIVILEGES, ADMIN_START_SLAVE, ADMIN_STOP_SLAVE, ADMIN_START_ALL_SLAVES, ADMIN_STOP_ALL_SLAVES, - ADMIN_FLUSH_THREADS, ADMIN_OLD_PASSWORD, ADMIN_FLUSH_SLOW_LOG, + ADMIN_FLUSH_THREADS, ADMIN_OLD_PASSWORD, ADMIN_FLUSH_BINARY_LOG, + ADMIN_FLUSH_ENGINE_LOG, ADMIN_FLUSH_ERROR_LOG, ADMIN_FLUSH_GENERAL_LOG, + ADMIN_FLUSH_RELAY_LOG, ADMIN_FLUSH_SLOW_LOG, ADMIN_FLUSH_TABLE_STATISTICS, ADMIN_FLUSH_INDEX_STATISTICS, ADMIN_FLUSH_USER_STATISTICS, ADMIN_FLUSH_CLIENT_STATISTICS, + ADMIN_FLUSH_USER_RESOURCES, ADMIN_FLUSH_ALL_STATUS, ADMIN_FLUSH_ALL_STATISTICS }; static const char *command_names[]= { @@ -116,9 +119,10 @@ static const char *command_names[]= { "ping", "extended-status", "flush-status", "flush-privileges", "start-slave", "stop-slave", "start-all-slaves", "stop-all-slaves", - "flush-threads", "old-password", "flush-slow-log", + "flush-threads", "old-password", "flush-binary-log", "flush-engine-log", + "flush-error-log", "flush-general-log", "flush-relay-log", "flush-slow-log", "flush-table-statistics", "flush-index-statistics", - "flush-user-statistics", "flush-client-statistics", + "flush-user-statistics", "flush-client-statistics", "flush-user-resources", "flush-all-status", "flush-all-statistics", NullS }; @@ -160,6 +164,9 @@ static struct my_option my_long_options[] = NO_ARG, 0, 0, 0, 0, 0, 0}, {"host", 'h', "Connect to host.", &host, &host, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"local", 'l', "Local command, don't write to binlog.", + &opt_local, &opt_local, 0, GET_BOOL, NO_ARG, 0, 0, 0, + 0, 0, 0}, {"no-beep", 'b', "Turn off beep on error.", &opt_nobeep, &opt_nobeep, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"password", 'p', @@ -617,6 +624,18 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) */ struct my_rnd_struct rand_st; + char buff[FN_REFLEN + 20]; + + if (opt_local) + { + sprintf(buff, "set local sql_log_bin=0"); + if (mysql_query(mysql, buff)) + { + my_printf_error(0, "SET LOCAL SQL_LOG_BIN=0 failed; error: '%-.200s'", + error_flags, mysql_error(mysql)); + return -1; + } + } for (; argc > 0 ; argv++,argc--) { @@ -624,7 +643,6 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) switch ((command= find_type(argv[0],&command_typelib,FIND_TYPE_BASIC))) { case ADMIN_CREATE: { - char buff[FN_REFLEN+20]; if (argc < 2) { my_printf_error(0, "Too few arguments to create", error_flags); @@ -902,6 +920,56 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) } break; } + case ADMIN_FLUSH_BINARY_LOG: + { + if (mysql_query(mysql, "flush binary logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_ENGINE_LOG: + { + if (mysql_query(mysql,"flush engine logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_ERROR_LOG: + { + if (mysql_query(mysql, "flush error logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_GENERAL_LOG: + { + if (mysql_query(mysql, "flush general logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_RELAY_LOG: + { + if (mysql_query(mysql, "flush relay logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } case ADMIN_FLUSH_SLOW_LOG: { if (mysql_query(mysql,"flush slow logs")) @@ -972,6 +1040,16 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) } break; } + case ADMIN_FLUSH_USER_RESOURCES: + { + if (mysql_query(mysql, "flush user_resources")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } case ADMIN_FLUSH_CLIENT_STATISTICS: { if (mysql_query(mysql,"flush client_statistics")) @@ -1295,12 +1373,18 @@ static void usage(void) flush-index-statistics Flush index statistics\n\ flush-logs Flush all logs\n\ flush-privileges Reload grant tables (same as reload)\n\ + flush-binary-log Flush binary log\n\ + flush-engine-log Flush engine log(s)\n\ + flush-error-log Flush error log\n\ + flush-general-log Flush general log\n\ + flush-relay-log Flush relay log\n\ flush-slow-log Flush slow query log\n\ - flush-status Clear status variables\n\ + flush-status Clear status variables\n\ flush-table-statistics Clear table statistics\n\ flush-tables Flush all tables\n\ flush-threads Flush the thread cache\n\ flush-user-statistics Flush user statistics\n\ + flush-user-resources Flush user resources\n\ kill id,id,... Kill mysql threads"); #if MYSQL_VERSION_ID >= 32200 puts("\ @@ -1567,8 +1651,10 @@ static my_bool get_pidfile(MYSQL *mysql, char *pidfile) if (mysql_query(mysql, "SHOW VARIABLES LIKE 'pid_file'")) { - my_printf_error(0, "query failed; error: '%s'", error_flags, - mysql_error(mysql)); + my_printf_error(mysql_errno(mysql), + "The query to get the server's pid file failed," + " error: '%s'. Continuing.", error_flags, + mysql_error(mysql)); } result = mysql_store_result(mysql); if (result) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 4be31421f9b..29355d013c7 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -106,7 +106,7 @@ static const char* sock= 0; static char *opt_plugindir= 0, *opt_default_auth= 0; #ifdef HAVE_SMEM -static char *shared_memory_base_name= 0; +static const char *shared_memory_base_name= 0; #endif static char* user = 0; static char* pass = 0; diff --git a/client/mysqlcheck.c b/client/mysqlcheck.c index 0c34ef35966..e436492d58e 100644 --- a/client/mysqlcheck.c +++ b/client/mysqlcheck.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/client/mysqldump.c b/client/mysqldump.c index 64ed21ac7fc..8ed10f1e5cb 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -698,8 +698,9 @@ static void write_header(FILE *sql_file, char *db_name) "-- MySQL dump %s Distrib %s, for %s (%s)\n--\n", DUMP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); - print_comment(sql_file, 0, "-- Host: %s Database: %s\n", - fix_for_comment(current_host ? current_host : "localhost"), + print_comment(sql_file, 0, "-- Host: %s ", + fix_for_comment(current_host ? current_host : "localhost")); + print_comment(sql_file, 0, "Database: %s\n", fix_for_comment(db_name ? db_name : "")); print_comment(sql_file, 0, "-- ------------------------------------------------------\n" @@ -2178,7 +2179,6 @@ static void print_comment(FILE *sql_file, my_bool is_error, const char *format, print_xml_comment(sql_file, strlen(comment_buff), comment_buff); } - /* create_delimiter Generate a new (null-terminated) string that does not exist in query @@ -2523,7 +2523,7 @@ static uint dump_routines_for_db(char *db) query_buff); print_comment(sql_file, 1, "-- does %s have permissions on mysql.proc?\n\n", - current_user); + fix_for_comment(current_user)); maybe_die(EX_MYSQLERR,"%s has insufficent privileges to %s!", current_user, query_buff); } diff --git a/client/mysqlimport.c b/client/mysqlimport.c index aee445d387d..216359c82ec 100644 --- a/client/mysqlimport.c +++ b/client/mysqlimport.c @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2011, 2016, MariaDB + Copyright (c) 2011, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/client/mysqlshow.c b/client/mysqlshow.c index eec4a8d3268..6f434d6770d 100644 --- a/client/mysqlshow.c +++ b/client/mysqlshow.c @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/client/mysqlslap.c b/client/mysqlslap.c index b3229980e77..a78bf35d51b 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1,6 +1,6 @@ /* Copyright (c) 2005, 2015, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/client/mysqltest.cc b/client/mysqltest.cc index ed401b6e4a2..d903484bd4d 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2009, 2016, Monty Program Ab. + Copyright (c) 2009, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1062,7 +1062,7 @@ void do_eval(DYNAMIC_STRING *query_eval, const char *query, if (!(v= var_get(p, &p, 0, 0))) { report_or_die( "Bad variable in eval"); - return; + DBUG_VOID_RETURN; } dynstr_append_mem(query_eval, v->str_val, v->str_val_len); } @@ -1777,7 +1777,7 @@ static int run_command(char* cmd, if (!(res_file= popen(cmd, "r"))) { report_or_die("popen(\"%s\", \"r\") failed", cmd); - return -1; + DBUG_RETURN(-1); } while (fgets(buf, sizeof(buf), res_file)) @@ -2685,7 +2685,7 @@ void var_query_set(VAR *var, const char *query, const char** query_end) report_or_die("Query '%s' didn't return a result set", ds_query.str); dynstr_free(&ds_query); eval_expr(var, "", 0); - return; + DBUG_VOID_RETURN; } dynstr_free(&ds_query); @@ -2876,7 +2876,7 @@ void var_set_query_get_value(struct st_command *command, VAR *var) dynstr_free(&ds_query); dynstr_free(&ds_col); eval_expr(var, "", 0); - return; + DBUG_VOID_RETURN; } { @@ -2901,7 +2901,7 @@ void var_set_query_get_value(struct st_command *command, VAR *var) ds_col.str, ds_query.str); dynstr_free(&ds_query); dynstr_free(&ds_col); - return; + DBUG_VOID_RETURN; } DBUG_PRINT("info", ("Found column %d with name '%s'", i, fields[i].name)); @@ -3341,13 +3341,15 @@ void do_exec(struct st_command *command) DBUG_ENTER("do_exec"); DBUG_PRINT("enter", ("cmd: '%s'", cmd)); + var_set_int("$sys_errno",0); + /* Skip leading space */ while (*cmd && my_isspace(charset_info, *cmd)) cmd++; if (!*cmd) { report_or_die("Missing argument in exec"); - return; + DBUG_VOID_RETURN; } command->last_argument= command->end; @@ -3381,7 +3383,7 @@ void do_exec(struct st_command *command) dynstr_free(&ds_cmd); if (command->abort_on_error) report_or_die("popen(\"%s\", \"r\") failed", command->first_argument); - return; + DBUG_VOID_RETURN; } ds_result= &ds_res; @@ -3439,7 +3441,7 @@ void do_exec(struct st_command *command) ds_cmd.str, error, status, errno, ds_res.str); dynstr_free(&ds_cmd); - return; + DBUG_VOID_RETURN; } DBUG_PRINT("info", @@ -3457,6 +3459,7 @@ void do_exec(struct st_command *command) report_or_die("command \"%s\" failed with wrong error: %d", command->first_argument, status); } + var_set_int("$sys_errno",status); } else if (command->expected_errors.err[0].type == ERR_ERRNO && command->expected_errors.err[0].code.errnum != 0) @@ -3571,7 +3574,7 @@ void do_system(struct st_command *command) if (strlen(command->first_argument) == 0) { report_or_die("Missing arguments to system, nothing to do!"); - return; + DBUG_VOID_RETURN; } init_dynamic_string(&ds_cmd, 0, command->query_len + 64, 256); @@ -4558,7 +4561,7 @@ void do_perl(struct st_command *command) if (command->abort_on_error) die("popen(\"%s\", \"r\") failed", buf); dynstr_free(&ds_delimiter); - return; + DBUG_VOID_RETURN; } while (fgets(buf, sizeof(buf), res_file)) @@ -5945,6 +5948,7 @@ void do_connect(struct st_command *command) my_bool con_shm __attribute__ ((unused))= 0; int read_timeout= 0; int write_timeout= 0; + int connect_timeout= 0; struct st_connection* con_slot; static DYNAMIC_STRING ds_connection_name; @@ -6051,6 +6055,11 @@ void do_connect(struct st_command *command) { write_timeout= atoi(con_options + sizeof("write_timeout=")-1); } + else if (strncasecmp(con_options, "connect_timeout=", + sizeof("connect_timeout=")-1) == 0) + { + connect_timeout= atoi(con_options + sizeof("connect_timeout=")-1); + } else die("Illegal option to connect: %.*s", (int) (end - con_options), con_options); @@ -6135,6 +6144,12 @@ void do_connect(struct st_command *command) (char*)&write_timeout); } + if (connect_timeout) + { + mysql_options(con_slot->mysql, MYSQL_OPT_CONNECT_TIMEOUT, + (char*)&connect_timeout); + } + #ifdef HAVE_SMEM if (con_shm) { diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index 02492f2dbc9..35d6f0c52ef 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -99,6 +99,8 @@ OPTION(ENABLED_LOCAL_INFILE "" ON) IF(RPM) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") + SET(CHECKMODULE /usr/bin/checkmodule CACHE STRING "") + SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE STRING "") ELSEIF(DEB) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake index 5125b9482cd..2e65e8b5ba3 100644 --- a/cmake/libutils.cmake +++ b/cmake/libutils.cmake @@ -188,7 +188,7 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE) # binaries properly) ADD_CUSTOM_COMMAND(TARGET ${TARGET} POST_BUILD COMMAND rm ${TARGET_LOCATION} - COMMAND /usr/bin/libtool -static -o ${TARGET_LOCATION} + COMMAND libtool -static -o ${TARGET_LOCATION} ${STATIC_LIBS} ) ELSE() diff --git a/cmake/mysql_add_executable.cmake b/cmake/mysql_add_executable.cmake index 0c93fb179f5..45575bdd536 100644 --- a/cmake/mysql_add_executable.cmake +++ b/cmake/mysql_add_executable.cmake @@ -38,6 +38,14 @@ FUNCTION (MYSQL_ADD_EXECUTABLE) SET(sources ${ARG_DEFAULT_ARGS}) ADD_VERSION_INFO(${target} EXECUTABLE sources) + + IF(MSVC) + # Add compatibility manifest, to fix GetVersionEx on Windows 8.1 and later + IF (CMAKE_VERSION VERSION_GREATER 3.3) + SET(sources ${sources} ${PROJECT_SOURCE_DIR}/cmake/win_compatibility.manifest) + ENDIF() + ENDIF() + ADD_EXECUTABLE(${target} ${ARG_WIN32} ${ARG_MACOSX_BUNDLE} ${ARG_EXCLUDE_FROM_ALL} ${sources}) # tell CPack where to install IF(NOT ARG_EXCLUDE_FROM_ALL) diff --git a/cmake/pcre.cmake b/cmake/pcre.cmake index 45d9bc01ddb..894bde38974 100644 --- a/cmake/pcre.cmake +++ b/cmake/pcre.cmake @@ -5,7 +5,7 @@ MACRO (CHECK_PCRE) IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto") CHECK_LIBRARY_EXISTS(pcre pcre_stack_guard "" HAVE_PCRE) ENDIF() - IF(NOT HAVE_PCRE) + IF(NOT HAVE_PCRE OR WITH_PCRE STREQUAL "bundled") IF (WITH_PCRE STREQUAL "system") MESSAGE(FATAL_ERROR "system pcre is not found or unusable") ENDIF() diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index e1d2af2add6..ba7bac837b1 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -179,8 +179,15 @@ MACRO(MYSQL_ADD_PLUGIN) # executable to the linker command line (it would result into link error). # Thus we skip TARGET_LINK_LIBRARIES on Linux, as it would only generate # an additional dependency. - IF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") - TARGET_LINK_LIBRARIES (${target} mysqld ${ARG_LINK_LIBRARIES}) + IF(MSVC) + ADD_DEPENDENCIES(${target} gen_mysqld_lib) + TARGET_LINK_LIBRARIES(${target} mysqld_import_lib) + ELSEIF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + TARGET_LINK_LIBRARIES (${target} mysqld) + ENDIF() + + IF(ARG_LINK_LIBRARIES) + TARGET_LINK_LIBRARIES (${target} ${ARG_LINK_LIBRARIES}) ENDIF() ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDENCIES}) diff --git a/cmake/win_compatibility.manifest b/cmake/win_compatibility.manifest new file mode 100644 index 00000000000..2e4b27a6dc4 --- /dev/null +++ b/cmake/win_compatibility.manifest @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<asmv1:assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1" xmlns:asmv1="urn:schemas-microsoft-com:asm.v1" xmlns:asmv2="urn:schemas-microsoft-com:asm.v2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1"> + <application> + <!-- Windows Vista and Windows Server 2008 --> + <supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/> + + <!-- Windows 7 and Windows Server 2008 R2 --> + <supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/> + + <!-- Windows 8 and Windows Server 2012 --> + <supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/> + + <!-- Windows 8.1 and Windows Server 2012 R2 --> + <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/> + + <!-- Windows 10 --> + <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/> + + </application> + </compatibility> +</asmv1:assembly> diff --git a/config.h.cmake b/config.h.cmake index 1fd1341f1d1..8618d9e6d02 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -484,7 +484,6 @@ #cmakedefine _LARGE_FILES 1 #cmakedefine _LARGEFILE_SOURCE 1 #cmakedefine _LARGEFILE64_SOURCE 1 -#cmakedefine _FILE_OFFSET_BITS @_FILE_OFFSET_BITS@ #cmakedefine TIME_WITH_SYS_TIME 1 diff --git a/debian/dist/Debian/mariadb-server-10.0.files.in b/debian/dist/Debian/mariadb-server-10.0.files.in index 88516bb6089..9feec684ff0 100644 --- a/debian/dist/Debian/mariadb-server-10.0.files.in +++ b/debian/dist/Debian/mariadb-server-10.0.files.in @@ -38,6 +38,7 @@ usr/bin/mysql_zap usr/bin/mysqlbinlog usr/bin/mysqld_multi usr/bin/mysqld_safe +usr/bin/mysqld_safe_helper usr/bin/mysqlhotcopy usr/bin/perror usr/bin/replace diff --git a/debian/dist/Ubuntu/mariadb-server-10.0.files.in b/debian/dist/Ubuntu/mariadb-server-10.0.files.in index 849a763dccd..a5e8bd6e717 100644 --- a/debian/dist/Ubuntu/mariadb-server-10.0.files.in +++ b/debian/dist/Ubuntu/mariadb-server-10.0.files.in @@ -40,6 +40,7 @@ usr/bin/mysql_zap usr/bin/mysqlbinlog usr/bin/mysqld_multi usr/bin/mysqld_safe +usr/bin/mysqld_safe_helper usr/bin/mysqlhotcopy usr/bin/perror usr/bin/replace diff --git a/debian/mariadb-server-10.0.mysql-server.logrotate b/debian/mariadb-server-10.0.mysql-server.logrotate index a19e9ec46a2..dbe2a7e7dfe 100644 --- a/debian/mariadb-server-10.0.mysql-server.logrotate +++ b/debian/mariadb-server-10.0.mysql-server.logrotate @@ -14,7 +14,8 @@ if [ -f `my_print_defaults --mysqld | grep -oP "pid-file=\K[^$]+"` ]; then # If this fails, check debian.conf! - mysqladmin --defaults-file=/etc/mysql/debian.cnf flush-logs + mysqladmin --defaults-file=/etc/mysql/debian.cnf --local flush-error-log \ + flush-engine-log flush-general-log flush-slow-log fi endscript } diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 585b5aef6f6..f2738752c56 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -80,6 +80,9 @@ IF(UNIX) MYSQL_ADD_EXECUTABLE(mysql_waitpid mysql_waitpid.c COMPONENT Client) TARGET_LINK_LIBRARIES(mysql_waitpid mysys) + + MYSQL_ADD_EXECUTABLE(mysqld_safe_helper mysqld_safe_helper.c COMPONENT Server) + TARGET_LINK_LIBRARIES(mysqld_safe_helper mysys) ENDIF() diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index c09458630c8..04807334baa 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -360,12 +360,13 @@ int main(int argc, char **argv) { if (verbose) printf("Number of pages: "); - printf("%lu\n", pages); + printf(ULINTPF "\n", pages); goto ok; } else if (verbose) { - printf("file %s = %llu bytes (%lu pages)...\n", filename, size, (ulong)pages); + printf("file %s = %llu bytes (" ULINTPF " pages)...\n", + filename, size, pages); if (do_one_page) printf("InnoChecksum; checking page %lu\n", do_page); else @@ -421,7 +422,8 @@ int main(int argc, char **argv) if (compressed) { /* compressed pages */ if (!page_zip_verify_checksum(buf, physical_page_size)) { - fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct); + fprintf(stderr, "Fail; page " ULINTPF + " invalid (fails compressed page checksum).\n", ct); if (!skip_corrupt) goto error; } @@ -431,10 +433,14 @@ int main(int argc, char **argv) logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4); logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4); if (debug) - printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield); + printf("page " ULINTPF + ": log sequence number: first = " ULINTPF + "; second = " ULINTPF "\n", + ct, logseq, logseqfield); if (logseq != logseqfield) { - fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct); + fprintf(stderr, "Fail; page " ULINTPF + " invalid (fails log sequence number check)\n", ct); if (!skip_corrupt) goto error; } @@ -443,10 +449,14 @@ int main(int argc, char **argv) oldcsum= buf_calc_page_old_checksum(buf); oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM); if (debug) - printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield); + printf("page " ULINTPF + ": old style: calculated = " ULINTPF + "; recorded = " ULINTPF "\n", + ct, oldcsum, oldcsumfield); if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum) { - fprintf(stderr, "Fail; page %lu invalid (fails old style checksum)\n", ct); + fprintf(stderr, "Fail; page " ULINTPF + " invalid (fails old style checksum)\n", ct); if (!skip_corrupt) goto error; } @@ -456,11 +466,14 @@ int main(int argc, char **argv) crc32= buf_calc_page_crc32(buf); csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM); if (debug) - printf("page %lu: new style: calculated = %lu; crc32 = %lu; recorded = %lu\n", + printf("page " ULINTPF + ": new style: calculated = " ULINTPF + "; crc32 = " ULINTPF "; recorded = " ULINTPF "\n", ct, csum, crc32, csumfield); if (csumfield != 0 && crc32 != csumfield && csum != csumfield) { - fprintf(stderr, "Fail; page %lu invalid (fails innodb and crc32 checksum)\n", ct); + fprintf(stderr, "Fail; page " ULINTPF + " invalid (fails innodb and crc32 checksum)\n", ct); if (!skip_corrupt) goto error; } @@ -479,7 +492,8 @@ int main(int argc, char **argv) if (!lastt) lastt= now; if (now - lastt >= 1) { - printf("page %lu okay: %.3f%% done\n", (ct - 1), (float) ct / pages * 100); + printf("page " ULINTPF " okay: %.3f%% done\n", + (ct - 1), (float) ct / pages * 100); lastt= now; } } diff --git a/extra/mysqld_safe_helper.c b/extra/mysqld_safe_helper.c new file mode 100644 index 00000000000..2ccff20b910 --- /dev/null +++ b/extra/mysqld_safe_helper.c @@ -0,0 +1,77 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#include <stdlib.h> +#include <stdio.h> + +void my_exit(int c) +{ + my_end(0); + exit(c); +} + +void do_usage() +{ + printf("Usage:\n" + " %s <user> log <filename>\n" + " %s <user> exec <command> <args>\n", + my_progname, my_progname); + my_exit(1); +} + +void do_log(const char *logfile) +{ + FILE *f; + uchar buf[4096]; + int size; + + if (!logfile) + do_usage(); + + f= my_fopen(logfile, O_WRONLY|O_APPEND|O_CREAT, MYF(MY_WME)); + if (!f) + my_exit(1); + + while ((size= my_fread(stdin, buf, sizeof(buf), MYF(MY_WME))) > 0) + if ((int)my_fwrite(f, buf, size, MYF(MY_WME)) != size) + my_exit(1); + + my_fclose(f, MYF(0)); + my_exit(0); +} + +void do_exec(char *args[]) +{ + if (!args[0]) + do_usage(); + + my_end(0); + execvp(args[0], args); +} + +int main(int argc, char *argv[]) +{ + struct passwd *user_info; + MY_INIT(argv[0]); + + if (argc < 3) + do_usage(); + + user_info= my_check_user(argv[1], MYF(0)); + if (user_info ? my_set_user(argv[1], user_info, MYF(MY_WME)) + : my_errno == EINVAL) + my_exit(1); + + if (strcmp(argv[2], "log") == 0) + do_log(argv[3]); + + if (strcmp(argv[2], "exec") == 0) + do_exec(argv+3); + + my_end(0); + return 1; +} diff --git a/extra/yassl/src/ssl.cpp b/extra/yassl/src/ssl.cpp index 7069140dcda..982d462e3b8 100644 --- a/extra/yassl/src/ssl.cpp +++ b/extra/yassl/src/ssl.cpp @@ -774,7 +774,6 @@ int SSL_CTX_load_verify_locations(SSL_CTX* ctx, const char* file, const char* path) { int ret = SSL_FAILURE; - const int HALF_PATH = 128; if (file) ret = read_file(ctx, file, SSL_FILETYPE_PEM, CA); diff --git a/include/my_global.h b/include/my_global.h index 007c9400ec0..a61fd765995 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -1078,10 +1078,9 @@ typedef ulong myf; /* Type of MyFlags in my_funcs */ static inline char *dlerror(void) { static char win_errormsg[2048]; - if(FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, - 0, GetLastError(), 0, win_errormsg, 2048, NULL)) - return win_errormsg; - return ""; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + 0, GetLastError(), 0, win_errormsg, 2048, NULL); + return win_errormsg; } #define HAVE_DLOPEN 1 #define HAVE_DLERROR 1 diff --git a/include/my_sys.h b/include/my_sys.h index f6bf57e50a4..22ed7616f1c 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -64,9 +64,9 @@ typedef struct my_aio_result { #define MY_FAE 8 /* Fatal if any error */ #define MY_WME 16 /* Write message on error */ #define MY_WAIT_IF_FULL 32 /* Wait and try again if disk full error */ -#define MY_IGNORE_BADFD 32 /* my_sync: ignore 'bad descriptor' errors */ -#define MY_UNUSED 64 /* Unused (was support for RAID) */ -#define MY_FULL_IO 512 /* For my_read - loop intil I/O is complete */ +#define MY_IGNORE_BADFD 32 /* my_sync(): ignore 'bad descriptor' errors */ +#define MY_NOSYMLINKS 512 /* my_open(): don't follow symlinks */ +#define MY_FULL_IO 512 /* my_read(): loop intil I/O is complete */ #define MY_DONT_CHECK_FILESIZE 128 /* Option to init_io_cache() */ #define MY_LINK_WARNING 32 /* my_redel() gives warning if links */ #define MY_COPYTIME 64 /* my_redel() copys time */ @@ -253,7 +253,7 @@ extern ulong my_file_opened,my_stream_opened, my_tmp_file_created; extern ulong my_file_total_opened; extern ulong my_sync_count; extern uint mysys_usage_id; -extern my_bool my_init_done; +extern my_bool my_init_done, my_thr_key_mysys_exists; extern my_bool my_assert_on_error; extern myf my_global_flags; /* Set to MY_WME for more error messages */ /* Point to current my_message() */ @@ -567,6 +567,7 @@ my_off_t my_b_safe_tell(IO_CACHE* info); /* picks the correct tell() */ typedef uint32 ha_checksum; extern ulong my_crc_dbug_check; +extern int (*mysys_test_invalid_symlink)(const char *filename); #include <my_alloc.h> /* Prototypes for mysys and my_func functions */ @@ -594,9 +595,10 @@ extern int my_realpath(char *to, const char *filename, myf MyFlags); extern File my_create_with_symlink(const char *linkname, const char *filename, int createflags, int access_flags, myf MyFlags); -extern int my_delete_with_symlink(const char *name, myf MyFlags); extern int my_rename_with_symlink(const char *from,const char *to,myf MyFlags); extern int my_symlink(const char *content, const char *linkname, myf MyFlags); +extern int my_handler_delete_with_symlink(const char *filename, myf sync_dir); + extern size_t my_read(File Filedes,uchar *Buffer,size_t Count,myf MyFlags); extern size_t my_pread(File Filedes,uchar *Buffer,size_t Count,my_off_t offset, myf MyFlags); @@ -620,8 +622,12 @@ extern void *my_memmem(const void *haystack, size_t haystacklen, #ifdef _WIN32 extern int my_access(const char *path, int amode); +#define my_check_user(A,B) (NULL) +#define my_set_user(A,B,C) (0) #else #define my_access access +struct passwd *my_check_user(const char *user, myf MyFlags); +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags); #endif extern int check_if_legal_filename(const char *path); @@ -900,6 +906,12 @@ extern ulonglong my_getcputime(void); #define hrtime_sec_part(X) ((ulong)((X).val % HRTIME_RESOLUTION)) #define my_time(X) hrtime_to_time(my_hrtime()) +#if STACK_DIRECTION < 0 +#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END)) +#else +#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR)) +#endif + #ifdef HAVE_SYS_MMAN_H #include <sys/mman.h> diff --git a/include/mysql/psi/mysql_file.h b/include/mysql/psi/mysql_file.h index c839b2b019b..be9d7116b9b 100644 --- a/include/mysql/psi/mysql_file.h +++ b/include/mysql/psi/mysql_file.h @@ -442,17 +442,17 @@ #endif /** - @def mysql_file_delete_with_symlink(K, P1, P2) + @def mysql_file_delete_with_symlink(K, P1, P2, P3) Instrumented delete with symbolic link. @c mysql_file_delete_with_symlink is a replacement - for @c my_delete_with_symlink. + for @c my_handler_delete_with_symlink. */ #ifdef HAVE_PSI_FILE_INTERFACE - #define mysql_file_delete_with_symlink(K, P1, P2) \ - inline_mysql_file_delete_with_symlink(K, __FILE__, __LINE__, P1, P2) + #define mysql_file_delete_with_symlink(K, P1, P2, P3) \ + inline_mysql_file_delete_with_symlink(K, __FILE__, __LINE__, P1, P2, P3) #else - #define mysql_file_delete_with_symlink(K, P1, P2) \ - inline_mysql_file_delete_with_symlink(P1, P2) + #define mysql_file_delete_with_symlink(K, P1, P2, P3) \ + inline_mysql_file_delete_with_symlink(P1, P2, P3) #endif /** @@ -1308,6 +1308,7 @@ inline_mysql_file_rename( return result; } + static inline File inline_mysql_file_create_with_symlink( #ifdef HAVE_PSI_FILE_INTERFACE @@ -1337,32 +1338,36 @@ inline_mysql_file_create_with_symlink( return file; } + static inline int inline_mysql_file_delete_with_symlink( #ifdef HAVE_PSI_FILE_INTERFACE PSI_file_key key, const char *src_file, uint src_line, #endif - const char *name, myf flags) + const char *name, const char *ext, myf flags) { int result; + char buf[FN_REFLEN]; + char *fullname= fn_format(buf, name, "", ext, MY_UNPACK_FILENAME | MY_APPEND_EXT); #ifdef HAVE_PSI_FILE_INTERFACE struct PSI_file_locker *locker; PSI_file_locker_state state; locker= PSI_FILE_CALL(get_thread_file_name_locker) - (&state, key, PSI_FILE_DELETE, name, &locker); + (&state, key, PSI_FILE_DELETE, fullname, &locker); if (likely(locker != NULL)) { PSI_FILE_CALL(start_file_close_wait)(locker, src_file, src_line); - result= my_delete_with_symlink(name, flags); + result= my_handler_delete_with_symlink(fullname, flags); PSI_FILE_CALL(end_file_close_wait)(locker, result); return result; } #endif - result= my_delete_with_symlink(name, flags); + result= my_handler_delete_with_symlink(fullname, flags); return result; } + static inline int inline_mysql_file_rename_with_symlink( #ifdef HAVE_PSI_FILE_INTERFACE @@ -1418,4 +1423,3 @@ inline_mysql_file_sync( /** @} (end of group File_instrumentation) */ #endif - diff --git a/include/mysql/psi/psi.h b/include/mysql/psi/psi.h index 7fcff89c8b6..3f43445e08a 100644 --- a/include/mysql/psi/psi.h +++ b/include/mysql/psi/psi.h @@ -417,7 +417,7 @@ enum PSI_file_operation PSI_FILE_FSTAT= 12, /** File chsize, as in @c my_chsize(). */ PSI_FILE_CHSIZE= 13, - /** File delete, such as @c my_delete() or @c my_delete_with_symlink(). */ + /** File delete, such as @c my_delete() or @c my_handler_delete_with_symlink(). */ PSI_FILE_DELETE= 14, /** File rename, such as @c my_rename() or @c my_rename_with_symlink(). */ PSI_FILE_RENAME= 15, diff --git a/include/welcome_copyright_notice.h b/include/welcome_copyright_notice.h index e9891856221..cd7cd6692be 100644 --- a/include/welcome_copyright_notice.h +++ b/include/welcome_copyright_notice.h @@ -1,5 +1,5 @@ -/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. - Copyright (c) 2011, 2016, MariaDB +/* Copyright (c) 2011, 2017, Oracle and/or its affiliates. + Copyright (c) 2011, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,7 +17,7 @@ #ifndef _welcome_copyright_notice_h_ #define _welcome_copyright_notice_h_ -#define COPYRIGHT_NOTICE_CURRENT_YEAR "2016" +#define COPYRIGHT_NOTICE_CURRENT_YEAR "2017" /* This define specifies copyright notice which is displayed by every MySQL diff --git a/libmysql/libmysql.c b/libmysql/libmysql.c index 3a08ea26b1d..c07fad38802 100644 --- a/libmysql/libmysql.c +++ b/libmysql/libmysql.c @@ -1405,7 +1405,7 @@ void set_stmt_errmsg(MYSQL_STMT *stmt, NET *net) DBUG_ASSERT(stmt != 0); stmt->last_errno= net->last_errno; - if (net->last_error && net->last_error[0]) + if (net->last_error[0]) strmov(stmt->last_error, net->last_error); strmov(stmt->sqlstate, net->sqlstate); @@ -4713,8 +4713,7 @@ my_bool STDCALL mysql_stmt_close(MYSQL_STMT *stmt) { uchar buff[MYSQL_STMT_HEADER]; /* 4 bytes - stmt id */ - if ((rc= reset_stmt_handle(stmt, RESET_ALL_BUFFERS | RESET_CLEAR_ERROR))) - return rc; + reset_stmt_handle(stmt, RESET_ALL_BUFFERS | RESET_CLEAR_ERROR); int4store(buff, stmt->stmt_id); if ((rc= stmt_command(mysql, COM_STMT_CLOSE, buff, 4, stmt))) diff --git a/man/mysql_secure_installation.1 b/man/mysql_secure_installation.1 index 20a77bc3f1e..2a04e5d0b98 100644 --- a/man/mysql_secure_installation.1 +++ b/man/mysql_secure_installation.1 @@ -1,6 +1,6 @@ '\" t .\" -.TH "\FBMYSQL_SECURE_INST" "1" "04/08/2015" "MariaDB 10\&.0" "MariaDB Database System" +.TH "\FBMYSQL_SECURE_INST" "1" "3 January 2017" "MariaDB 10\&.0" "MariaDB Database System" .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- @@ -71,9 +71,8 @@ test database, which by default can be accessed by anonymous users\&. .RE .PP -Invoke \fBmysql_secure_installation\fR -without arguments: +can be invoked without arguments: .sp .if n \{\ .RS 4 @@ -86,10 +85,75 @@ shell> \fBmysql_secure_installation\fR .\} .PP The script will prompt you to determine which actions to perform\&. +.PP +\fBmysql_secure_installation\fR +accepts some options: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: basedir option +.\" basedir option: mysql_secure_installation +\fB\-\-basedir=\fR\fB\fIdir_name\fR\fR +.sp +Base directory\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-extra-file option +.\" defaults-extra-file option: mysql_secure_installation +\fB\-\-defaults\-extra\-file=\fR\fB\fIfile_name\fR\fR +.sp +Additional option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-file option +.\" defaults-file option: mysql_secure_installation +\fB\-\-defaults\-file=\fR\fB\fIfile_name\fR\fR +.sp +Option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: no-defaults option +.\" no-defaults option: mysql_secure_installation +\fB\-\-no\-defaults\fR +.sp +Don't read any defaults file\&. +.RE +.sp +Other unrecognized options will be passed on to the server\&. .SH "COPYRIGHT" .br .PP -Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2015 MariaDB Foundation +Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2017 MariaDB Foundation .PP This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. .PP diff --git a/man/mysqladmin.1 b/man/mysqladmin.1 index e2b1753ff9f..65f348cab0c 100644 --- a/man/mysqladmin.1 +++ b/man/mysqladmin.1 @@ -816,6 +816,18 @@ Connect to the MariaDB server on the given host\&. .sp -1 .IP \(bu 2.3 .\} +.\" mysqladmin: local option +.\" local option: mysqladmin +\fB\-\-local\fR, +\fB\-l\fR +.sp +Suppress the SQL command(s) from being written to the binary log by enabled sql_log_bin=0 for the session\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} .\" mysqladmin: no-beep option .\" no-beep option: mysqladmin \fB\-\-no\-beep\fR, diff --git a/mysql-test/disabled.def b/mysql-test/disabled.def index 5c4b7421b21..17bc921ab66 100644 --- a/mysql-test/disabled.def +++ b/mysql-test/disabled.def @@ -12,7 +12,6 @@ tablespace : disabled in MariaDB (no TABLESPACE table attribute) events_time_zone : Test is not predictable as it depends on precise timing. read_many_rows_innodb : Bug#11748886 2010-11-15 mattiasj report already exists -log_tables-big : Bug#11756699 2010-11-15 mattiasj report already exists mysql_embedded : Bug#12561297 2011-05-14 Anitha Dependent on PB2 changes - eventum#41836 #show_explain : Psergey: random timeout in range-checked-for-each record query. ssl_crl_clients_valid : broken upstream diff --git a/mysql-test/include/gap_lock_error_all.inc b/mysql-test/include/gap_lock_error_all.inc new file mode 100644 index 00000000000..fc69dfb56f0 --- /dev/null +++ b/mysql-test/include/gap_lock_error_all.inc @@ -0,0 +1,27 @@ +--source include/have_partition.inc +--source include/gap_lock_error_init.inc + +let $select_lock=for update; +let $autocommit = 0; +--source include/gap_lock_error_select.inc +let $autocommit = 1; +--source include/gap_lock_error_select.inc + +let $select_lock=lock in share mode; +let $autocommit = 0; +--source include/gap_lock_error_select.inc +let $autocommit = 1; +--source include/gap_lock_error_select.inc + +let $select_lock=; +let $autocommit = 0; +--source include/gap_lock_error_select.inc +let $autocommit = 1; +--source include/gap_lock_error_select.inc + +let $autocommit = 0; +--source include/gap_lock_error_update.inc +let $autocommit = 1; +--source include/gap_lock_error_update.inc + +--source include/gap_lock_error_cleanup.inc diff --git a/mysql-test/include/gap_lock_error_cleanup.inc b/mysql-test/include/gap_lock_error_cleanup.inc new file mode 100644 index 00000000000..8558b5a528f --- /dev/null +++ b/mysql-test/include/gap_lock_error_cleanup.inc @@ -0,0 +1 @@ +drop table gap1, gap2, gap3, gap4; diff --git a/mysql-test/include/gap_lock_error_init.inc b/mysql-test/include/gap_lock_error_init.inc new file mode 100644 index 00000000000..26aa852a398 --- /dev/null +++ b/mysql-test/include/gap_lock_error_init.inc @@ -0,0 +1,24 @@ +eval CREATE TABLE gap1 (id1 INT, id2 INT, id3 INT, c1 INT, value INT, + PRIMARY KEY (id1, id2, id3), + INDEX i (c1)) ENGINE=$engine; +CREATE TABLE gap2 like gap1; +eval CREATE TABLE gap3 (id INT, value INT, + PRIMARY KEY (id), + UNIQUE KEY ui(value)) ENGINE=$engine; +eval CREATE TABLE gap4 (id INT, value INT, + PRIMARY KEY (id)) ENGINE=$engine + PARTITION BY HASH(id) PARTITIONS 2; +--disable_query_log +let $max = 1000; +let $i = 1; +while ($i <= $max) { + eval INSERT INTO gap1 (id1, id2, id3, c1, value) + VALUES ($i div 2, $i div 10, $i, $i, $i); + eval INSERT INTO gap2 (id1, id2, id3, c1, value) + VALUES ($i div 2, $i div 10, $i, $i, $i); + inc $i; +} +--enable_query_log + +insert into gap3 values (1,1), (2,2),(3,3),(4,4),(5,5); +insert into gap4 values (1,1), (2,2),(3,3),(4,4),(5,5); diff --git a/mysql-test/include/gap_lock_error_select.inc b/mysql-test/include/gap_lock_error_select.inc new file mode 100644 index 00000000000..91db9bed68a --- /dev/null +++ b/mysql-test/include/gap_lock_error_select.inc @@ -0,0 +1,89 @@ +eval set session autocommit=$autocommit; +let $is_gaplock_target = `SELECT @@autocommit = 0 && '$select_lock' != '' && '$expect_gap_lock_errors' = 1`; + +if ($is_gaplock_target) +{ +# rnd_init +--error ER_UNKNOWN_ERROR +eval select * from gap1 limit 1 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 where value != 100 limit 1 $select_lock; +# index_read_map +--error ER_UNKNOWN_ERROR +eval select * from gap1 where id1=1 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 where id1=1 and id2= 1 $select_lock; +# read_range_first +--error ER_UNKNOWN_ERROR +eval select * from gap1 where id1=1 and id2= 1 and id3 != 1 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 where id1=1 and id2= 1 and id3 + between 1 and 3 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 where id1=1 and id2= 1 order by id3 asc + limit 1 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 where id1=1 and id2= 1 order by id3 desc + limit 1 $select_lock; +# index_first +--error ER_UNKNOWN_ERROR +eval select * from gap1 order by id1 asc limit 1 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 $select_lock; +# index_last +--error ER_UNKNOWN_ERROR +eval select * from gap1 order by id1 desc limit 1 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap1 order by id1 desc, id2 desc, id3 desc + limit 1 $select_lock; +# secondary index lookup +--error ER_UNKNOWN_ERROR +eval select * from gap1 force index(i) where c1=1 $select_lock; +# unique index lookup, ensure no gap lock errors as this is effectively a +# single point select that does not lock ranges or gaps of keys +eval select * from gap3 force index(ui) where value=1 $select_lock; +# primary key lookup, ensure no gap lock errors as these are effectively +# single point selects that do not lock ranges or gaps of keys +eval select * from gap1 where id1=1 and id2=1 and id3=1 $select_lock; +eval select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) $select_lock; +eval select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 + order by c1 $select_lock; +eval select * from gap3 where id=1 $select_lock; +eval select * from gap4 where id=1 $select_lock; +eval select * from gap4 where id in (1, 2, 3) $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap4 $select_lock; +--error ER_UNKNOWN_ERROR +eval select * from gap4 where id between 3 and 7 $select_lock; +} + +if (!$is_gaplock_target) +{ +eval select * from gap1 limit 1 $select_lock; +eval select * from gap1 where value != 100 limit 1 $select_lock; +eval select * from gap1 where id1=1 $select_lock; +eval select * from gap1 where id1=1 and id2= 1 $select_lock; +eval select * from gap1 where id1=1 and id2= 1 and id3 != 1 $select_lock; +eval select * from gap1 where id1=1 and id2= 1 and id3 + between 1 and 3 $select_lock; +eval select * from gap1 where id1=1 and id2= 1 order by id3 asc + limit 1 $select_lock; +eval select * from gap1 where id1=1 and id2= 1 order by id3 desc + limit 1 $select_lock; +eval select * from gap1 order by id1 asc limit 1 $select_lock; +eval select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 $select_lock; +eval select * from gap1 order by id1 desc limit 1 $select_lock; +eval select * from gap1 order by id1 desc, id2 desc, id3 desc + limit 1 $select_lock; +eval select * from gap1 force index(i) where c1=1 $select_lock; +eval select * from gap3 force index(ui) where value=1 $select_lock; +eval select * from gap1 where id1=1 and id2=1 and id3=1 $select_lock; +eval select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) $select_lock; +eval select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 + order by c1 $select_lock; +eval select * from gap3 where id=1 $select_lock; +eval select * from gap4 where id=1 $select_lock; +eval select * from gap4 where id in (1, 2, 3) $select_lock; +eval select * from gap4 $select_lock; +eval select * from gap4 where id between 3 and 7 $select_lock; +} diff --git a/mysql-test/include/gap_lock_error_update.inc b/mysql-test/include/gap_lock_error_update.inc new file mode 100644 index 00000000000..d456cf81588 --- /dev/null +++ b/mysql-test/include/gap_lock_error_update.inc @@ -0,0 +1,91 @@ +eval set session autocommit=$autocommit; +let $is_gaplock_target = `SELECT @@autocommit = 0 && '$expect_gap_lock_errors' = 1`; + +if ($is_gaplock_target) +{ +## single-table insert,update,delete +insert into gap1 (id1, id2, id3) values (-1,-1,-1); +insert into gap1 (id1, id2, id3) values (-1,-1,-1) + on duplicate key update value=100; +--error ER_UNKNOWN_ERROR +update gap1 set value=100 where id1=1; +update gap1 set value=100 where id1=1 and id2=1 and id3=1; +--error ER_UNKNOWN_ERROR +delete from gap1 where id1=2; +delete from gap1 where id1=-1 and id2=-1 and id3=-1; +commit; + +## multi-table statements (preventing all gap locks with autocommit) +# insert into select +--error ER_UNKNOWN_ERROR +insert into gap2 select * from gap1; +--error ER_UNKNOWN_ERROR +insert into gap2 select * from gap1 where id1=1; +insert into gap2 select * from gap1 where id1=1 and id2=1 and id3=1; + +# create table select +create table t4 select * from gap1 where id1=1 and id2=1 and id3=1; +drop table t4; +--error ER_UNKNOWN_ERROR +create table t4 select * from gap1; +--error ER_UNKNOWN_ERROR +create table t4 select * from gap1 where id1=1; + +# update join +update gap1 join gap2 on gap1.id1 and gap1.id2=gap2.id2 set gap1.value=100 where gap2.id1=3 + and gap2.id2=3 and gap2.id3=3; +--error ER_UNKNOWN_ERROR +update gap1 join gap2 on gap1.id1 and gap1.id2=gap2.id2 set gap1.value=100 where gap2.id1=3; +--error ER_UNKNOWN_ERROR +update gap1 join gap2 on gap1.id1 and gap1.id2=gap2.id2 join gap3 on gap1.id1=gap3.id + set gap1.value=100 where gap2.id1=3; +--error ER_UNKNOWN_ERROR +update gap1 set gap1.value= (select count(*) from gap2); + +# delete join +delete gap1 from gap1 join gap2 on gap1.id1 and gap1.id2=gap2.id2 where gap2.id1=3 + and gap2.id2=3 and gap2.id3=3; +--error ER_UNKNOWN_ERROR +delete gap1 from gap1 join gap2 on gap1.id1 and gap1.id2=gap2.id2 where gap2.id1=3; + +# select join / self join +--error ER_UNKNOWN_ERROR +select * from gap1, gap2 limit 1 for update; +--error ER_UNKNOWN_ERROR +select * from gap1 a, gap1 b limit 1 for update; + +# unique secondary key +create table u1( + c1 int, + c2 int, + c3 int, + c4 int, + primary key (c1, c2, c3), + unique key (c3, c1) +); +begin; +insert into u1 values (1,1,1,1); +commit; +begin; +insert into u1 values (1,2,1,1) on duplicate key update c4=10; +commit; +begin; +select * from u1 where c3=1 and c1 = 1 for update; +--error ER_UNKNOWN_ERROR +select * from u1 where c3=1 for update; +commit; +drop table u1; +} + +if (!$is_gaplock_target) +{ +# autocommit doesn't prevent single table operations +insert into gap1 (id1, id2, id3) values (-1,-1,-1); +insert into gap1 (id1, id2, id3) values (-1,-1,-1) + on duplicate key update value=100; +update gap1 set value=100 where id1=1; +update gap1 set value=100 where id1=1 and id2=1 and id3=1; +delete from gap1 where id1=2; +delete from gap1 where id1=-1 and id2=-1 and id3=-1; +commit; +} diff --git a/mysql-test/include/kill_and_restart_mysqld.inc b/mysql-test/include/kill_and_restart_mysqld.inc new file mode 100644 index 00000000000..f2ac9b504d2 --- /dev/null +++ b/mysql-test/include/kill_and_restart_mysqld.inc @@ -0,0 +1,19 @@ +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect + +if ($restart_parameters) +{ + --echo # Kill and restart: $restart_parameters + --exec echo "restart: $restart_parameters" > $_expect_file_name +} +if (!$restart_parameters) +{ + --echo # Kill and restart + --exec echo "restart" > $_expect_file_name +} + +--shutdown_server 0 +--source include/wait_until_disconnected.inc +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect diff --git a/mysql-test/include/kill_mysqld.inc b/mysql-test/include/kill_mysqld.inc new file mode 100644 index 00000000000..86ee048a0f1 --- /dev/null +++ b/mysql-test/include/kill_mysqld.inc @@ -0,0 +1,7 @@ +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect + +--echo # Kill the server +--exec echo "wait" > $_expect_file_name +--shutdown_server 0 +--source include/wait_until_disconnected.inc diff --git a/mysql-test/include/mtr_warnings.sql b/mysql-test/include/mtr_warnings.sql index 8f5a4fe6685..0382edf8097 100644 --- a/mysql-test/include/mtr_warnings.sql +++ b/mysql-test/include/mtr_warnings.sql @@ -293,6 +293,7 @@ CREATE DEFINER=root@localhost PROCEDURE add_suppression(pattern VARCHAR(255)) BEGIN INSERT INTO test_suppressions (pattern) VALUES (pattern); + FLUSH NO_WRITE_TO_BINLOG TABLE test_suppressions; END */|| diff --git a/mysql-test/include/restart_mysqld.inc b/mysql-test/include/restart_mysqld.inc index 71efb141917..dcaf47c55a2 100644 --- a/mysql-test/include/restart_mysqld.inc +++ b/mysql-test/include/restart_mysqld.inc @@ -35,7 +35,14 @@ if ($shutdown_timeout == 0) shutdown_server $server_shutdown_timeout; # Write file to make mysql-test-run.pl start up the server again ---exec echo "restart" > $_expect_file_name +if ($restart_parameters) +{ + --exec echo "restart: $restart_parameters" > $_expect_file_name +} +if (!$restart_parameters) +{ + --exec echo "restart" > $_expect_file_name +} # Turn on reconnect --enable_reconnect diff --git a/mysql-test/include/search_pattern_in_file.inc b/mysql-test/include/search_pattern_in_file.inc index 84237026ed0..f77a7c60916 100644 --- a/mysql-test/include/search_pattern_in_file.inc +++ b/mysql-test/include/search_pattern_in_file.inc @@ -60,25 +60,36 @@ perl; use strict; - my $search_file= $ENV{'SEARCH_FILE'} or die "SEARCH_FILE not set"; + die "SEARCH_FILE not set" unless $ENV{'SEARCH_FILE'}; + my @search_files= glob($ENV{'SEARCH_FILE'}); my $search_pattern= $ENV{'SEARCH_PATTERN'} or die "SEARCH_PATTERN not set"; my $search_range= $ENV{'SEARCH_RANGE'}; - my $file_content; + my $content; $search_range= 50000 unless $search_range =~ /-?[0-9]+/; - open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n"); - if ($search_range >= 0) { - read(FILE, $file_content, $search_range, 0); - } else { - my $size= -s $search_file; - $search_range = -$size if $size > -$search_range; - seek(FILE, $search_range, 2); - read(FILE, $file_content, -$search_range, 0); + foreach my $search_file (@search_files) { + open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n"); + my $file_content; + if ($search_range >= 0) { + read(FILE, $file_content, $search_range, 0); + } else { + my $size= -s $search_file; + $search_range = -$size if $size > -$search_range; + seek(FILE, $search_range, 2); + read(FILE, $file_content, -$search_range, 0); + } + close(FILE); + $content.= $file_content; } - close(FILE); - $search_file =~ s{^.*?([^/\\]+)$}{$1}; - if ($file_content =~ m{$search_pattern}) { - print "FOUND /$search_pattern/ in $search_file\n" + $ENV{'SEARCH_FILE'} =~ s{^.*?([^/\\]+)$}{$1}; + if ($content =~ m{$search_pattern}) { + die "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + if $ENV{SEARCH_ABORT} eq 'FOUND'; + print "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + unless defined $ENV{SEARCH_ABORT}; } else { - print "NOT FOUND /$search_pattern/ in $search_file\n" + die "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + if $ENV{SEARCH_ABORT} eq 'NOT FOUND'; + print "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + unless defined $ENV{SEARCH_ABORT}; } EOF diff --git a/mysql-test/include/start_mysqld.inc b/mysql-test/include/start_mysqld.inc index 983c566821e..e31f26aad8c 100644 --- a/mysql-test/include/start_mysqld.inc +++ b/mysql-test/include/start_mysqld.inc @@ -1,7 +1,14 @@ # Include this script only after using shutdown_mysqld.inc # where $_expect_file_name was initialized. # Write file to make mysql-test-run.pl start up the server again ---exec echo "restart" > $_expect_file_name +if ($restart_parameters) +{ + --exec echo "restart: $restart_parameters" > $_expect_file_name +} +if (!$restart_parameters) +{ + --exec echo "restart" > $_expect_file_name +} # Turn on reconnect --enable_reconnect diff --git a/mysql-test/include/sync_slave_sql_with_io.inc b/mysql-test/include/sync_slave_sql_with_io.inc index 8048f7a177c..9efede9a61f 100644 --- a/mysql-test/include/sync_slave_sql_with_io.inc +++ b/mysql-test/include/sync_slave_sql_with_io.inc @@ -26,6 +26,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } --let $_master_log_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1) diff --git a/mysql-test/include/sync_with_master_gtid.inc b/mysql-test/include/sync_with_master_gtid.inc index 97ada8eea29..777711b979c 100644 --- a/mysql-test/include/sync_with_master_gtid.inc +++ b/mysql-test/include/sync_with_master_gtid.inc @@ -34,6 +34,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 120; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1200; + } } --let $_result= `SELECT master_gtid_wait('$master_pos', $_slave_timeout)` diff --git a/mysql-test/include/wait_for_slave_param.inc b/mysql-test/include/wait_for_slave_param.inc index d3f7ec56614..25020d46ed9 100644 --- a/mysql-test/include/wait_for_slave_param.inc +++ b/mysql-test/include/wait_for_slave_param.inc @@ -50,6 +50,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } if ($slave_error_param == '') diff --git a/mysql-test/lib/My/CoreDump.pm b/mysql-test/lib/My/CoreDump.pm index f9f7b3d8d4b..a421d51ec98 100644 --- a/mysql-test/lib/My/CoreDump.pm +++ b/mysql-test/lib/My/CoreDump.pm @@ -53,9 +53,19 @@ sub _verify_binpath { sub _gdb { my ($core_name)= @_; - print "\nTrying 'gdb' to get a backtrace\n"; + # Check that gdb exists + `gdb --version`; + if ($?) { + print "gdb not found, cannot get the stack trace\n"; + return; + } - return unless -f $core_name; + if (-f $core_name) { + print "\nTrying 'gdb' to get a backtrace from coredump $core_name\n"; + } else { + print "\nCoredump $core_name does not exist, cannot run 'gdb'\n"; + return; + } # Find out name of binary that generated core `gdb -c '$core_name' --batch 2>&1` =~ diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 9bfea2577c6..6f9481c7491 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1054,7 +1054,7 @@ sub print_global_resfile { resfile_global("gprof", $opt_gprof ? 1 : 0); resfile_global("valgrind", $opt_valgrind ? 1 : 0); resfile_global("callgrind", $opt_callgrind ? 1 : 0); - resfile_global("mem", $opt_mem ? 1 : 0); + resfile_global("mem", $opt_mem); resfile_global("tmpdir", $opt_tmpdir); resfile_global("vardir", $opt_vardir); resfile_global("fast", $opt_fast ? 1 : 0); @@ -1452,12 +1452,14 @@ sub command_line_setup { # Search through list of locations that are known # to be "fast disks" to find a suitable location - # Use --mem=<dir> as first location to look. - my @tmpfs_locations= ($opt_mem,"/run/shm", "/dev/shm", "/tmp"); + my @tmpfs_locations= ("/run/shm", "/dev/shm", "/tmp"); + + # Use $ENV{'MTR_MEM'} as first location to look (if defined) + unshift(@tmpfs_locations, $ENV{'MTR_MEM'}) if defined $ENV{'MTR_MEM'}; foreach my $fs (@tmpfs_locations) { - if ( -d $fs ) + if ( -d $fs && ! -l $fs ) { my $template= "var_${opt_build_thread}_XXXX"; $opt_mem= tempdir( $template, DIR => $fs, CLEANUP => 0); @@ -3220,13 +3222,10 @@ sub mysql_server_start($) { if (! $opt_start_dirty) # If dirty, keep possibly grown system db { # Copy datadir from installed system db - for my $path ( "$opt_vardir", "$opt_vardir/..") { - my $install_db= "$path/install.db"; - copytree($install_db, $datadir) - if -d $install_db; - } - mtr_error("Failed to copy system db to '$datadir'") - unless -d $datadir; + my $path= ($opt_parallel == 1) ? "$opt_vardir" : "$opt_vardir/.."; + my $install_db= "$path/install.db"; + copytree($install_db, $datadir) if -d $install_db; + mtr_error("Failed to copy system db to '$datadir'") unless -d $datadir; } } else @@ -4792,6 +4791,7 @@ sub extract_warning_lines ($$) { qr/InnoDB: Error: table `test`.`t[12]` .*does not exist in the InnoDB internal/, qr/InnoDB: Warning: Setting innodb_use_sys_malloc/, qr/InnoDB: Warning: a long semaphore wait:/, + qr/InnoDB: Warning: Writer thread is waiting this semaphore:/, qr/Slave: Unknown table 't1' .* 1051/, qr/Slave SQL:.*(Internal MariaDB error code: [[:digit:]]+|Query:.*)/, qr/slave SQL thread aborted/, @@ -6427,9 +6427,9 @@ Options to control directories to use vardir=DIR The directory where files generated from the test run is stored (default: ./var). Specifying a ramdisk or tmpfs will speed up tests. - mem Run testsuite in "memory" using tmpfs or ramdisk - Attempts to find a suitable location - using a builtin list of standard locations + mem[=DIR] Run testsuite in "memory" using tmpfs or ramdisk + Attempts to use DIR first if specified else + uses a builtin list of standard locations for tmpfs (/run/shm, /dev/shm, /tmp) The option can also be set using environment variable MTR_MEM=[DIR] diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 67de5f3160a..cac4c477b5a 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2076,3 +2076,39 @@ tab1 CREATE TABLE `tab1` ( PRIMARY KEY (`id`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 DROP TABLE `tab1`; +# +# MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS +# +CREATE TABLE t1 (id INT UNSIGNED NOT NULL PRIMARY KEY); +CREATE TABLE t2 (id1 INT UNSIGNED NOT NULL); +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); +Warnings: +Note 1061 Duplicate key name 'id1' +DROP TABLE t2; +DROP TABLE t1; +# +# MDEV-6390 CONVERT TO CHARACTER SET utf8 doesn't change DEFAULT CHARSET. +# +CREATE TABLE t1 (id int(11) NOT NULL, a int(11) NOT NULL, b int(11)) +ENGINE=InnoDB DEFAULT CHARSET=latin1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL, + `a` int(11) NOT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +ALTER TABLE t1 CONVERT TO CHARACTER SET utf8; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL, + `a` int(11) NOT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8 +DROP TABLE t1; diff --git a/mysql-test/r/contributors.result b/mysql-test/r/contributors.result index f3f5e227d3a..4a26d0f19dd 100644 --- a/mysql-test/r/contributors.result +++ b/mysql-test/r/contributors.result @@ -1,15 +1,17 @@ SHOW CONTRIBUTORS; Name Location Comment -Booking.com http://www.booking.com Founding member, Platinum Sponsor of the MariaDB Foundation +Booking.com https://www.booking.com Founding member, Platinum Sponsor of the MariaDB Foundation +Alibaba Cloud https://intl.aliyun.com Platinum Sponsor of the MariaDB Foundation MariaDB Corporation https://mariadb.com Founding member, Gold Sponsor of the MariaDB Foundation -Visma http://visma.com Gold Sponsor of the MariaDB Foundation -DBS http://dbs.com Gold Sponsor of the MariaDB Foundation +Visma https://visma.com Gold Sponsor of the MariaDB Foundation +DBS https://dbs.com Gold Sponsor of the MariaDB Foundation Nexedi https://www.nexedi.com Silver Sponsor of the MariaDB Foundation Acronis http://www.acronis.com Silver Sponsor of the MariaDB Foundation Auttomattic https://automattic.com Bronze Sponsor of the MariaDB Foundation -Verkkokauppa.com https://virtuozzo.com Bronze Sponsor of the MariaDB Foundation -Virtuozzo https://virtuozzo.com/ Bronze Sponsor of the MariaDB Foundation -Tencent Game DBA http://tencentdba.com/about/ Bronze Sponsor of the MariaDB Foundation +Verkkokauppa.com https://www.verkkokauppa.com Bronze Sponsor of the MariaDB Foundation +Virtuozzo https://virtuozzo.com Bronze Sponsor of the MariaDB Foundation +Tencent Game DBA http://tencentdba.com/about Bronze Sponsor of the MariaDB Foundation +Tencent TDSQL http://tdsql.org Bronze Sponsor of the MariaDB Foundation Google USA Sponsoring encryption, parallel replication and GTID Facebook USA Sponsoring non-blocking API, LIMIT ROWS EXAMINED etc Ronald Bradford Brisbane, Australia EFF contribution for UC2006 Auction diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index 5ef11e24820..a664801a310 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -2678,6 +2678,13 @@ end| create table t1 as select f1(); ERROR 42S02: Table 'test.t1' doesn't exist drop function f1; +# +# MDEV-10274 Bundling insert with create statement +# for table with unsigned Decimal primary key issues warning 1194 +# +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory +select 2.1 ID; +drop table t1; End of 5.5 tests create table t1; ERROR 42000: A table must have at least 1 column diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result index 6b3d87b79e1..843b49c1f75 100644 --- a/mysql-test/r/ctype_ucs.result +++ b/mysql-test/r/ctype_ucs.result @@ -4550,6 +4550,16 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)); CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)) 1 # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=ucs2; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_ucs2_def.result b/mysql-test/r/ctype_ucs2_def.result index 5ca7d1689d2..9d8f5181103 100644 --- a/mysql-test/r/ctype_ucs2_def.result +++ b/mysql-test/r/ctype_ucs2_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); show variables like 'collation_server'; Variable_name Value collation_server ucs2_unicode_ci diff --git a/mysql-test/r/ctype_ucs2_query_cache.result b/mysql-test/r/ctype_ucs2_query_cache.result index 9a7580324c1..eba7f2fb0fc 100644 --- a/mysql-test/r/ctype_ucs2_query_cache.result +++ b/mysql-test/r/ctype_ucs2_query_cache.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # Start of 5.5 tests # diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result index 2d91ce3dd6f..958ade902df 100644 --- a/mysql-test/r/ctype_utf16.result +++ b/mysql-test/r/ctype_utf16.result @@ -1580,6 +1580,16 @@ ERROR HY000: Invalid utf16 character string: 'DE9899' DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); ERROR HY000: Invalid utf16 character string: 'DE9899' # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=utf16; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_utf16_def.result b/mysql-test/r/ctype_utf16_def.result index 9d3d110fc99..98b6f7d913d 100644 --- a/mysql-test/r/ctype_utf16_def.result +++ b/mysql-test/r/ctype_utf16_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); SHOW VARIABLES LIKE 'collation_server'; Variable_name Value collation_server utf16_general_ci diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result index 3269e6c900e..076cabf16f7 100644 --- a/mysql-test/r/ctype_utf32.result +++ b/mysql-test/r/ctype_utf32.result @@ -1662,6 +1662,16 @@ select hex(lower(cast(0xffff0000 as char character set utf32))) as c; c FFFF0000 # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=utf32; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/derived.result b/mysql-test/r/derived.result index 0b392305bbf..ce94fe49f5c 100644 --- a/mysql-test/r/derived.result +++ b/mysql-test/r/derived.result @@ -1014,4 +1014,27 @@ David Yes 210 Edward Yes 150 DROP TABLE example1463; set sql_mode= @save_sql_mode; +# +# MDEV-9028: SELECT DISTINCT constant column of derived table +# used as the second operand of LEFT JOIN +# +create table t1 (id int, data varchar(255)); +insert into t1 values (1,'yes'),(2,'yes'); +select distinct t1.id, tt.id, tt.data +from t1 +left join +(select t1.id, 'yes' as data from t1) as tt +on t1.id = tt.id; +id id data +1 1 yes +2 2 yes +select distinct t1.id, tt.id, tt.data +from t1 +left join +(select t1.id, 'yes' as data from t1 where id > 1) as tt +on t1.id = tt.id; +id id data +2 2 yes +1 NULL NULL +drop table t1; # end of 5.5 diff --git a/mysql-test/r/events_2.result b/mysql-test/r/events_2.result index 06075b0e66e..65a2de3bae5 100644 --- a/mysql-test/r/events_2.result +++ b/mysql-test/r/events_2.result @@ -1,10 +1,10 @@ drop database if exists events_test; create database events_test; use events_test; -create event e_26 on schedule at '2027-01-01 00:00:00' disable do set @a = 5; +create event e_26 on schedule at '2037-01-01 00:00:00' disable do set @a = 5; select db, name, body, definer, convert_tz(execute_at, 'UTC', 'SYSTEM'), on_completion from mysql.event; db name body definer convert_tz(execute_at, 'UTC', 'SYSTEM') on_completion -events_test e_26 set @a = 5 root@localhost 2027-01-01 00:00:00 DROP +events_test e_26 set @a = 5 root@localhost 2037-01-01 00:00:00 DROP drop event e_26; create event e_26 on schedule at NULL disable do set @a = 5; ERROR HY000: Incorrect AT value: 'NULL' diff --git a/mysql-test/r/events_slowlog.result b/mysql-test/r/events_slowlog.result new file mode 100644 index 00000000000..7de5925bc0f --- /dev/null +++ b/mysql-test/r/events_slowlog.result @@ -0,0 +1,13 @@ +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); +FOUND /update t1 set i=1/ in mysqld-slow.log +drop table t1; +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/r/func_regexp_pcre.result b/mysql-test/r/func_regexp_pcre.result index aa090c9faf6..af7230d2ad0 100644 --- a/mysql-test/r/func_regexp_pcre.result +++ b/mysql-test/r/func_regexp_pcre.result @@ -874,3 +874,8 @@ SET @regCheck= '\\xE0\\x01'; SELECT CAST(0xE001 AS BINARY) REGEXP @regCheck; CAST(0xE001 AS BINARY) REGEXP @regCheck 1 +# MDEV-12420: Testing recursion overflow +SELECT 1 FROM dual WHERE ('Alpha,Bravo,Charlie,Delta,Echo,Foxtrot,StrataCentral,Golf,Hotel,India,Juliet,Kilo,Lima,Mike,StrataL3,November,Oscar,StrataL2,Sand,P3,P4SwitchTest,Arsys,Poppa,ExtensionMgr,Arp,Quebec,Romeo,StrataApiV2,PtReyes,Sierra,SandAcl,Arrow,Artools,BridgeTest,Tango,SandT,PAlaska,Namespace,Agent,Qos,PatchPanel,ProjectReport,Ark,Gimp,Agent,SliceAgent,Arnet,Bgp,Ale,Tommy,Central,AsicPktTestLib,Hsc,SandL3,Abuild,Pca9555,Standby,ControllerDut,CalSys,SandLib,Sb820,PointV2,BfnLib,Evpn,BfnSdk,Sflow,ManagementActive,AutoTest,GatedTest,Bgp,Sand,xinetd,BfnAgentLib,bf-utils,Hello,BfnState,Eos,Artest,Qos,Scd,ThermoMgr,Uniform,EosUtils,Eb,FanController,Central,BfnL3,BfnL2,tcp_wrappers,Victor,Environment,Route,Failover,Whiskey,Xray,Gimp,BfnFixed,Strata,SoCal,XApi,Msrp,XpProfile,tcpdump,PatchPanel,ArosTest,FhTest,Arbus,XpAcl,MacConc,XpApi,telnet,QosTest,Alpha2,BfnVlan,Stp,VxlanControllerTest,MplsAgent,Bravo2,Lanz,BfnMbb,Intf,XCtrl,Unicast,SandTunnel,L3Unicast,Ipsec,MplsTest,Rsvp,EthIntf,StageMgr,Sol,MplsUtils,Nat,Ira,P4NamespaceDut,Counters,Charlie2,Aqlc,Mlag,Power,OpenFlow,Lag,RestApi,BfdTest,strongs,Sfa,CEosUtils,Adt746,MaintenanceMode,MlagDut,EosImage,IpEth,MultiProtocol,Launcher,Max3179,Snmp,Acl,IpEthTest,PhyEee,bf-syslibs,tacc,XpL2,p4-ar-switch,p4-bf-switch,LdpTest,BfnPhy,Mirroring,Phy6,Ptp' REGEXP '^((?!\b(Strata|StrataApi|StrataApiV2)\b).)*$'); +1 +Warnings: +Warning 1139 Got error 'pcre_exec: recursion limit of NUM exceeded' from regexp diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 243a2b1f6d4..33aaaf21edb 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -2739,6 +2739,12 @@ id date1 date2 DATE_ADD(a.date1,INTERVAL -10 DAY) TO_DAYS(a.date1)-10 18 2010-10-13 2010-10-03 2010-10-03 734413 DROP TABLE t1; # +# MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +# +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; +1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2 +3 +# # Start of 10.0 tests # # diff --git a/mysql-test/r/gis.result b/mysql-test/r/gis.result index 174088bf944..2bd492008bc 100644 --- a/mysql-test/r/gis.result +++ b/mysql-test/r/gis.result @@ -1630,3 +1630,16 @@ SELECT ASTEXT(p) FROM v1; ASTEXT(p) POINT(1 1) DROP VIEW v1; +# +# Start of 10.0 tests +# +# +# MDEV-12495 Conditional jump depends on uninitialised value for: SELECT NULL UNION geom_expression +# +SELECT AsText(g) FROM (SELECT NULL AS g UNION SELECT Point(1,1)) AS t1; +AsText(g) +NULL +POINT(1 1) +# +# End 10.0 tests +# diff --git a/mysql-test/r/grant.result b/mysql-test/r/grant.result index 3194573e2ef..0b17e82e5a9 100644 --- a/mysql-test/r/grant.result +++ b/mysql-test/r/grant.result @@ -2527,6 +2527,54 @@ DROP USER mysqltest_u1@localhost; # End of Bug#38347. # +# BUG#11759114 - '51401: GRANT TREATS NONEXISTENT FUNCTIONS/PRIVILEGES +# DIFFERENTLY'. +# +drop database if exists mysqltest_db1; +create database mysqltest_db1; +create user mysqltest_u1; +# Both GRANT statements below should fail with the same error. +grant execute on function mysqltest_db1.f1 to mysqltest_u1; +ERROR 42000: FUNCTION or PROCEDURE f1 does not exist +grant execute on procedure mysqltest_db1.p1 to mysqltest_u1; +ERROR 42000: FUNCTION or PROCEDURE p1 does not exist +# Let us show that GRANT behaviour for routines is consistent +# with GRANT behaviour for tables. Attempt to grant privilege +# on non-existent table also results in an error. +grant select on mysqltest_db1.t1 to mysqltest_u1; +ERROR 42S02: Table 'mysqltest_db1.t1' doesn't exist +show grants for mysqltest_u1; +Grants for mysqltest_u1@% +GRANT USAGE ON *.* TO 'mysqltest_u1'@'%' +drop database mysqltest_db1; +drop user mysqltest_u1; +# +# Bug#12766319 - 61865: RENAME USER DOES NOT WORK CORRECTLY - +# REQUIRES FLUSH PRIVILEGES +# +CREATE USER foo@'127.0.0.1'; +GRANT ALL ON *.* TO foo@'127.0.0.1'; +# First attempt, should connect successfully +SELECT user(), current_user(); +user() current_user() +foo@localhost foo@127.0.0.1 +# Rename the user +RENAME USER foo@'127.0.0.1' to foo@'127.0.0.0/255.0.0.0'; +# Second attempt, should connect successfully as its valid mask +# This was failing without fix +SELECT user(), current_user(); +user() current_user() +foo@localhost foo@127.0.0.0/255.0.0.0 +# Rename the user back to original +RENAME USER foo@'127.0.0.0/255.0.0.0' to foo@'127.0.0.1'; +# Third attempt, should connect successfully +SELECT user(), current_user(); +user() current_user() +foo@localhost foo@127.0.0.1 +# Clean-up +DROP USER foo@'127.0.0.1'; +# End of Bug#12766319 +# # Bug#11756966 - 48958: STORED PROCEDURES CAN BE LEVERAGED TO BYPASS # DATABASE SECURITY # @@ -2552,25 +2600,3 @@ ERROR 42000: Access denied for user 'untrusted'@'localhost' to database 'secret' # Connection default DROP USER untrusted@localhost; DROP DATABASE secret; -# -# BUG#11759114 - '51401: GRANT TREATS NONEXISTENT FUNCTIONS/PRIVILEGES -# DIFFERENTLY'. -# -drop database if exists mysqltest_db1; -create database mysqltest_db1; -create user mysqltest_u1; -# Both GRANT statements below should fail with the same error. -grant execute on function mysqltest_db1.f1 to mysqltest_u1; -ERROR 42000: FUNCTION or PROCEDURE f1 does not exist -grant execute on procedure mysqltest_db1.p1 to mysqltest_u1; -ERROR 42000: FUNCTION or PROCEDURE p1 does not exist -# Let us show that GRANT behaviour for routines is consistent -# with GRANT behaviour for tables. Attempt to grant privilege -# on non-existent table also results in an error. -grant select on mysqltest_db1.t1 to mysqltest_u1; -ERROR 42S02: Table 'mysqltest_db1.t1' doesn't exist -show grants for mysqltest_u1; -Grants for mysqltest_u1@% -GRANT USAGE ON *.* TO 'mysqltest_u1'@'%' -drop database mysqltest_db1; -drop user mysqltest_u1; diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 5bf56e213ab..b3007408368 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -799,3 +799,32 @@ a b c 9 d d DROP TABLE t1; set optimizer_switch= @optimizer_switch_save; +# +# MDEV-10927: Crash When Using sort_union Optimization +# +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +sum(col1) +33632261 +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/r/information_schema_part.result b/mysql-test/r/information_schema_part.result index b34183ebdee..77959de256e 100644 --- a/mysql-test/r/information_schema_part.result +++ b/mysql-test/r/information_schema_part.result @@ -151,3 +151,11 @@ select create_options from information_schema.tables where table_schema="test"; create_options partitioned drop table t1; +# +# MDEV-11353 - Identical logical conditions +# +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +CHECKSUM +3036305396 +DROP TABLE t1; diff --git a/mysql-test/r/innodb_icp.result b/mysql-test/r/innodb_icp.result index 33a8027adf1..fb6cdefa14f 100644 --- a/mysql-test/r/innodb_icp.result +++ b/mysql-test/r/innodb_icp.result @@ -432,7 +432,7 @@ WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10 id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 Using where 2 DEPENDENT SUBQUERY it eq_ref PRIMARY PRIMARY 4 func 1 Using where -2 DEPENDENT SUBQUERY t2 index NULL PRIMARY 4 NULL 3 Using where; Using index; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t2 index NULL PRIMARY 4 NULL 3 Using index; Using join buffer (flat, BNL join) SELECT * FROM t1 WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10); pk i diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result index c1dbef08c04..5c4875546e6 100644 --- a/mysql-test/r/join_cache.result +++ b/mysql-test/r/join_cache.result @@ -5701,11 +5701,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 set join_cache_level=2; set @init_time:=now(); @@ -5737,11 +5739,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 EXPLAIN SELECT t.* @@ -5772,6 +5776,8 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id select_type table type possible_keys key key_len ref rows Extra @@ -5801,6 +5807,8 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE c23 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c24 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c25 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c26 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c27 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) set join_buffer_size=default; set join_cache_level = default; DROP TABLE t1,t2; diff --git a/mysql-test/r/join_nested.result b/mysql-test/r/join_nested.result index 84b6ff640e9..6ddd39cbfec 100644 --- a/mysql-test/r/join_nested.result +++ b/mysql-test/r/join_nested.result @@ -1870,4 +1870,99 @@ f4 NULL NULL DROP TABLE t1,t2,t3,t4,t5; +# +# MDEV-7992: Nested left joins + 'not exists' optimization +# +CREATE TABLE t1( +K1 INT PRIMARY KEY, +Name VARCHAR(15) +); +INSERT INTO t1 VALUES +(1,'T1Row1'), (2,'T1Row2'); +CREATE TABLE t2( +K2 INT PRIMARY KEY, +K1r INT, +rowTimestamp DATETIME, +Event VARCHAR(15) +); +INSERT INTO t2 VALUES +(1, 1, '2015-04-13 10:42:11' ,'T1Row1Event1'), +(2, 1, '2015-04-13 10:42:12' ,'T1Row1Event2'), +(3, 1, '2015-04-13 10:42:12' ,'T1Row1Event3'); +SELECT t1a.*, t2a.*, +t2i.K2 AS K2B, t2i.K1r AS K1rB, +t2i.rowTimestamp AS rowTimestampB, t2i.Event AS EventB +FROM +t1 t1a JOIN t2 t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +( t1 t1i LEFT JOIN t2 t2i ON t2i.K1r = t1i.K1) +ON (t1i.K1 = 1) AND +(((t2i.K1r = t1a.K1 AND t2i.rowTimestamp > t2a.rowTimestamp ) OR +(t2i.rowTimestamp = t2a.rowTimestamp AND t2i.K2 > t2a.K2)) +OR (t2i.K2 IS NULL)) +WHERE +t2a.K1r = 1 AND t2i.K2 IS NULL; +K1 Name K2 K1r rowTimestamp Event K2B K1rB rowTimestampB EventB +1 T1Row1 3 1 2015-04-13 10:42:12 T1Row1Event3 NULL NULL NULL NULL +EXPLAIN EXTENDED SELECT t1a.*, t2a.*, +t2i.K2 AS K2B, t2i.K1r AS K1rB, +t2i.rowTimestamp AS rowTimestampB, t2i.Event AS EventB +FROM +t1 t1a JOIN t2 t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +( t1 t1i LEFT JOIN t2 t2i ON t2i.K1r = t1i.K1) +ON (t1i.K1 = 1) AND +(((t2i.K1r = t1a.K1 AND t2i.rowTimestamp > t2a.rowTimestamp ) OR +(t2i.rowTimestamp = t2a.rowTimestamp AND t2i.K2 > t2a.K2)) +OR (t2i.K2 IS NULL)) +WHERE +t2a.K1r = 1 AND t2i.K2 IS NULL; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1a const PRIMARY PRIMARY 4 const 1 100.00 +1 SIMPLE t2a ALL NULL NULL NULL NULL 3 100.00 Using where +1 SIMPLE t1i const PRIMARY PRIMARY 4 const 1 100.00 Using index +1 SIMPLE t2i ALL NULL NULL NULL NULL 3 100.00 Using where; Not exists +Warnings: +Note 1003 select 1 AS `K1`,'T1Row1' AS `Name`,`test`.`t2a`.`K2` AS `K2`,`test`.`t2a`.`K1r` AS `K1r`,`test`.`t2a`.`rowTimestamp` AS `rowTimestamp`,`test`.`t2a`.`Event` AS `Event`,`test`.`t2i`.`K2` AS `K2B`,`test`.`t2i`.`K1r` AS `K1rB`,`test`.`t2i`.`rowTimestamp` AS `rowTimestampB`,`test`.`t2i`.`Event` AS `EventB` from `test`.`t1` `t1a` join `test`.`t2` `t2a` left join (`test`.`t1` `t1i` left join `test`.`t2` `t2i` on((`test`.`t2i`.`K1r` = 1))) on(((`test`.`t1i`.`K1` = 1) and (((`test`.`t2i`.`K1r` = 1) and (`test`.`t2i`.`rowTimestamp` > `test`.`t2a`.`rowTimestamp`)) or ((`test`.`t2i`.`rowTimestamp` = `test`.`t2a`.`rowTimestamp`) and (`test`.`t2i`.`K2` > `test`.`t2a`.`K2`)) or isnull(`test`.`t2i`.`K2`)))) where ((`test`.`t2a`.`K1r` = 1) and isnull(`test`.`t2i`.`K2`)) +CREATE VIEW v1 AS +SELECT t2i.* +FROM t1 as t1i LEFT JOIN t2 as t2i ON t2i.K1r = t1i.K1 +WHERE t1i.K1 = 1 ; +SELECT +t1a.*, t2a.*, t2b.K2 as K2B, t2b.K1r as K1rB, +t2b.rowTimestamp as rowTimestampB, t2b.Event as EventB +FROM +t1 as t1a JOIN t2 as t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +v1 as t2b +ON ((t2b.K1r = t1a.K1 AND t2b.rowTimestamp > t2a.rowTimestamp) OR +(t2b.rowTimestamp = t2a.rowTimestamp AND t2b.K2 > t2a.K2)) +OR (t2b.K2 IS NULL) +WHERE +t1a.K1 = 1 AND +t2b.K2 IS NULL; +K1 Name K2 K1r rowTimestamp Event K2B K1rB rowTimestampB EventB +1 T1Row1 3 1 2015-04-13 10:42:12 T1Row1Event3 NULL NULL NULL NULL +EXPLAIN EXTENDED SELECT +t1a.*, t2a.*, t2b.K2 as K2B, t2b.K1r as K1rB, +t2b.rowTimestamp as rowTimestampB, t2b.Event as EventB +FROM +t1 as t1a JOIN t2 as t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +v1 as t2b +ON ((t2b.K1r = t1a.K1 AND t2b.rowTimestamp > t2a.rowTimestamp) OR +(t2b.rowTimestamp = t2a.rowTimestamp AND t2b.K2 > t2a.K2)) +OR (t2b.K2 IS NULL) +WHERE +t1a.K1 = 1 AND +t2b.K2 IS NULL; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1a const PRIMARY PRIMARY 4 const 1 100.00 +1 SIMPLE t2a ALL NULL NULL NULL NULL 3 100.00 Using where +1 SIMPLE t1i const PRIMARY PRIMARY 4 const 1 100.00 Using index +1 SIMPLE t2i ALL NULL NULL NULL NULL 3 100.00 Using where; Not exists +Warnings: +Note 1003 select 1 AS `K1`,'T1Row1' AS `Name`,`t2a`.`K2` AS `K2`,`t2a`.`K1r` AS `K1r`,`t2a`.`rowTimestamp` AS `rowTimestamp`,`t2a`.`Event` AS `Event`,`test`.`t2i`.`K2` AS `K2B`,`test`.`t2i`.`K1r` AS `K1rB`,`test`.`t2i`.`rowTimestamp` AS `rowTimestampB`,`test`.`t2i`.`Event` AS `EventB` from `test`.`t1` `t1a` join `test`.`t2` `t2a` left join (`test`.`t1` `t1i` left join `test`.`t2` `t2i` on((`test`.`t2i`.`K1r` = 1))) on(((`test`.`t1i`.`K1` = 1) and (((`test`.`t2i`.`K1r` = 1) and (`test`.`t2i`.`rowTimestamp` > `t2a`.`rowTimestamp`)) or ((`test`.`t2i`.`rowTimestamp` = `t2a`.`rowTimestamp`) and (`test`.`t2i`.`K2` > `t2a`.`K2`)) or isnull(`test`.`t2i`.`K2`)))) where ((`t2a`.`K1r` = 1) and isnull(`test`.`t2i`.`K2`)) +DROP VIEW v1; +DROP TABLE t1,t2; set optimizer_search_depth= @tmp_mdev621; diff --git a/mysql-test/r/join_nested_jcl6.result b/mysql-test/r/join_nested_jcl6.result index 3b47645ca79..bac8e1cb7db 100644 --- a/mysql-test/r/join_nested_jcl6.result +++ b/mysql-test/r/join_nested_jcl6.result @@ -1881,6 +1881,101 @@ f4 NULL NULL DROP TABLE t1,t2,t3,t4,t5; +# +# MDEV-7992: Nested left joins + 'not exists' optimization +# +CREATE TABLE t1( +K1 INT PRIMARY KEY, +Name VARCHAR(15) +); +INSERT INTO t1 VALUES +(1,'T1Row1'), (2,'T1Row2'); +CREATE TABLE t2( +K2 INT PRIMARY KEY, +K1r INT, +rowTimestamp DATETIME, +Event VARCHAR(15) +); +INSERT INTO t2 VALUES +(1, 1, '2015-04-13 10:42:11' ,'T1Row1Event1'), +(2, 1, '2015-04-13 10:42:12' ,'T1Row1Event2'), +(3, 1, '2015-04-13 10:42:12' ,'T1Row1Event3'); +SELECT t1a.*, t2a.*, +t2i.K2 AS K2B, t2i.K1r AS K1rB, +t2i.rowTimestamp AS rowTimestampB, t2i.Event AS EventB +FROM +t1 t1a JOIN t2 t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +( t1 t1i LEFT JOIN t2 t2i ON t2i.K1r = t1i.K1) +ON (t1i.K1 = 1) AND +(((t2i.K1r = t1a.K1 AND t2i.rowTimestamp > t2a.rowTimestamp ) OR +(t2i.rowTimestamp = t2a.rowTimestamp AND t2i.K2 > t2a.K2)) +OR (t2i.K2 IS NULL)) +WHERE +t2a.K1r = 1 AND t2i.K2 IS NULL; +K1 Name K2 K1r rowTimestamp Event K2B K1rB rowTimestampB EventB +1 T1Row1 3 1 2015-04-13 10:42:12 T1Row1Event3 NULL NULL NULL NULL +EXPLAIN EXTENDED SELECT t1a.*, t2a.*, +t2i.K2 AS K2B, t2i.K1r AS K1rB, +t2i.rowTimestamp AS rowTimestampB, t2i.Event AS EventB +FROM +t1 t1a JOIN t2 t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +( t1 t1i LEFT JOIN t2 t2i ON t2i.K1r = t1i.K1) +ON (t1i.K1 = 1) AND +(((t2i.K1r = t1a.K1 AND t2i.rowTimestamp > t2a.rowTimestamp ) OR +(t2i.rowTimestamp = t2a.rowTimestamp AND t2i.K2 > t2a.K2)) +OR (t2i.K2 IS NULL)) +WHERE +t2a.K1r = 1 AND t2i.K2 IS NULL; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1a const PRIMARY PRIMARY 4 const 1 100.00 +1 SIMPLE t2a ALL NULL NULL NULL NULL 3 100.00 Using where +1 SIMPLE t1i const PRIMARY PRIMARY 4 const 1 100.00 Using index +1 SIMPLE t2i ALL NULL NULL NULL NULL 3 100.00 Using where; Not exists +Warnings: +Note 1003 select 1 AS `K1`,'T1Row1' AS `Name`,`test`.`t2a`.`K2` AS `K2`,`test`.`t2a`.`K1r` AS `K1r`,`test`.`t2a`.`rowTimestamp` AS `rowTimestamp`,`test`.`t2a`.`Event` AS `Event`,`test`.`t2i`.`K2` AS `K2B`,`test`.`t2i`.`K1r` AS `K1rB`,`test`.`t2i`.`rowTimestamp` AS `rowTimestampB`,`test`.`t2i`.`Event` AS `EventB` from `test`.`t1` `t1a` join `test`.`t2` `t2a` left join (`test`.`t1` `t1i` left join `test`.`t2` `t2i` on((`test`.`t2i`.`K1r` = 1))) on(((`test`.`t1i`.`K1` = 1) and (((`test`.`t2i`.`K1r` = 1) and (`test`.`t2i`.`rowTimestamp` > `test`.`t2a`.`rowTimestamp`)) or ((`test`.`t2i`.`rowTimestamp` = `test`.`t2a`.`rowTimestamp`) and (`test`.`t2i`.`K2` > `test`.`t2a`.`K2`)) or isnull(`test`.`t2i`.`K2`)))) where ((`test`.`t2a`.`K1r` = 1) and isnull(`test`.`t2i`.`K2`)) +CREATE VIEW v1 AS +SELECT t2i.* +FROM t1 as t1i LEFT JOIN t2 as t2i ON t2i.K1r = t1i.K1 +WHERE t1i.K1 = 1 ; +SELECT +t1a.*, t2a.*, t2b.K2 as K2B, t2b.K1r as K1rB, +t2b.rowTimestamp as rowTimestampB, t2b.Event as EventB +FROM +t1 as t1a JOIN t2 as t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +v1 as t2b +ON ((t2b.K1r = t1a.K1 AND t2b.rowTimestamp > t2a.rowTimestamp) OR +(t2b.rowTimestamp = t2a.rowTimestamp AND t2b.K2 > t2a.K2)) +OR (t2b.K2 IS NULL) +WHERE +t1a.K1 = 1 AND +t2b.K2 IS NULL; +K1 Name K2 K1r rowTimestamp Event K2B K1rB rowTimestampB EventB +1 T1Row1 3 1 2015-04-13 10:42:12 T1Row1Event3 NULL NULL NULL NULL +EXPLAIN EXTENDED SELECT +t1a.*, t2a.*, t2b.K2 as K2B, t2b.K1r as K1rB, +t2b.rowTimestamp as rowTimestampB, t2b.Event as EventB +FROM +t1 as t1a JOIN t2 as t2a ON t2a.K1r = t1a.K1 +LEFT JOIN +v1 as t2b +ON ((t2b.K1r = t1a.K1 AND t2b.rowTimestamp > t2a.rowTimestamp) OR +(t2b.rowTimestamp = t2a.rowTimestamp AND t2b.K2 > t2a.K2)) +OR (t2b.K2 IS NULL) +WHERE +t1a.K1 = 1 AND +t2b.K2 IS NULL; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1a const PRIMARY PRIMARY 4 const 1 100.00 +1 SIMPLE t2a ALL NULL NULL NULL NULL 3 100.00 Using where +1 SIMPLE t1i const PRIMARY PRIMARY 4 const 1 100.00 Using index +1 SIMPLE t2i ALL NULL NULL NULL NULL 3 100.00 Using where; Not exists +Warnings: +Note 1003 select 1 AS `K1`,'T1Row1' AS `Name`,`t2a`.`K2` AS `K2`,`t2a`.`K1r` AS `K1r`,`t2a`.`rowTimestamp` AS `rowTimestamp`,`t2a`.`Event` AS `Event`,`test`.`t2i`.`K2` AS `K2B`,`test`.`t2i`.`K1r` AS `K1rB`,`test`.`t2i`.`rowTimestamp` AS `rowTimestampB`,`test`.`t2i`.`Event` AS `EventB` from `test`.`t1` `t1a` join `test`.`t2` `t2a` left join (`test`.`t1` `t1i` left join `test`.`t2` `t2i` on((`test`.`t2i`.`K1r` = 1))) on(((`test`.`t1i`.`K1` = 1) and (((`test`.`t2i`.`K1r` = 1) and (`test`.`t2i`.`rowTimestamp` > `t2a`.`rowTimestamp`)) or ((`test`.`t2i`.`rowTimestamp` = `t2a`.`rowTimestamp`) and (`test`.`t2i`.`K2` > `t2a`.`K2`)) or isnull(`test`.`t2i`.`K2`)))) where ((`t2a`.`K1r` = 1) and isnull(`test`.`t2i`.`K2`)) +DROP VIEW v1; +DROP TABLE t1,t2; set optimizer_search_depth= @tmp_mdev621; CREATE TABLE t5 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b)); CREATE TABLE t6 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b)); diff --git a/mysql-test/r/join_outer.result b/mysql-test/r/join_outer.result index ac838997e41..4ea6f4ba192 100644 --- a/mysql-test/r/join_outer.result +++ b/mysql-test/r/join_outer.result @@ -1745,7 +1745,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 index PRIMARY PRIMARY 4 NULL 2 100.00 Using where; Using index; Using temporary; Using filesort 1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 func 1 100.00 Using where; Using index 2 DEPENDENT SUBQUERY t3 eq_ref PRIMARY PRIMARY 4 func 1 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t3.pk 1 100.00 Using where; Using index +2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t3.pk 1 100.00 Using index Warnings: Note 1276 Field or reference 'test.t2.pk' of SELECT #2 was resolved in SELECT #1 Note 1003 select `test`.`t2`.`pk` AS `pk`,<expr_cache><`test`.`t2`.`pk`>((select (`test`.`t3`.`pk` + if(isnull(`test`.`t4`.`pk`),0,`test`.`t4`.`pk`)) from `test`.`t3` left join `test`.`t4` on((`test`.`t4`.`pk` = `test`.`t3`.`pk`)) where (`test`.`t3`.`pk` = (`test`.`t2`.`pk` + 1000)) limit 1)) AS `t` from `test`.`t1` join `test`.`t2` where ((`test`.`t2`.`pk` = (`test`.`t1`.`pk` + 1000)) and (`test`.`t1`.`pk` > 1000)) group by `test`.`t2`.`pk` diff --git a/mysql-test/r/join_outer_jcl6.result b/mysql-test/r/join_outer_jcl6.result index 38518c45eae..c8305ccfd2d 100644 --- a/mysql-test/r/join_outer_jcl6.result +++ b/mysql-test/r/join_outer_jcl6.result @@ -1756,7 +1756,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t1 index PRIMARY PRIMARY 4 NULL 2 100.00 Using where; Using index; Using temporary; Using filesort 1 PRIMARY t2 eq_ref PRIMARY PRIMARY 4 func 1 100.00 Using where; Using index 2 DEPENDENT SUBQUERY t3 eq_ref PRIMARY PRIMARY 4 func 1 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t3.pk 1 100.00 Using where; Using index +2 DEPENDENT SUBQUERY t4 eq_ref PRIMARY PRIMARY 4 test.t3.pk 1 100.00 Using index Warnings: Note 1276 Field or reference 'test.t2.pk' of SELECT #2 was resolved in SELECT #1 Note 1003 select `test`.`t2`.`pk` AS `pk`,<expr_cache><`test`.`t2`.`pk`>((select (`test`.`t3`.`pk` + if(isnull(`test`.`t4`.`pk`),0,`test`.`t4`.`pk`)) from `test`.`t3` left join `test`.`t4` on((`test`.`t4`.`pk` = `test`.`t3`.`pk`)) where (`test`.`t3`.`pk` = (`test`.`t2`.`pk` + 1000)) limit 1)) AS `t` from `test`.`t1` join `test`.`t2` where ((`test`.`t2`.`pk` = (`test`.`t1`.`pk` + 1000)) and (`test`.`t1`.`pk` > 1000)) group by `test`.`t2`.`pk` diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index 12462305dc8..f446a503317 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -316,7 +316,7 @@ FIELDS ESCAPED BY '\\' TERMINATED BY ',' ENCLOSED BY '"' LINES TERMINATED BY '\n' (c0, c2); -ERROR HY000: Invalid column reference (v2.c0) in LOAD DATA +ERROR HY000: Column 'c0' is not updatable LOAD DATA INFILE '../../std_data/bug35469.dat' INTO TABLE v3 FIELDS ESCAPED BY '\\' @@ -507,7 +507,7 @@ DROP TABLE t1; # Bug#11765139 58069: LOAD DATA INFILE: VALGRIND REPORTS INVALID MEMORY READS AND WRITES WITH U # CREATE TABLE t1(f1 INT); -SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; DROP TABLE t1; # @@ -532,28 +532,21 @@ FIELDS TERMINATED BY 't' LINES TERMINATED BY ''; Got one of the listed errors SET @@sql_mode= @old_mode; DROP TABLE t1; - # -# Bug#23080148 - Backport of Bug#20683959. -# Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY -# UNDER DB CHARSET IS UTF8. +# MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -COUNT(*) -1 -SELECT HEX(val) FROM t1; -HEX(val) -C38322525420406E696F757A656368756E3A20E98198E2889AF58081AEE7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE78999E880B3E7B8BAEFBDAAE7B9A7E89699E296A1E7B8BAE4BBA3EFBD8CE7B8BAEFBDA9E7B8B2E2889AE38184E7B99DEFBDB3E7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE5B3A8EFBD84E8ABA0EFBDA8E89C89F580948EE599AAE7B8BAEFBDAAE7B8BAE9A198EFBDA9EFBDB1E7B9A7E581B5E289A0E7B8BAEFBDBEE7B9A7E9A194EFBDA9E882B4EFBDA5EFBDB5E980A7F5808B96E28693E99EABE38287E58F99E7B8BAE58AB1E28691E7B8BAF5808B9AE7828AE98095EFBDB1E7B8BAEFBDAFE7B8B2E288ABE6A89FE89EB3E6BA98F58081ADE88EA0EFBDBAE98095E6BA98F58081AEE89D93EFBDBAE8AD9BEFBDACE980A7F5808B96E28693E7B8BAF580918EE288AAE7B8BAE4B88AEFBC9EE7B8BAE4B99DE28691E7B8BAF5808B96EFBCA0E88DB3E6A68AEFBDB9EFBDB3E981B2E5B3A8E296A1E7B8BAE7A4BCE7828AE88DB3E6A68AEFBDB0EFBDBDE7B8BAA0E7B8BAE88B93EFBDBEE5B899EFBC9E -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -Warnings: -Warning 1366 Incorrect string value: '\xF5\x80\x81\xAE\xE7\xB9...' for column 'val' at row 1 -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +HEX(a) +25AAABAC +DROP TABLE t1; +# +# MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +# +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +HEX(a) +C3A4 +DROP TABLE t1; diff --git a/mysql-test/r/log_slow.result b/mysql-test/r/log_slow.result index ca8b9e7c439..4f5eb9359f6 100644 --- a/mysql-test/r/log_slow.result +++ b/mysql-test/r/log_slow.result @@ -67,9 +67,9 @@ sleep(0.5) select count(*) FROM mysql.slow_log; count(*) 1 -truncate mysql.slow_log; set @@long_query_time=default; set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/r/log_tables-big.result b/mysql-test/r/log_tables-big.result index 9b81127c825..1e189a7726f 100644 --- a/mysql-test/r/log_tables-big.result +++ b/mysql-test/r/log_tables-big.result @@ -1,29 +1,31 @@ +set @@global.log_output = 'TABLE'; set session long_query_time=10; select get_lock('bug27638', 1); get_lock('bug27638', 1) 1 set session long_query_time=1; -truncate table mysql.slow_log; select get_lock('bug27638', 2); get_lock('bug27638', 2) 0 -select if (query_time between '00:00:01' and '00:00:10', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log; +select if (query_time >= '00:00:01', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log +where sql_text = 'select get_lock(\'bug27638\', 2)'; qt sql_text OK select get_lock('bug27638', 2) -truncate table mysql.slow_log; select get_lock('bug27638', 60); get_lock('bug27638', 60) 0 -select if (query_time between '00:00:59' and '00:01:10', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log; +select if (query_time >= '00:00:59', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log +where sql_text = 'select get_lock(\'bug27638\', 60)'; qt sql_text OK select get_lock('bug27638', 60) -truncate table mysql.slow_log; select get_lock('bug27638', 101); get_lock('bug27638', 101) 0 -select if (query_time between '00:01:40' and '00:01:50', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log; +select if (query_time >= '00:01:40', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log +where sql_text = 'select get_lock(\'bug27638\', 101)'; qt sql_text OK select get_lock('bug27638', 101) select release_lock('bug27638'); release_lock('bug27638') 1 +set @@global.log_output=default; diff --git a/mysql-test/r/myisam_debug.result b/mysql-test/r/myisam_debug.result index 36a8fe6c724..77de991bafd 100644 --- a/mysql-test/r/myisam_debug.result +++ b/mysql-test/r/myisam_debug.result @@ -27,3 +27,15 @@ CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK DROP TABLE t1,t2; +call mtr.add_suppression('Incorrect key file for table'); +create table t1 (a int, index(a)); +lock tables t1 write; +insert t1 values (1),(2),(1); +set @old_dbug=@@debug_dbug; +set debug_dbug='+d,mi_lock_database_failure'; +unlock tables; +Warnings: +Error 126 Incorrect key file for table './test/t1.MYI'; try to repair it +Error 1015 Can't lock file (errno: 22 "Invalid argument") +set debug_dbug=@old_dbug; +drop table t1; diff --git a/mysql-test/r/myisam_icp.result b/mysql-test/r/myisam_icp.result index 95befeb5a42..ea57146dad2 100644 --- a/mysql-test/r/myisam_icp.result +++ b/mysql-test/r/myisam_icp.result @@ -436,7 +436,7 @@ WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10 id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 Using where 2 DEPENDENT SUBQUERY it eq_ref PRIMARY PRIMARY 4 func 1 Using index condition -2 DEPENDENT SUBQUERY t2 index NULL PRIMARY 4 NULL 3 Using where; Using index; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t2 index NULL PRIMARY 4 NULL 3 Using index; Using join buffer (flat, BNL join) SELECT * FROM t1 WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10); pk i @@ -800,7 +800,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t ALL PRIMARY,c NULL NULL NULL 64 Using where 1 PRIMARY t2 ref g g 5 test.t.c 19 Using where 2 DEPENDENT SUBQUERY t1 index PRIMARY PRIMARY 4 NULL 64 Using where; Using index -2 DEPENDENT SUBQUERY t2 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 Using index condition; Using where +2 DEPENDENT SUBQUERY t2 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 Using where SELECT COUNT(*) FROM t1 AS t, t2 WHERE c = g AND (EXISTS (SELECT * FROM t1, t2 WHERE a = f AND h <= t.e AND a > t.b) diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index cb3c28f42cd..2d26bc774cb 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -5310,6 +5310,69 @@ a DROP TABLE t1; DROP TABLE t2; DROP DATABASE db_20772273; +# +# Bug #25717383: MYSQLDUMP MAY EXECUTE ANY ARBITRARY QUERY +# +CREATE DATABASE bug25717383; +use bug25717383; +CREATE TABLE `tab +one` (a int); +CREATE VIEW `view +one` as SELECT * FROM `tab +one`; +CREATE PROCEDURE `proc +one`() SELECT * from `tab +one`; +CREATE TEMPORARY TABLE `temp +one` (id INT); +CREATE TRIGGER `trig +one` BEFORE INSERT ON `tab +one` FOR EACH ROW SET NEW.a = 1; +CREATE EVENT `event +one` ON SCHEDULE AT '2030-01-01 00:00:00' DO SET @a=5; +SHOW TABLES FROM bug25717383; +Tables_in_bug25717383 +tab +one +view +one +SHOW TRIGGERS FROM bug25717383; +Trigger Event Table Statement Timing Created sql_mode Definer character_set_client collation_connection Database Collation +trig +one INSERT tab +one SET NEW.a = 1 BEFORE NULL root@localhost utf8 utf8_general_ci latin1_swedish_ci +SHOW EVENTS FROM bug25717383; +Db Name Definer Time zone Type Execute at Interval value Interval field Starts Ends Status Originator character_set_client collation_connection Database Collation +bug25717383 event +one root@localhost SYSTEM ONE TIME # NULL NULL NULL NULL ENABLED 1 utf8 utf8_general_ci latin1_swedish_ci +SELECT ROUTINE_NAME FROM INFORMATION_SCHEMA.ROUTINES +WHERE ROUTINE_SCHEMA='bug25717383' AND ROUTINE_TYPE= 'PROCEDURE' + ORDER BY ROUTINE_NAME; +ROUTINE_NAME +proc +one +SHOW TABLES FROM bug25717383; +Tables_in_bug25717383 +tab +one +view +one +SHOW TRIGGERS FROM bug25717383; +Trigger Event Table Statement Timing Created sql_mode Definer character_set_client collation_connection Database Collation +trig +one INSERT tab +one SET NEW.a = 1 BEFORE NULL root@localhost utf8 utf8_general_ci latin1_swedish_ci +SHOW EVENTS FROM bug25717383; +Db Name Definer Time zone Type Execute at Interval value Interval field Starts Ends Status Originator character_set_client collation_connection Database Collation +bug25717383 event +one root@localhost SYSTEM ONE TIME # NULL NULL NULL NULL ENABLED 1 utf8 utf8_general_ci latin1_swedish_ci +SELECT ROUTINE_NAME FROM INFORMATION_SCHEMA.ROUTINES +WHERE ROUTINE_SCHEMA='bug25717383' AND ROUTINE_TYPE= 'PROCEDURE' + ORDER BY ROUTINE_NAME; +ROUTINE_NAME +proc +one +DROP DATABASE bug25717383; Usage: mysqldump [OPTIONS] database [tables] OR mysqldump [OPTIONS] --databases [OPTIONS] DB1 [DB2 DB3...] OR mysqldump [OPTIONS] --all-databases [OPTIONS] @@ -5340,3 +5403,4 @@ DELIMITER ; /*!50003 SET collation_connection = @saved_col_connection */ ; ALTER DATABASE `a\"'``b` CHARACTER SET utf8 COLLATE utf8_general_ci ; DROP DATABASE `a\"'``b`; +FOUND /Database: mysql/ in bug11505.sql diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result index 9c63570561d..07f278a6563 100644 --- a/mysql-test/r/order_by.result +++ b/mysql-test/r/order_by.result @@ -2937,6 +2937,17 @@ t2 A, t2 B where A.b = B.b order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +# +# mdev-10705 : long order by list that can be skipped +# +SELECT 1 +UNION +( SELECT 2 +ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, +@a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); +1 +1 +2 End of 5.5 tests # # MDEV-5884: EXPLAIN UPDATE ... ORDER BY LIMIT shows wrong #rows diff --git a/mysql-test/r/partition_innodb.result b/mysql-test/r/partition_innodb.result index e1bc075f14a..f863ec5522a 100644 --- a/mysql-test/r/partition_innodb.result +++ b/mysql-test/r/partition_innodb.result @@ -732,6 +732,97 @@ SELECT * FROM t1 WHERE d = '1991-01-01'; d 1991-01-01 DROP TABLE t1; +set global default_storage_engine=default; +# +# MDEV-9455: [ERROR] mysqld got signal 11 +# +CREATE TABLE `t1` ( +`DIARY_TOTAL_DAY_SEQ` bigint(20) unsigned NOT NULL AUTO_INCREMENT, +`IMORY_ID` bigint(20) NOT NULL, +`NAME` varchar(75) DEFAULT NULL, +`DATETIME` varchar(10) NOT NULL DEFAULT '', +`DAILY_CALL_CNT` int(11) DEFAULT NULL, +`DAILY_SMS_CNT` int(11) DEFAULT NULL, +`NUMBER` varchar(64) DEFAULT NULL, +`DURATION` varchar(16) DEFAULT NULL, +PRIMARY KEY (`DIARY_TOTAL_DAY_SEQ`,`DATETIME`), +KEY `IDX_t1_01` (`IMORY_ID`,`DATETIME`) +) AUTO_INCREMENT=328702514 DEFAULT CHARSET=utf8mb4 +PARTITION BY RANGE COLUMNS(`DATETIME`) +(PARTITION p0 VALUES LESS THAN ('2015-10-01') ENGINE = InnoDB, +PARTITION p1 VALUES LESS THAN ('2015-11-01') ENGINE = InnoDB, +PARTITION p2 VALUES LESS THAN ('2015-12-01') ENGINE = InnoDB, +PARTITION p3 VALUES LESS THAN ('2016-01-01') ENGINE = InnoDB, +PARTITION p4 VALUES LESS THAN ('2016-02-01') ENGINE = InnoDB, +PARTITION p5 VALUES LESS THAN ('2016-03-01') ENGINE = InnoDB, +PARTITION p6 VALUES LESS THAN ('2016-04-01') ENGINE = InnoDB, +PARTITION p7 VALUES LESS THAN ('2016-05-01') ENGINE = InnoDB, +PARTITION p8 VALUES LESS THAN ('2016-06-01') ENGINE = InnoDB, +PARTITION p9 VALUES LESS THAN ('2016-07-01') ENGINE = InnoDB, +PARTITION p10 VALUES LESS THAN ('2016-08-01') ENGINE = InnoDB) +; +CREATE TABLE `t2` ( +`DIARY_SEQ` bigint(20) unsigned NOT NULL AUTO_INCREMENT, +`IMORY_ID` bigint(20) NOT NULL, +`CALL_TYPE` varchar(1) DEFAULT NULL, +`DATA_TYPE` varchar(1) DEFAULT NULL, +`FEATURES` varchar(1) DEFAULT NULL, +`NAME` varchar(75) DEFAULT NULL, +`NUMBER` varchar(64) DEFAULT NULL, +`DATETIME` datetime NOT NULL, +`REG_DATE` datetime NOT NULL, +`TITLE` varchar(50) DEFAULT NULL, +`BODY` varchar(4200) DEFAULT NULL, +`MIME_TYPE` varchar(32) DEFAULT NULL, +`DURATION` varchar(16) DEFAULT NULL, +`DEVICE_ID` varchar(64) DEFAULT NULL, +`DEVICE_NAME` varchar(32) DEFAULT NULL, +PRIMARY KEY (`DIARY_SEQ`,`DATETIME`,`REG_DATE`), +KEY `IDX_TB_DIARY_01` (`IMORY_ID`,`DATETIME`,`CALL_TYPE`,`NUMBER`), +KEY `IDX_TB_DIARY_02` (`REG_DATE`) +) AUTO_INCREMENT=688799006 DEFAULT CHARSET=utf8mb4 +PARTITION BY RANGE COLUMNS(REG_DATE) +(PARTITION p0 VALUES LESS THAN ('2015-10-01') ENGINE = InnoDB, +PARTITION p1 VALUES LESS THAN ('2015-11-01') ENGINE = InnoDB, +PARTITION p2 VALUES LESS THAN ('2015-12-01') ENGINE = InnoDB, +PARTITION p3 VALUES LESS THAN ('2016-01-01') ENGINE = InnoDB, +PARTITION p4 VALUES LESS THAN ('2016-02-01') ENGINE = InnoDB, +PARTITION p5 VALUES LESS THAN ('2016-03-01') ENGINE = InnoDB, +PARTITION p6 VALUES LESS THAN ('2016-04-01') ENGINE = InnoDB, +PARTITION p7 VALUES LESS THAN ('2016-05-01') ENGINE = InnoDB, +PARTITION p8 VALUES LESS THAN ('2016-06-01') ENGINE = InnoDB, +PARTITION p9 VALUES LESS THAN ('2016-07-01') ENGINE = InnoDB, +PARTITION p10 VALUES LESS THAN ('2016-08-01') ENGINE = InnoDB) +; +SELECT +A.IMORY_ID, +A.NUMBER, +A.NAME, +DATE_FORMAT(A.DATETIME, '%Y-%m-%d') AS TARGET_DATE, +SUM( CASE WHEN A.DATA_TYPE='1' THEN 1 ELSE 0 END) AS CALL_CNT, +SUM( CASE WHEN A.DATA_TYPE IN ('2', '3') THEN 1 ELSE 0 END) AS SMS_CNT, +SUM(CAST(A.DURATION AS INT)) AS DURATION, +( SELECT COUNT(*) +FROM t1 +WHERE IMORY_ID=A.IMORY_ID +AND NUMBER=A.NUMBER +AND NAME=A.NAME +AND DATETIME = DATE_FORMAT(A.DATETIME, '%Y-%m-%d') +) STATS_COUNT +FROM t2 A +WHERE A.IMORY_ID = 55094102 +AND A.DATETIME LIKE ( +SELECT CONCAT (DATE_FORMAT(DATETIME, '%Y-%m-%d') ,'%') +FROM t2 +WHERE IMORY_ID=55094102 +AND DIARY_SEQ IN ( 608351221, 608351225, 608351229 ) +group by DATE_FORMAT(DATETIME, '%Y-%m-%d') +) +GROUP BY A.IMORY_ID, A.NUMBER, A.NAME, DATE_FORMAT(A.DATETIME, '%Y-%m-%d') +; +IMORY_ID NUMBER NAME TARGET_DATE CALL_CNT SMS_CNT DURATION STATS_COUNT +drop table t2, t1; +set global default_storage_engine='innodb'; # # MDEV-5963: InnoDB: Assertion failure in file row0sel.cc line 2503, # Failing assertion: 0 with "key ptr now exceeds key end by 762 bytes" diff --git a/mysql-test/r/partition_myisam.result b/mysql-test/r/partition_myisam.result index bb1a7b19a9d..08a5bf72f64 100644 --- a/mysql-test/r/partition_myisam.result +++ b/mysql-test/r/partition_myisam.result @@ -230,6 +230,22 @@ PARTITION pMax VALUES LESS THAN MAXVALUE); INSERT INTO t1 VALUES (1, "Partition p1, first row"); DROP TABLE t1; # +# MDEV-10418 Assertion `m_extra_cache' failed +# in ha_partition::late_extra_cache(uint) +# +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (f2 INT) ENGINE=MyISAM PARTITION BY RANGE(f2) (PARTITION pmax VALUES LESS THAN MAXVALUE); +INSERT INTO t2 VALUES (8); +CREATE ALGORITHM = MERGE VIEW v AS SELECT f2 FROM t2, t1; +UPDATE v SET f2 = 1; +SELECT * FROM t2; +f2 +1 +DROP VIEW v; +DROP TABLE t2; +DROP TABLE t1; +# # bug#11760213-52599: ALTER TABLE REMOVE PARTITIONING ON NON-PARTITIONED # TABLE CORRUPTS MYISAM DROP TABLE if exists `t1`; diff --git a/mysql-test/r/pool_of_threads.result b/mysql-test/r/pool_of_threads.result index 9611d7ff43b..ddc451516e2 100644 --- a/mysql-test/r/pool_of_threads.result +++ b/mysql-test/r/pool_of_threads.result @@ -2157,23 +2157,22 @@ Warnings: Warning 1052 Column 'kundentyp' in group statement is ambiguous drop table t1; SET optimizer_switch=@save_optimizer_switch; -SELECT sleep(5.5); -SELECT sleep(5); +SELECT sleep(50); +SELECT sleep(50); # -- Success: more than --thread_pool_max_threads normal connections not possible -sleep(5.5) -0 -sleep(5) -0 -SELECT sleep(5); -SELECT sleep(5); SELECT 'Connection on extra port ok'; Connection on extra port ok Connection on extra port ok +SELECT sleep(5.5); SELECT 'Connection on extra port 2 ok'; Connection on extra port 2 ok Connection on extra port 2 ok # -- Success: more than --extra-max-connections + 1 normal connections not possible -sleep(5) -0 -sleep(5) +KILL QUERY <default_connection_ID>; +KILL QUERY <con2_connection_ID>; +sleep(50) +1 +sleep(50) +1 +sleep(5.5) 0 diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index eb5c8ca9377..c8fa17a4e9e 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -4107,4 +4107,76 @@ NULL NULL deallocate prepare stmt; drop table t1,t2,t3,t4; +# +# MDEV-11859: the plans for the first and the second executions +# of PS are not the same +# +create table t1 (id int, c varchar(3), key idx(c))engine=myisam; +insert into t1 values (3,'bar'), (1,'xxx'), (2,'foo'), (5,'yyy'); +prepare stmt1 from +"explain extended + select * from t1 where (1, 2) in ( select 3, 4 ) or c = 'foo'"; +execute stmt1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ref idx idx 6 const 1 100.00 Using index condition +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select `test`.`t1`.`id` AS `id`,`test`.`t1`.`c` AS `c` from `test`.`t1` where (`test`.`t1`.`c` = 'foo') +execute stmt1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ref idx idx 6 const 1 100.00 Using index condition +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select `test`.`t1`.`id` AS `id`,`test`.`t1`.`c` AS `c` from `test`.`t1` where (`test`.`t1`.`c` = 'foo') +deallocate prepare stmt1; +prepare stmt1 from +"select * from t1 where (1, 2) in ( select 3, 4 ) or c = 'foo'"; +flush status; +execute stmt1; +id c +2 foo +show status like '%Handler_read%'; +Variable_name Value +Handler_read_first 0 +Handler_read_key 1 +Handler_read_last 0 +Handler_read_next 1 +Handler_read_prev 0 +Handler_read_retry 0 +Handler_read_rnd 0 +Handler_read_rnd_deleted 0 +Handler_read_rnd_next 0 +flush status; +execute stmt1; +id c +2 foo +show status like '%Handler_read%'; +Variable_name Value +Handler_read_first 0 +Handler_read_key 1 +Handler_read_last 0 +Handler_read_next 1 +Handler_read_prev 0 +Handler_read_retry 0 +Handler_read_rnd 0 +Handler_read_rnd_deleted 0 +Handler_read_rnd_next 0 +deallocate prepare stmt1; +prepare stmt2 from +"explain extended + select * from t1 where (1, 2) in ( select 3, 4 )"; +execute stmt2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select `test`.`t1`.`id` AS `id`,`test`.`t1`.`c` AS `c` from `test`.`t1` where 0 +execute stmt2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select `test`.`t1`.`id` AS `id`,`test`.`t1`.`c` AS `c` from `test`.`t1` where 0 +deallocate prepare stmt2; +drop table t1; # End of 5.5 tests diff --git a/mysql-test/r/range_vs_index_merge.result b/mysql-test/r/range_vs_index_merge.result index 9af359a55f3..6813c40a5cf 100644 --- a/mysql-test/r/range_vs_index_merge.result +++ b/mysql-test/r/range_vs_index_merge.result @@ -60,11 +60,11 @@ id select_type table type possible_keys key key_len ref rows Extra EXPLAIN SELECT * FROM City WHERE Population > 100000 AND Name LIKE 'Aba%' OR -Country IN ('CAN', 'ARG') AND ID < 3800 OR -Country < 'U' AND Name LIKE 'Zhu%' OR -ID BETWEEN 3800 AND 3810; +Country IN ('CAN', 'ARG') AND ID BETWEEN 120 AND 130 OR +Country <= 'ALB' AND Name LIKE 'L%' OR +ID BETWEEN 3807 AND 3810; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,Country,PRIMARY 35,3,4 NULL 132 Using sort_union(Name,Country,PRIMARY); Using where +1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,PRIMARY,Country 35,4,3 NULL 31 Using sort_union(Name,PRIMARY,Country); Using where EXPLAIN SELECT * FROM City WHERE (Population > 101000 AND Population < 115000); @@ -1769,4 +1769,42 @@ a b 167 9999 168 10000 DROP TABLE t1; +# +# MDEV-8603: Wrong result OR/AND condition over index fields +# +CREATE TABLE t1 ( +id INT NOT NULL, +state VARCHAR(64), +capital VARCHAR(64), +UNIQUE KEY (id), +KEY state (state,id), +KEY capital (capital, id) +); +INSERT INTO t1 VALUES +(1,'Arizona','Phoenix'), +(2,'Hawaii','Honolulu'), +(3,'Georgia','Atlanta'), +(4,'Florida','Tallahassee'), +(5,'Alaska','Juneau'), +(6,'Michigan','Lansing'), +(7,'Pennsylvania','Harrisburg'), +(8,'Virginia','Richmond') +; +EXPLAIN +SELECT * FROM t1 FORCE KEY (state,capital) +WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 +OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range state,capital state 71 NULL 12 Using index condition; Using where +SELECT * FROM t1 FORCE KEY (state,capital) +WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 +OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas'; +id state capital +4 Florida Tallahassee +3 Georgia Atlanta +2 Hawaii Honolulu +6 Michigan Lansing +7 Pennsylvania Harrisburg +8 Virginia Richmond +DROP TABLE t1; set session optimizer_switch='index_merge_sort_intersection=default'; diff --git a/mysql-test/r/range_vs_index_merge_innodb.result b/mysql-test/r/range_vs_index_merge_innodb.result index 601ae9b7613..13fbc0ac3ef 100644 --- a/mysql-test/r/range_vs_index_merge_innodb.result +++ b/mysql-test/r/range_vs_index_merge_innodb.result @@ -61,11 +61,11 @@ id select_type table type possible_keys key key_len ref rows Extra EXPLAIN SELECT * FROM City WHERE Population > 100000 AND Name LIKE 'Aba%' OR -Country IN ('CAN', 'ARG') AND ID < 3800 OR -Country < 'U' AND Name LIKE 'Zhu%' OR -ID BETWEEN 3800 AND 3810; +Country IN ('CAN', 'ARG') AND ID BETWEEN 120 AND 130 OR +Country <= 'ALB' AND Name LIKE 'L%' OR +ID BETWEEN 3807 AND 3810; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,Country,PRIMARY 35,7,4 NULL 123 Using sort_union(Name,Country,PRIMARY); Using where +1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,Country,PRIMARY 35,3,4 NULL 33 Using sort_union(Name,Country,PRIMARY); Using where EXPLAIN SELECT * FROM City WHERE (Population > 101000 AND Population < 115000); @@ -369,7 +369,7 @@ WHERE ((ID < 200) AND (Name LIKE 'Ha%' OR (Country > 'A' AND Country < 'ARG'))) OR ((ID BETWEEN 100 AND 200) AND (Name LIKE 'Pa%' OR (Population > 103000 AND Population < 104000))); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City range PRIMARY,Population,Country,Name PRIMARY 4 NULL 200 Using where +1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,Population,PRIMARY 39,4,4 NULL 305 Using sort_union(Name,Population,PRIMARY); Using where SELECT * FROM City USE INDEX () WHERE ((ID < 10) AND (Name LIKE 'H%' OR (Country > 'A' AND Country < 'ARG'))) OR ((ID BETWEEN 100 AND 110) AND @@ -1770,5 +1770,43 @@ a b 167 9999 168 10000 DROP TABLE t1; +# +# MDEV-8603: Wrong result OR/AND condition over index fields +# +CREATE TABLE t1 ( +id INT NOT NULL, +state VARCHAR(64), +capital VARCHAR(64), +UNIQUE KEY (id), +KEY state (state,id), +KEY capital (capital, id) +); +INSERT INTO t1 VALUES +(1,'Arizona','Phoenix'), +(2,'Hawaii','Honolulu'), +(3,'Georgia','Atlanta'), +(4,'Florida','Tallahassee'), +(5,'Alaska','Juneau'), +(6,'Michigan','Lansing'), +(7,'Pennsylvania','Harrisburg'), +(8,'Virginia','Richmond') +; +EXPLAIN +SELECT * FROM t1 FORCE KEY (state,capital) +WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 +OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range state,capital state 71 NULL 10 Using index condition; Using where +SELECT * FROM t1 FORCE KEY (state,capital) +WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 +OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas'; +id state capital +4 Florida Tallahassee +3 Georgia Atlanta +2 Hawaii Honolulu +6 Michigan Lansing +7 Pennsylvania Harrisburg +8 Virginia Richmond +DROP TABLE t1; set session optimizer_switch='index_merge_sort_intersection=default'; SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/mysql-test/r/sp-prelocking.result b/mysql-test/r/sp-prelocking.result index 5b594e9ea55..eb47cc21f41 100644 --- a/mysql-test/r/sp-prelocking.result +++ b/mysql-test/r/sp-prelocking.result @@ -340,3 +340,26 @@ f1() DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1,t2; +# +# Bug #16672723 "CAN'T FIND TEMPORARY TABLE". +# +CREATE FUNCTION f1() RETURNS INT RETURN 1; +CREATE TEMPORARY TABLE tmp1(a INT); +PREPARE stmt1 FROM "CREATE TEMPORARY TABLE tmp2 AS SELECT b FROM (SELECT f1() AS b FROM tmp1) AS t"; +# The below statement failed before the fix. +EXECUTE stmt1; +DROP TEMPORARY TABLES tmp1, tmp2; +DEALLOCATE PREPARE stmt1; +DROP FUNCTION f1; +create procedure sp1() +begin +drop table if exists t1, t2; +create temporary table t1 select 1 v; +create table t2 (col varchar(45)) select distinct col from (select sf1() as col from t1) t; +end$$ +create function sf1() returns text return 'blah'; +call test.sp1(); +call test.sp1(); +drop procedure sp1; +drop function sf1; +drop table t2; diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index c059428be16..2cb1b701e2d 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -7952,3 +7952,41 @@ set global table_open_cache= @tmp_toc; set global table_definition_cache= @tmp_tdc; drop procedure p1; drop table t1,t2,t3,t4,t5,t6; +# +# MDEV-11935: Queries in stored procedures with and +# EXISTS(SELECT * FROM VIEW) crashes and closes hte conneciton. +# +CREATE TABLE ANY_TABLE ( +ENTITY_UID BIGINT NOT NULL +); +CREATE TABLE SECURITY_PATH( +origid BIGINT UNSIGNED NOT NULL, +destid BIGINT UNSIGNED NOT NULL, +KEY (destid) +); +CREATE VIEW ENTITY_ACCESS ( +ENTITY_UID, +OWNER_UID +) AS +SELECT SP1.origid, +SP2.destid +FROM SECURITY_PATH SP1 +JOIN SECURITY_PATH SP2 ON SP1.destid = SP2.origid +; +CREATE PROCEDURE SP_EXAMPLE_SELECT () +BEGIN +SELECT * +FROM ANY_TABLE AT1 +WHERE EXISTS ( SELECT * +FROM ENTITY_ACCESS EA +WHERE AT1.ENTITY_UID = EA.ENTITY_UID +AND EA.OWNER_UID IS NULL ); +END +// +CALL SP_EXAMPLE_SELECT (); +ENTITY_UID +CALL SP_EXAMPLE_SELECT (); +ENTITY_UID +drop procedure SP_EXAMPLE_SELECT; +drop view ENTITY_ACCESS; +drop table ANY_TABLE, SECURITY_PATH; diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 30919592ba6..623c28ff7ca 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7136,3 +7136,14 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; diff --git a/mysql-test/r/subselect2.result b/mysql-test/r/subselect2.result index 034cba58c8f..401d333ccfb 100644 --- a/mysql-test/r/subselect2.result +++ b/mysql-test/r/subselect2.result @@ -348,4 +348,49 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); a a a FRA FRA FRA drop table t1,t2,t3; +# +# MDEV-10148: Database crashes in the query to the View +# +CREATE TABLE t1 ( +key_code INT(11) NOT NULL, +value_string VARCHAR(50) NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t2 ( +key_code INT(11) NOT NULL, +target_date DATE NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t3 ( +now_date DATE NOT NULL, +PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE VIEW v1 +AS +SELECT +B.key_code, +B.target_date +FROM +t2 B INNER JOIN t3 C ON +B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; +key_code value_string +EXECUTE stmt; +key_code value_string +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; set optimizer_switch=@subselect2_test_tmp; +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +select null in (select a from t1 where a < out3.a union select a from t2 where +(select a from t3) +1 < out3.a+1) from t3 out3; +ERROR 21000: Subquery returns more than 1 row +drop table t1, t2, t3; diff --git a/mysql-test/r/subselect3.result b/mysql-test/r/subselect3.result index 0c03959a96a..9d88d0da778 100644 --- a/mysql-test/r/subselect3.result +++ b/mysql-test/r/subselect3.result @@ -267,7 +267,7 @@ from t2; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 2 100.00 2 DEPENDENT SUBQUERY t1 ref_or_null a a 5 func 2 100.00 Using where; Full scan on NULL key -2 DEPENDENT SUBQUERY t4 ALL NULL NULL NULL NULL 100 100.00 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t4 ALL NULL NULL NULL NULL 100 100.00 Using join buffer (flat, BNL join) Warnings: Note 1276 Field or reference 'test.t2.oref' of SELECT #2 was resolved in SELECT #1 Note 1003 select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`oref` AS `oref`,<expr_cache><`test`.`t2`.`a`,`test`.`t2`.`b`,`test`.`t2`.`oref`>(<in_optimizer>((`test`.`t2`.`a`,`test`.`t2`.`b`),<exists>(select `test`.`t1`.`a`,`test`.`t1`.`b` from `test`.`t1` join `test`.`t4` where ((`test`.`t1`.`c` = `test`.`t2`.`oref`) and trigcond(trigcond(((<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`) or isnull(`test`.`t1`.`a`)))) and trigcond(trigcond(((<cache>(`test`.`t2`.`b`) = `test`.`t1`.`b`) or isnull(`test`.`t1`.`b`))))) having (trigcond(<is_not_null_test>(`test`.`t1`.`a`)) and trigcond(<is_not_null_test>(`test`.`t1`.`b`)))))) AS `Z` from `test`.`t2` @@ -1241,19 +1241,19 @@ explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 10 Using where 2 DEPENDENT SUBQUERY A ALL NULL NULL NULL NULL 10 Using where -2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 10 Using where 2 DEPENDENT SUBQUERY A ALL NULL NULL NULL NULL 10 Using where -2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) explain select straight_join * from t2 X, t2 Y where X.a in (select straight_join A.a from t1 A, t1 B); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY X ALL NULL NULL NULL NULL 10 Using where 1 PRIMARY Y ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) 2 DEPENDENT SUBQUERY A ALL NULL NULL NULL NULL 10 Using where -2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) create table t0 (a int, b int); insert into t0 values(1,1); explain select * from t0, t3 where t3.a in (select a from t2) and (t3.a < 10 or t3.a >30); diff --git a/mysql-test/r/subselect3_jcl6.result b/mysql-test/r/subselect3_jcl6.result index 415963af882..069b9a32ae5 100644 --- a/mysql-test/r/subselect3_jcl6.result +++ b/mysql-test/r/subselect3_jcl6.result @@ -277,7 +277,7 @@ from t2; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 2 100.00 2 DEPENDENT SUBQUERY t1 ref_or_null a a 5 func 2 100.00 Using where; Full scan on NULL key -2 DEPENDENT SUBQUERY t4 ALL NULL NULL NULL NULL 100 100.00 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t4 ALL NULL NULL NULL NULL 100 100.00 Using join buffer (flat, BNL join) Warnings: Note 1276 Field or reference 'test.t2.oref' of SELECT #2 was resolved in SELECT #1 Note 1003 select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`oref` AS `oref`,<expr_cache><`test`.`t2`.`a`,`test`.`t2`.`b`,`test`.`t2`.`oref`>(<in_optimizer>((`test`.`t2`.`a`,`test`.`t2`.`b`),<exists>(select `test`.`t1`.`a`,`test`.`t1`.`b` from `test`.`t1` join `test`.`t4` where ((`test`.`t1`.`c` = `test`.`t2`.`oref`) and trigcond(trigcond(((<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`) or isnull(`test`.`t1`.`a`)))) and trigcond(trigcond(((<cache>(`test`.`t2`.`b`) = `test`.`t1`.`b`) or isnull(`test`.`t1`.`b`))))) having (trigcond(<is_not_null_test>(`test`.`t1`.`a`)) and trigcond(<is_not_null_test>(`test`.`t1`.`b`)))))) AS `Z` from `test`.`t2` @@ -1251,19 +1251,19 @@ explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 10 Using where 2 DEPENDENT SUBQUERY A ALL NULL NULL NULL NULL 10 Using where -2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 10 Using where 2 DEPENDENT SUBQUERY A ALL NULL NULL NULL NULL 10 Using where -2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) explain select straight_join * from t2 X, t2 Y where X.a in (select straight_join A.a from t1 A, t1 B); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY X ALL NULL NULL NULL NULL 10 Using where 1 PRIMARY Y ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) 2 DEPENDENT SUBQUERY A ALL NULL NULL NULL NULL 10 Using where -2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY B ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) create table t0 (a int, b int); insert into t0 values(1,1); explain select * from t0, t3 where t3.a in (select a from t2) and (t3.a < 10 or t3.a >30); diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index a909230121d..2b9e952652e 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -19,7 +19,7 @@ SELECT 1 FROM t1 WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE 1 = (SELECT MIN(t2.b) FROM t3)) ORDER BY count(*); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t1 index NULL a 5 NULL 2 Using where; Using index; Using temporary +1 PRIMARY t1 index NULL a 5 NULL 2 Using index; Using temporary 2 DEPENDENT SUBQUERY t2 ALL NULL NULL NULL NULL 2 Using where 3 DEPENDENT SUBQUERY t3 system NULL NULL NULL NULL 0 const row not found # should not crash the next statement @@ -1405,7 +1405,7 @@ GROUP BY f9; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE 3 SUBQUERY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables -2 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 2 Using where +2 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 2 SELECT COUNT(t2.f3), (SELECT COUNT(f3) FROM t1 WHERE t2.f1) AS f9 FROM t2 JOIN t1 ON t1.f3 @@ -1421,7 +1421,7 @@ ORDER BY f9; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE 3 SUBQUERY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables -2 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 2 Using where +2 DEPENDENT SUBQUERY t1 ALL NULL NULL NULL NULL 2 SELECT COUNT(t2.f3), (SELECT COUNT(f3) FROM t1 WHERE t2.f1) AS f9 FROM t2 JOIN t1 ON t1.f3 @@ -2397,5 +2397,93 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; x 0 drop table t1; +# +# MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +# +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; +EXECUTE stmt; +b +EXECUTE stmt; +b +# Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; +EXECUTE stmt; +i +4 +6 +EXECUTE stmt; +i +4 +6 +drop table t1, t2, t3; +# +# MDEV-11078: NULL NOT IN (non-empty subquery) should never return results +# +create table t1(a int,b int); +create table t2(a int,b int); +insert into t1 value (1,2); +select (NULL) in (select 1 from t1); +(NULL) in (select 1 from t1) +NULL +select (null) in (select 1 from t2); +(null) in (select 1 from t2) +0 +select 1 in (select 1 from t1); +1 in (select 1 from t1) +1 +select 1 in (select 1 from t2); +1 in (select 1 from t2) +0 +select 1 from dual where null in (select 1 from t1); +1 +select 1 from dual where null in (select 1 from t2); +1 +select (null,null) in (select * from t1); +(null,null) in (select * from t1) +NULL +select (null,null) in (select * from t2); +(null,null) in (select * from t2) +0 +select 1 from dual where null not in (select 1 from t1); +1 +select 1 from dual where null not in (select 1 from t2); +1 +1 +drop table t1,t2; +# +# MDEV-6486: Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' +# failed with SELECT SQ, TEXT field +# +CREATE TABLE t1 (a VARCHAR(8), KEY(a)) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('foo'),( 'bar'); +CREATE TABLE t2 (b VARCHAR(8), c TINYTEXT, KEY(b)) ENGINE=MyISAM; +INSERT INTO t2 VALUES ('baz','baz'),('qux', 'qux'); +SELECT ( SELECT COUNT(*) FROM t1 WHERE a = c ) AS field, COUNT(DISTINCT c) +FROM t2 WHERE b <= 'quux' GROUP BY field; +field COUNT(DISTINCT c) +0 1 +drop table t1,t2; SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; diff --git a/mysql-test/r/subselect_exists2in.result b/mysql-test/r/subselect_exists2in.result index 1d0732060b7..5deb2dfa9c5 100644 --- a/mysql-test/r/subselect_exists2in.result +++ b/mysql-test/r/subselect_exists2in.result @@ -884,5 +884,55 @@ a deallocate prepare stmt; drop view v1; drop table t1,t2; +# +#MDEV-10053: EXIST to IN transformation turned down +# +CREATE TABLE t1 ( +pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) +ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7); +set @optimizer_switch_save=@@optimizer_switch; +set optimizer_switch='exists_to_in=off'; +explain extended SELECT STRAIGHT_JOIN sq1.f2 +FROM ( SELECT * FROM t1 ) AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 Using where +3 DEPENDENT SUBQUERY <subquery4> eq_ref distinct_key distinct_key 4 func 1 100.00 +3 DEPENDENT SUBQUERY sq2 ALL NULL NULL NULL NULL 2 100.00 Using where; Using join buffer (flat, BNL join) +4 MATERIALIZED t1 ALL NULL NULL NULL NULL 2 100.00 +Warnings: +Note 1276 Field or reference 'sq1.pk' of SELECT #3 was resolved in SELECT #1 +Note 1276 Field or reference 'sq1.f1' of SELECT #3 was resolved in SELECT #1 +Note 1003 select straight_join `test`.`t1`.`f2` AS `f2` from `test`.`t1` where <expr_cache><`test`.`t1`.`f1`,`test`.`t1`.`pk`>(exists(select 1 from `test`.`t1` `sq2` semi join (`test`.`t1`) where ((`test`.`sq2`.`f1` = `test`.`t1`.`f1`) and (`test`.`t1`.`pk` = `test`.`t1`.`f1`)))) +SELECT STRAIGHT_JOIN sq1.f2 +FROM ( SELECT * FROM t1 ) AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +f2 +foo +set optimizer_switch='exists_to_in=on'; +explain extended SELECT STRAIGHT_JOIN sq1.f2 +FROM ( SELECT * FROM t1 ) AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 Using where +3 DEPENDENT SUBQUERY <subquery4> eq_ref distinct_key distinct_key 4 func 1 100.00 +3 DEPENDENT SUBQUERY sq2 ALL NULL NULL NULL NULL 2 100.00 Using where; Using join buffer (flat, BNL join) +4 MATERIALIZED t1 ALL NULL NULL NULL NULL 2 100.00 +Warnings: +Note 1276 Field or reference 'sq1.pk' of SELECT #3 was resolved in SELECT #1 +Note 1276 Field or reference 'sq1.f1' of SELECT #3 was resolved in SELECT #1 +Note 1003 select straight_join `test`.`t1`.`f2` AS `f2` from `test`.`t1` where <expr_cache><`test`.`t1`.`f1`,`test`.`t1`.`pk`>(<in_optimizer>(`test`.`t1`.`f1`,<exists>(select `test`.`sq2`.`f1` from `test`.`t1` `sq2` semi join (`test`.`t1`) where ((`test`.`t1`.`pk` = `test`.`t1`.`f1`) and (<cache>(`test`.`t1`.`f1`) = `test`.`sq2`.`f1`))))) +SELECT STRAIGHT_JOIN sq1.f2 +FROM ( SELECT * FROM t1 ) AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +f2 +foo +set optimizer_switch= @optimizer_switch_save; +DROP TABLE t1; # End of 10.0 tests set optimizer_switch=default; diff --git a/mysql-test/r/subselect_innodb.result b/mysql-test/r/subselect_innodb.result index 07d00e96549..15f1d6377e3 100644 --- a/mysql-test/r/subselect_innodb.result +++ b/mysql-test/r/subselect_innodb.result @@ -314,7 +314,7 @@ EXPLAIN SELECT 1 FROM t1 WHERE NOT EXISTS id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 1 Using where 2 DEPENDENT SUBQUERY t2 eq_ref PRIMARY,d PRIMARY 1 func 1 Using where -3 DEPENDENT SUBQUERY t2 index NULL d 2 NULL 1 Using where; Using index +3 DEPENDENT SUBQUERY t2 index NULL d 2 NULL 1 Using index DROP TABLE t2; CREATE TABLE t2 (b INT, c INT, UNIQUE KEY (b), UNIQUE KEY (b, c )) ENGINE=INNODB; INSERT INTO t2 VALUES (1, 1); @@ -496,6 +496,35 @@ HAVING SQ2_alias1 . col_int_key >= 7 drop table t1; set optimizer_switch=@subselect_innodb_tmp; # +# MDEV-9635:Server crashes in part_of_refkey or assertion +# `!created && key_to_save < (int)s->keys' failed in +# TABLE::use_index(int) or with join_cache_level>2 +# +SET join_cache_level=3; +CREATE TABLE t1 (f1 VARCHAR(1024)) ENGINE=InnoDB; +CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1; +CREATE TABLE t2 (f2 VARCHAR(4)) ENGINE=InnoDB; +INSERT INTO t2 VALUES ('foo'),('bar'); +SELECT * FROM v1, t2 WHERE ( f1, f2 ) IN ( SELECT f1, f1 FROM t1 ); +f1 f2 +set join_cache_level = default; +drop view v1; +drop table t1,t2; +# +# MDEV-10693: cost-based choice between materialization and in-to-exists +# for a subquery from the expression used in ref access +# +CREATE TABLE t1 (i1 INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (i2 INT) ENGINE=InnoDB; +CREATE TABLE t3 (i3 INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t3 VALUES (3); +SELECT * FROM t1 +WHERE NULL IN ( SELECT i2 FROM t2 +WHERE i1 IN ( i2 IN ( SELECT i3 FROM t3 ) ) AND i2 = 2 ); +i1 +DROP TABLE t1,t2,t3; +# # MDEV-6041: ORDER BY+subqueries: subquery_table.key=outer_table.col is not recongized as binding # create table t1(a int) engine=innodb; diff --git a/mysql-test/r/subselect_mat.result b/mysql-test/r/subselect_mat.result index 1ea71b6ee32..09223d36429 100644 --- a/mysql-test/r/subselect_mat.result +++ b/mysql-test/r/subselect_mat.result @@ -2163,6 +2163,117 @@ execute stmt; a 0 drop table t1; +# +# MDEV-12429: IN subquery used in WHERE of EXISTS subquery +# +CREATE TABLE t1 ( +pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7); +SELECT sq1.f2 FROM t1 AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +f2 +foo +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch='exists_to_in=off'; +EXPLAIN +SELECT sq1.f2 FROM t1 AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY sq1 ALL NULL NULL NULL NULL 2 Using where +2 DEPENDENT SUBQUERY <subquery3> eq_ref distinct_key distinct_key 4 func 1 +2 DEPENDENT SUBQUERY sq2 ALL NULL NULL NULL NULL 2 Using where; Using join buffer (flat, BNL join) +3 MATERIALIZED t1 ALL NULL NULL NULL NULL 2 +# this checks the result set above +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT sq1.f2 FROM t1 AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +f2 +foo +set optimizer_switch= @save_optimizer_switch; +DROP TABLE t1; +# +# MDEV-12145: IN subquery used in WHERE of EXISTS subquery +# +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); +CREATE TABLE t2 (i2 INT, KEY(i2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (8),(7),(1); +CREATE TABLE t3 (f3 INT, i3 INT, KEY(i3)) ENGINE=MyISAM; +INSERT INTO t3 VALUES (8,0),(6,3),(2,8),(3,8),(1,6),(0,0),(1,0),(1,5); +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch='exists_to_in=off'; +SELECT * FROM t1 +WHERE EXISTS ( SELECT * FROM t2, t3 +WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +f1 +6 +EXPLAIN EXTENDED +SELECT * FROM t1 +WHERE EXISTS ( SELECT * FROM t2, t3 +WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 Using where +2 DEPENDENT SUBQUERY <subquery3> eq_ref distinct_key distinct_key 4 func 1 100.00 +2 DEPENDENT SUBQUERY t2 index i2 i2 5 NULL 3 100.00 Using where; Using index; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t3 ref i3 i3 5 test.t2.i2 2 100.00 Using index +3 MATERIALIZED t3 ALL NULL NULL NULL NULL 8 100.00 +Warnings: +Note 1276 Field or reference 'test.t1.f1' of SELECT #2 was resolved in SELECT #1 +Note 1003 select `test`.`t1`.`f1` AS `f1` from `test`.`t1` where <expr_cache><`test`.`t1`.`f1`>(exists(select 1 from `test`.`t2` semi join (`test`.`t3`) join `test`.`t3` where ((`test`.`t3`.`i3` = `test`.`t2`.`i2`) and (`test`.`t1`.`f1` = `test`.`t3`.`f3`)))) +# this checks the result set above +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT * FROM t1 +WHERE EXISTS ( SELECT * FROM t2, t3 +WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +f1 +6 +set optimizer_switch= @save_optimizer_switch; +DROP TABLE t1,t2,t3; +# +# MDEV-9686: IN subquery used in WHERE of a subquery from select list +# +CREATE TABLE t1 (pk INT PRIMARY KEY, f1 INT); +INSERT INTO t1 VALUES (1, 4),(2, 3),(3, 3),(4, 6),(5, 3); +CREATE TABLE t2 (f2 INT); +INSERT INTO t2 VALUES (1),(2),(3),(4),(5); +# t1.pk is always IN ( SELECT f2 FROM t2 ), +# so the IN condition should be true for every row, +# and thus COUNT(*) should always return 5 +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 +WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +pk f1 sq +1 4 5 +2 3 5 +3 3 5 +4 6 5 +5 3 5 +EXPLAIN EXTENDED +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 +WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 5 100.00 +2 DEPENDENT SUBQUERY <subquery3> eq_ref distinct_key distinct_key 4 func 1 100.00 +2 DEPENDENT SUBQUERY t2 ALL NULL NULL NULL NULL 5 100.00 Using join buffer (flat, BNL join) +3 MATERIALIZED t2 ALL NULL NULL NULL NULL 5 100.00 +Warnings: +Note 1276 Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1 +Note 1003 select `test`.`t1`.`pk` AS `pk`,`test`.`t1`.`f1` AS `f1`,<expr_cache><`test`.`t1`.`pk`>((select count(0) from `test`.`t2` semi join (`test`.`t2`) where (`test`.`t1`.`pk` = `test`.`t2`.`f2`))) AS `sq` from `test`.`t1` +# this checks the result set above +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 +WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +pk f1 sq +1 4 5 +2 3 5 +3 3 5 +4 6 5 +5 3 5 +set optimizer_switch= @save_optimizer_switch; +DROP TABLE t1,t2; # End of 5.5 tests # # MDEV-7220: Materialization strategy is not used for REPLACE ... SELECT diff --git a/mysql-test/r/subselect_mat_cost_bugs.result b/mysql-test/r/subselect_mat_cost_bugs.result index 24640154c59..9bfd5bd67b5 100644 --- a/mysql-test/r/subselect_mat_cost_bugs.result +++ b/mysql-test/r/subselect_mat_cost_bugs.result @@ -400,3 +400,45 @@ id select_type table type possible_keys key key_len ref rows Extra select * from t1 where c1 = 'a2' and (c1, c2) not in (select * from t2); c1 c2 drop table t1, t2; +# +# MDEV-12673: cost-based choice between materialization and in-to-exists +# +CREATE TABLE t1 ( +pk1 int, a1 varchar(3), b1 varchar(3), PRIMARY KEY (pk1), KEY(a1), KEY(b1) +) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,'foo','bar'),(2,'bar','foo'); +CREATE TABLE t2 (pk2 INT PRIMARY KEY, a2 VARCHAR(3), KEY(a2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1,'abc'),(2,'xyz'),(3,'foo'); +SELECT 'qux' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ); +'qux' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ) +0 +SELECT 'bar' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ); +'bar' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ) +1 +EXPLAIN +SELECT 'bar' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL No tables used +2 SUBQUERY t2 const PRIMARY,a2 PRIMARY 4 const 1 +2 SUBQUERY t1 ref a1,b1 b1 6 const 1 Using where +DROP TABLE t1,t2; +CREATE TABLE t1 (i1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (i2 int, c2 varchar(3), KEY(i2,c2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1,'abc'),(2,'foo'); +CREATE TABLE t3 (pk3 int PRIMARY KEY, c3 varchar(3)) ENGINE=MyISAM; +INSERT INTO t3 VALUES (1,'foo'),(2,'bar'); +SELECT * FROM t1 WHERE i1 NOT IN ( +SELECT i2 FROM t2 RIGHT JOIN t3 ON (c3 = c2) WHERE pk3 = i1 +); +i1 +1 +EXPLAIN +SELECT * FROM t1 WHERE i1 NOT IN ( +SELECT i2 FROM t2 RIGHT JOIN t3 ON (c3 = c2) WHERE pk3 = i1 +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 system NULL NULL NULL NULL 1 +2 DEPENDENT SUBQUERY t3 const PRIMARY PRIMARY 4 const 1 +2 DEPENDENT SUBQUERY t2 index NULL i2 11 NULL 2 Using where; Using index +DROP TABLE t1,t2,t3; diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result index dfcbec1c595..8c0e6041a90 100644 --- a/mysql-test/r/subselect_no_exists_to_in.result +++ b/mysql-test/r/subselect_no_exists_to_in.result @@ -7136,6 +7136,17 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set optimizer_switch=default; select @@optimizer_switch like '%exists_to_in=off%'; @@optimizer_switch like '%exists_to_in=off%' diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index 02068584287..5e2b4f2d374 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7129,6 +7129,17 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set optimizer_switch=default; select @@optimizer_switch like '%materialization=on%'; @@optimizer_switch like '%materialization=on%' diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index 33a01c807f6..afe79f80015 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7127,4 +7127,15 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set @optimizer_switch_for_subselect_test=null; diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index abd2799769d..15f7b977c24 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7142,6 +7142,17 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set optimizer_switch=default; select @@optimizer_switch like '%subquery_cache=on%'; @@optimizer_switch like '%subquery_cache=on%' diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index b59dcd14285..1c5578b3575 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7127,5 +7127,16 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set @optimizer_switch_for_subselect_test=null; set @join_cache_level_for_subselect_test=NULL; diff --git a/mysql-test/r/subselect_sj.result b/mysql-test/r/subselect_sj.result index 3ca3f0d35fb..325578b27fd 100644 --- a/mysql-test/r/subselect_sj.result +++ b/mysql-test/r/subselect_sj.result @@ -178,7 +178,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DEPENDENT SUBQUERY m16 ALL NULL NULL NULL NULL 3 Using join buffer (flat, BNL join) 2 DEPENDENT SUBQUERY m17 ALL NULL NULL NULL NULL 3 Using join buffer (flat, BNL join) 2 DEPENDENT SUBQUERY m18 ALL NULL NULL NULL NULL 3 Using join buffer (flat, BNL join) -2 DEPENDENT SUBQUERY m19 ALL NULL NULL NULL NULL 3 Using where; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY m19 ALL NULL NULL NULL NULL 3 Using join buffer (flat, BNL join) select * from t1 left join t2 on (t2.a= t1.a and t2.a in (select pk from t10)) where t1.a < 5; diff --git a/mysql-test/r/subselect_sj_jcl6.result b/mysql-test/r/subselect_sj_jcl6.result index ed6bf8bc11b..595252daafe 100644 --- a/mysql-test/r/subselect_sj_jcl6.result +++ b/mysql-test/r/subselect_sj_jcl6.result @@ -191,7 +191,7 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DEPENDENT SUBQUERY m16 ALL NULL NULL NULL NULL 3 Using join buffer (incremental, BNL join) 2 DEPENDENT SUBQUERY m17 ALL NULL NULL NULL NULL 3 Using join buffer (incremental, BNL join) 2 DEPENDENT SUBQUERY m18 ALL NULL NULL NULL NULL 3 Using join buffer (incremental, BNL join) -2 DEPENDENT SUBQUERY m19 ALL NULL NULL NULL NULL 3 Using where; Using join buffer (incremental, BNL join) +2 DEPENDENT SUBQUERY m19 ALL NULL NULL NULL NULL 3 Using join buffer (incremental, BNL join) select * from t1 left join t2 on (t2.a= t1.a and t2.a in (select pk from t10)) where t1.a < 5; diff --git a/mysql-test/r/subselect_sj_mat.result b/mysql-test/r/subselect_sj_mat.result index fee63d8204e..3f91b510c5b 100644 --- a/mysql-test/r/subselect_sj_mat.result +++ b/mysql-test/r/subselect_sj_mat.result @@ -2203,6 +2203,117 @@ execute stmt; a 0 drop table t1; +# +# MDEV-12429: IN subquery used in WHERE of EXISTS subquery +# +CREATE TABLE t1 ( +pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7); +SELECT sq1.f2 FROM t1 AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +f2 +foo +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch='exists_to_in=off'; +EXPLAIN +SELECT sq1.f2 FROM t1 AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY sq1 ALL NULL NULL NULL NULL 2 Using where +2 DEPENDENT SUBQUERY <subquery3> eq_ref distinct_key distinct_key 4 func 1 +2 DEPENDENT SUBQUERY sq2 ALL NULL NULL NULL NULL 2 Using where; Using join buffer (flat, BNL join) +3 MATERIALIZED t1 ALL NULL NULL NULL NULL 2 +# this checks the result set above +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT sq1.f2 FROM t1 AS sq1 +WHERE EXISTS ( SELECT * FROM t1 AS sq2 +WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); +f2 +foo +set optimizer_switch= @save_optimizer_switch; +DROP TABLE t1; +# +# MDEV-12145: IN subquery used in WHERE of EXISTS subquery +# +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); +CREATE TABLE t2 (i2 INT, KEY(i2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (8),(7),(1); +CREATE TABLE t3 (f3 INT, i3 INT, KEY(i3)) ENGINE=MyISAM; +INSERT INTO t3 VALUES (8,0),(6,3),(2,8),(3,8),(1,6),(0,0),(1,0),(1,5); +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch='exists_to_in=off'; +SELECT * FROM t1 +WHERE EXISTS ( SELECT * FROM t2, t3 +WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +f1 +6 +EXPLAIN EXTENDED +SELECT * FROM t1 +WHERE EXISTS ( SELECT * FROM t2, t3 +WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 Using where +2 DEPENDENT SUBQUERY <subquery3> eq_ref distinct_key distinct_key 4 func 1 100.00 +2 DEPENDENT SUBQUERY t2 index i2 i2 5 NULL 3 100.00 Using where; Using index; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t3 ref i3 i3 5 test.t2.i2 2 100.00 Using index +3 MATERIALIZED t3 ALL NULL NULL NULL NULL 8 100.00 +Warnings: +Note 1276 Field or reference 'test.t1.f1' of SELECT #2 was resolved in SELECT #1 +Note 1003 select `test`.`t1`.`f1` AS `f1` from `test`.`t1` where <expr_cache><`test`.`t1`.`f1`>(exists(select 1 from `test`.`t2` semi join (`test`.`t3`) join `test`.`t3` where ((`test`.`t3`.`i3` = `test`.`t2`.`i2`) and (`test`.`t1`.`f1` = `test`.`t3`.`f3`)))) +# this checks the result set above +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT * FROM t1 +WHERE EXISTS ( SELECT * FROM t2, t3 +WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +f1 +6 +set optimizer_switch= @save_optimizer_switch; +DROP TABLE t1,t2,t3; +# +# MDEV-9686: IN subquery used in WHERE of a subquery from select list +# +CREATE TABLE t1 (pk INT PRIMARY KEY, f1 INT); +INSERT INTO t1 VALUES (1, 4),(2, 3),(3, 3),(4, 6),(5, 3); +CREATE TABLE t2 (f2 INT); +INSERT INTO t2 VALUES (1),(2),(3),(4),(5); +# t1.pk is always IN ( SELECT f2 FROM t2 ), +# so the IN condition should be true for every row, +# and thus COUNT(*) should always return 5 +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 +WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +pk f1 sq +1 4 5 +2 3 5 +3 3 5 +4 6 5 +5 3 5 +EXPLAIN EXTENDED +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 +WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 5 100.00 +2 DEPENDENT SUBQUERY <subquery3> eq_ref distinct_key distinct_key 4 func 1 100.00 +2 DEPENDENT SUBQUERY t2 ALL NULL NULL NULL NULL 5 100.00 Using join buffer (flat, BNL join) +3 MATERIALIZED t2 ALL NULL NULL NULL NULL 5 100.00 +Warnings: +Note 1276 Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1 +Note 1003 select `test`.`t1`.`pk` AS `pk`,`test`.`t1`.`f1` AS `f1`,<expr_cache><`test`.`t1`.`pk`>((select count(0) from `test`.`t2` semi join (`test`.`t2`) where (`test`.`t1`.`pk` = `test`.`t2`.`f2`))) AS `sq` from `test`.`t1` +# this checks the result set above +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 +WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +pk f1 sq +1 4 5 +2 3 5 +3 3 5 +4 6 5 +5 3 5 +set optimizer_switch= @save_optimizer_switch; +DROP TABLE t1,t2; # End of 5.5 tests # # MDEV-7220: Materialization strategy is not used for REPLACE ... SELECT diff --git a/mysql-test/r/symlink-aria-11902.result b/mysql-test/r/symlink-aria-11902.result new file mode 100644 index 00000000000..e563780b3ec --- /dev/null +++ b/mysql-test/r/symlink-aria-11902.result @@ -0,0 +1,39 @@ +set default_storage_engine=Aria; +call mtr.add_suppression("File.*t1.* not found"); +create table mysql.t1 (a int, b char(16), index(a)); +insert mysql.t1 values (100, 'test'),(101,'test'); +create table t1 (a int, b char(16), index(a)) +data directory="MYSQLTEST_VARDIR/tmp/foo"; +insert t1 values (200, 'some'),(201,'some'); +select * from t1; +a b +200 some +201 some +flush tables; +set debug_sync='mi_open_datafile SIGNAL ok WAIT_FOR go'; +select * from t1; +set debug_sync='now WAIT_FOR ok'; +set debug_sync='now SIGNAL go'; +ERROR HY000: File 'MYSQLTEST_VARDIR/tmp/foo/t1.MAD' not found (Errcode: 20 <errmsg>) +flush tables; +drop table if exists t1; +create table t1 (a int, b char(16), index (a)) +index directory="MYSQLTEST_VARDIR/tmp/foo"; +insert t1 values (200, 'some'),(201,'some'); +explain select a from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL a 5 NULL 2 Using index +select a from t1; +a +200 +201 +flush tables; +set debug_sync='mi_open_kfile SIGNAL waiting WAIT_FOR run'; +select a from t1; +set debug_sync='now WAIT_FOR waiting'; +set debug_sync='now SIGNAL run'; +ERROR HY000: Can't find file: './test/t1.MAI' (errno: 20 <errmsg>) +flush tables; +drop table if exists t1; +drop table mysql.t1; +set debug_sync='RESET'; diff --git a/mysql-test/r/symlink-myisam-11902.result b/mysql-test/r/symlink-myisam-11902.result new file mode 100644 index 00000000000..33357a80700 --- /dev/null +++ b/mysql-test/r/symlink-myisam-11902.result @@ -0,0 +1,38 @@ +call mtr.add_suppression("File.*t1.* not found"); +create table mysql.t1 (a int, b char(16), index(a)); +insert mysql.t1 values (100, 'test'),(101,'test'); +create table t1 (a int, b char(16), index(a)) +data directory="MYSQLTEST_VARDIR/tmp/foo"; +insert t1 values (200, 'some'),(201,'some'); +select * from t1; +a b +200 some +201 some +flush tables; +set debug_sync='mi_open_datafile SIGNAL ok WAIT_FOR go'; +select * from t1; +set debug_sync='now WAIT_FOR ok'; +set debug_sync='now SIGNAL go'; +ERROR HY000: File 'MYSQLTEST_VARDIR/tmp/foo/t1.MYD' not found (Errcode: 20 <errmsg>) +flush tables; +drop table if exists t1; +create table t1 (a int, b char(16), index (a)) +index directory="MYSQLTEST_VARDIR/tmp/foo"; +insert t1 values (200, 'some'),(201,'some'); +explain select a from t1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL a 5 NULL 2 Using index +select a from t1; +a +200 +201 +flush tables; +set debug_sync='mi_open_kfile SIGNAL waiting WAIT_FOR run'; +select a from t1; +set debug_sync='now WAIT_FOR waiting'; +set debug_sync='now SIGNAL run'; +ERROR HY000: Can't find file: './test/t1.MYI' (errno: 20 <errmsg>) +flush tables; +drop table if exists t1; +drop table mysql.t1; +set debug_sync='RESET'; diff --git a/mysql-test/r/symlink.result b/mysql-test/r/symlink.result index 279af1e44d2..3218de4b225 100644 --- a/mysql-test/r/symlink.result +++ b/mysql-test/r/symlink.result @@ -213,3 +213,14 @@ t2 CREATE TABLE `t2` ( PRIMARY KEY (`a`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 drop tables t1, t2; +# +# Test for bug #25514146 DB_NAME IS IGNORED WHEN CREATING TABLE +# WITH DATA DIRECTORY +# +# Make sure we have no current database +CREATE DATABASE x; +USE x; +DROP DATABASE x; +CREATE TABLE test.t1(id INT(11)) ENGINE MYISAM +DATA DIRECTORY "MYSQLTEST_VARDIR/tmp"; +DROP TABLE test.t1; diff --git a/mysql-test/r/table_elim.result b/mysql-test/r/table_elim.result index c633261bcd3..3e68a877f02 100644 --- a/mysql-test/r/table_elim.result +++ b/mysql-test/r/table_elim.result @@ -597,7 +597,8 @@ CREATE TABLE t1 (a int(11), b varchar(1)) ; INSERT IGNORE INTO t1 VALUES (0,'g'); CREATE TABLE t3 ( a varchar(1)) ; INSERT IGNORE INTO t3 VALUES ('g'); -CREATE TABLE t2 ( a int(11) NOT NULL, PRIMARY KEY (a)) ; +CREATE TABLE t2 ( a int(11) NOT NULL, PRIMARY KEY (a)); +INSERT INTO t2 VALUES (9), (10); create view v1 as SELECT t1.* FROM t1 LEFT JOIN t2 ON ( t1.a = t2.a ) WHERE t2.a <> 0; SELECT alias1.* FROM t3 LEFT JOIN v1 as alias1 ON ( t3.a = alias1.b ); a b @@ -606,7 +607,7 @@ EXPLAIN SELECT alias1.* FROM t3 LEFT JOIN v1 as alias1 ON ( t3.a = alias1.b ); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t3 system NULL NULL NULL NULL 1 1 SIMPLE t1 ALL NULL NULL NULL NULL 1 Using where -1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 Using where; Using index +1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 Using index drop view v1; DROP TABLE t1,t2,t3; # diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result index 40f5a77e3d0..5a6cd8907e9 100644 --- a/mysql-test/r/union.result +++ b/mysql-test/r/union.result @@ -362,7 +362,7 @@ a 2 select found_rows(); found_rows() -6 +5 SELECT SQL_CALC_FOUND_ROWS * FROM t1 UNION SELECT * FROM t2 LIMIT 100; a 1 @@ -1169,12 +1169,9 @@ a b select * from ((select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b select * from ((select * from t1 limit 1) union (select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b -3 c select * from ((((select * from t1))) union (select * from t1) union (select * from t1)) a; a b 1 a @@ -1553,7 +1550,6 @@ NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL NULL Using filesort Warnings: Note 1003 select NULL AS `a` from `test`.`t1` union select NULL AS `a` from `test`.`t1` order by `a` DROP TABLE t1; -End of 5.0 tests # # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take # subselects into account @@ -1659,6 +1655,14 @@ a 4 5 6 +(select a from t1 where false) UNION (select a from t1) limit 8; +a +10 +2 +3 +4 +5 +6 7 8 drop table t1; @@ -1955,3 +1959,22 @@ cccc bbbb dddd drop table t1; +# +# MDEV-10172: UNION query returns incorrect rows outside +# conditional evaluation +# +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( +select * from t1 where d between '2016-06-02' and '2016-06-05' + union +(select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +d +2016-06-02 00:00:00 +2016-06-03 00:00:00 +2016-06-04 00:00:00 +drop table t1; +End of 5.0 tests diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 924b3a11fef..d6da2a03b46 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -5536,6 +5536,89 @@ Warnings: Warning 1356 View 'test.v1' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them drop view v1; drop table t1,t2; +# +# MDEV-12099: usage of mergeable view with LEFT JOIN +# that can be converted to INNER JOIN +# +create table t1 (a int, b int, key(a)) engine=myisam; +insert into t1 values +(3,20), (7,10), (2,10), (4,30), (8,70), +(7,70), (9,100), (9,60), (8,80), (7,60); +create table t2 (c int, d int, key (c)) engine=myisam; +insert into t2 values +(50,100), (20, 200), (10,300), +(150,100), (120, 200), (110,300), +(250,100), (220, 200), (210,300); +create table t3(e int, f int not null, key(e), unique (f)) engine=myisam; +insert into t3 values +(100, 3), (300, 5), (400, 4), (300,7), +(300,2), (600, 13), (800, 15), (700, 14), +(600, 23), (800, 25), (700, 24); +create view v1 as +select * from t2 left join t3 on t3.e=t2.d where t3.f is not null; +select * +from t1 left join v1 on v1.c=t1.b +where t1.a < 5; +a b c d e f +2 10 10 300 300 5 +2 10 10 300 300 7 +2 10 10 300 300 2 +3 20 NULL NULL NULL NULL +4 30 NULL NULL NULL NULL +select * +from t1 left join ( t2 left join t3 on t3.e=t2.d ) +on t2.c=t1.b and t3.f is not null +where t1.a < 5; +a b c d e f +2 10 10 300 300 5 +2 10 10 300 300 7 +2 10 10 300 300 2 +3 20 NULL NULL NULL NULL +4 30 NULL NULL NULL NULL +explain extended +select * +from t1 left join v1 on v1.c=t1.b +where t1.a < 5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 range a a 5 NULL 3 100.00 Using index condition +1 SIMPLE t2 ref c c 5 test.t1.b 2 100.00 Using where +1 SIMPLE t3 ref f,e e 5 test.t2.d 2 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d`,`test`.`t3`.`e` AS `e`,`test`.`t3`.`f` AS `f` from `test`.`t1` left join (`test`.`t2` join `test`.`t3`) on(((`test`.`t2`.`c` = `test`.`t1`.`b`) and (`test`.`t3`.`e` = `test`.`t2`.`d`) and (`test`.`t3`.`f` is not null) and (`test`.`t1`.`b` is not null) and (`test`.`t2`.`d` is not null))) where (`test`.`t1`.`a` < 5) +explain extended +select * +from t1 left join ( t2 left join t3 on t3.e=t2.d ) +on t2.c=t1.b and t3.f is not null +where t1.a < 5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 range a a 5 NULL 3 100.00 Using index condition +1 SIMPLE t2 ref c c 5 test.t1.b 2 100.00 Using where +1 SIMPLE t3 ref f,e e 5 test.t2.d 2 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d`,`test`.`t3`.`e` AS `e`,`test`.`t3`.`f` AS `f` from `test`.`t1` left join (`test`.`t2` join `test`.`t3`) on(((`test`.`t2`.`c` = `test`.`t1`.`b`) and (`test`.`t3`.`e` = `test`.`t2`.`d`) and (`test`.`t3`.`f` is not null) and (`test`.`t1`.`b` is not null) and (`test`.`t2`.`d` is not null))) where (`test`.`t1`.`a` < 5) +explain extended +select * +from t1 left join v1 on v1.c=t1.b and v1.f=t1.a +where t1.a < 5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 range a a 5 NULL 3 100.00 Using index condition +1 SIMPLE t3 eq_ref f,e f 4 test.t1.a 1 100.00 Using where +1 SIMPLE t2 ref c c 5 test.t1.b 2 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d`,`test`.`t3`.`e` AS `e`,`test`.`t3`.`f` AS `f` from `test`.`t1` left join (`test`.`t2` join `test`.`t3`) on(((`test`.`t2`.`c` = `test`.`t1`.`b`) and (`test`.`t3`.`f` = `test`.`t1`.`a`) and (`test`.`t2`.`d` = `test`.`t3`.`e`) and (`test`.`t1`.`a` is not null) and (`test`.`t1`.`a` is not null) and (`test`.`t1`.`b` is not null))) where (`test`.`t1`.`a` < 5) +explain extended +select * +from t1 left join ( t2 left join t3 on t3.e=t2.d ) +on t2.c=t1.b and t3.f=t1.a and t3.f is not null +where t1.a < 5; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 range a a 5 NULL 3 100.00 Using index condition +1 SIMPLE t3 eq_ref f,e f 4 test.t1.a 1 100.00 Using where +1 SIMPLE t2 ref c c 5 test.t1.b 2 100.00 Using where +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d`,`test`.`t3`.`e` AS `e`,`test`.`t3`.`f` AS `f` from `test`.`t1` left join (`test`.`t2` join `test`.`t3`) on(((`test`.`t2`.`c` = `test`.`t1`.`b`) and (`test`.`t3`.`f` = `test`.`t1`.`a`) and (`test`.`t2`.`d` = `test`.`t3`.`e`) and (`test`.`t1`.`a` is not null) and (`test`.`t1`.`a` is not null) and (`test`.`t1`.`b` is not null))) where (`test`.`t1`.`a` < 5) +drop view v1; +drop table t1,t2,t3; # ----------------------------------------------------------------- # -- End of 5.5 tests. # ----------------------------------------------------------------- @@ -5799,6 +5882,68 @@ f1 f2 drop table t1, t2; SELECT 1 FROM (SELECT 1 as a) AS b HAVING (SELECT `SOME_GARBAGE`.b.a)=1; ERROR 42S22: Unknown column 'SOME_GARBAGE.b.a' in 'field list' +# +# MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 +# FOR UPDATE +# +CREATE TABLE t1 (a INT); +insert into t1 values (1),(2); +CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `t1`.`a` AS `a` from `t1` for update latin1 latin1_swedish_ci +select * from v1; +a +1 +2 +DROP VIEW v1; +CREATE VIEW v1 AS SELECT * FROM t1 LOCK IN SHARE MODE; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `t1`.`a` AS `a` from `t1` lock in share mode latin1 latin1_swedish_ci +select * from v1; +a +1 +2 +DROP VIEW v1; +DROP TABLE t1; +# +# MDEV-8642: WHERE Clause not applied on View - Empty result set returned +# +CREATE TABLE `t1` ( +`id` int(20) NOT NULL AUTO_INCREMENT, +`use_case` int(11) DEFAULT NULL, +`current_deadline` date DEFAULT NULL, +`ts_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, +PRIMARY KEY (`id`), +UNIQUE KEY `id_UNIQUE` (`id`) +) ENGINE=MyISAM AUTO_INCREMENT=13976 DEFAULT CHARSET=latin1; +INSERT INTO `t1` VALUES (1,10,'2015-12-18','2015-08-18 08:38:16'); +INSERT INTO `t1` VALUES (2,20,'2015-10-18','2015-08-18 08:43:30'); +CREATE VIEW v1 AS SELECT +use_case as use_case_id, +( +SELECT +deadline_sub.current_deadline +FROM +t1 deadline_sub +WHERE +deadline_sub.use_case = use_case_id +AND ts_create = (SELECT +MIN(ts_create) +FROM +t1 startdate_sub +WHERE +startdate_sub.use_case = use_case_id +) +) AS InitialDeadline +FROM +t1; +SELECT * FROM v1 where use_case_id = 10; +use_case_id InitialDeadline +10 2015-12-18 +drop view v1; +drop table t1; # ----------------------------------------------------------------- # -- End of 10.0 tests. # ----------------------------------------------------------------- diff --git a/mysql-test/std_data/bug20683959loaddata.txt b/mysql-test/std_data/bug20683959loaddata.txt deleted file mode 100644 index 1878cc78879..00000000000 --- a/mysql-test/std_data/bug20683959loaddata.txt +++ /dev/null @@ -1 +0,0 @@ -Ã"RT @niouzechun: é˜âˆšõ€®ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹ç‰™è€³ç¸ºï½ªç¹§è–™â–¡ç¸ºä»£ï½Œç¸ºï½©ç¸²âˆšã„ç¹ï½³ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹å³¨ï½„è« ï½¨èœ‰õ€”Žå™ªç¸ºï½ªç¸ºé¡˜ï½©ï½±ç¹§åµâ‰ 縺ｾ繧顔ｩ肴･ｵ逧õ€‹–↓鞫ょå™ç¸ºåŠ±â†‘縺õ€‹šç‚Šé€•ï½±ç¸ºï½¯ç¸²âˆ«æ¨Ÿèž³æº˜õ€èŽ ｺ逕溘õ€®è“ｺè›ï½¬é€§õ€‹–↓縺õ€‘Žâˆªç¸ºä¸Šï¼žç¸ºä¹â†‘縺õ€‹–ï¼ è³æ¦Šï½¹ï½³é²å³¨â–¡ç¸ºç¤¼ç‚Šè³æ¦Šï½°ï½½ç¸º 縺苓ｾ帙＞ diff --git a/mysql-test/std_data/loaddata/mdev-11079.txt b/mysql-test/std_data/loaddata/mdev-11079.txt new file mode 100644 index 00000000000..a792f984d5f --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11079.txt @@ -0,0 +1 @@ +"%ª«¬" diff --git a/mysql-test/std_data/loaddata/mdev-11631.txt b/mysql-test/std_data/loaddata/mdev-11631.txt new file mode 100644 index 00000000000..87b824b71ae --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11631.txt @@ -0,0 +1 @@ +\ä diff --git a/mysql-test/suite/archive/discover.result b/mysql-test/suite/archive/discover.result index e1ca9cb6a65..0619ca2051a 100644 --- a/mysql-test/suite/archive/discover.result +++ b/mysql-test/suite/archive/discover.result @@ -139,3 +139,10 @@ flush tables; create table t1 (a int) engine=archive; ERROR 42S01: Table 't1' already exists drop table t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +DROP TABLE t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +SELECT * FROM t1; +pk +DROP TABLE t1; diff --git a/mysql-test/suite/archive/discover.test b/mysql-test/suite/archive/discover.test index 20cb69efa00..4ab35cf1115 100644 --- a/mysql-test/suite/archive/discover.test +++ b/mysql-test/suite/archive/discover.test @@ -132,3 +132,13 @@ flush tables; create table t1 (a int) engine=archive; drop table t1; +# +# MDEV-11317: Error in deleting non existing .frm for tables with disocvery +# + +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +DROP TABLE t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +SELECT * FROM t1; +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_max_binlog_stmt_cache_size.result b/mysql-test/suite/binlog/r/binlog_max_binlog_stmt_cache_size.result new file mode 100644 index 00000000000..00f01b59732 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_max_binlog_stmt_cache_size.result @@ -0,0 +1,22 @@ +# +# MDEV-4774: Strangeness with max_binlog_stmt_cache_size Settings +# +CALL mtr.add_suppression("unsigned value 18446744073709547520 adjusted to 4294963200"); +SELECT @@global.max_binlog_stmt_cache_size; +@@global.max_binlog_stmt_cache_size +MAX_BINLOG_STMT_CACHE_SIZE +SET @cache_size= @@max_binlog_stmt_cache_size; +SET @@global.max_binlog_stmt_cache_size= @cache_size+1; +SELECT @@global.max_binlog_stmt_cache_size; +@@global.max_binlog_stmt_cache_size +MAX_BINLOG_STMT_CACHE_SIZE +SET @@global.max_binlog_stmt_cache_size = @cache_size+4095; +SELECT @@global.max_binlog_stmt_cache_size; +@@global.max_binlog_stmt_cache_size +MAX_BINLOG_STMT_CACHE_SIZE +SET @@global.max_binlog_stmt_cache_size= @cache_size-1; +SELECT @@global.max_binlog_stmt_cache_size = @cache_size-4096; +@@global.max_binlog_stmt_cache_size = @cache_size-4096 +1 +SET @@global.max_binlog_stmt_cache_size= @cache_size; +# End of test diff --git a/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.opt b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.opt new file mode 100644 index 00000000000..c52ef14d5d0 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.opt @@ -0,0 +1 @@ +--max_binlog_stmt_cache_size=18446744073709547520 diff --git a/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.test b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.test new file mode 100644 index 00000000000..ca3f45c154c --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.test @@ -0,0 +1,36 @@ + +--echo # +--echo # MDEV-4774: Strangeness with max_binlog_stmt_cache_size Settings +--echo # + +CALL mtr.add_suppression("unsigned value 18446744073709547520 adjusted to 4294963200"); + +--replace_result 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE 4294963200 MAX_BINLOG_STMT_CACHE_SIZE +SELECT @@global.max_binlog_stmt_cache_size; + +# Save the initial value of @@global.max_binlog_stmt_cache_size. +--replace_result 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE 4294963200 MAX_BINLOG_STMT_CACHE_SIZE +SET @cache_size= @@max_binlog_stmt_cache_size; + +--disable_warnings +SET @@global.max_binlog_stmt_cache_size= @cache_size+1; +--enable_warnings +--replace_result 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE 4294963200 MAX_BINLOG_STMT_CACHE_SIZE +SELECT @@global.max_binlog_stmt_cache_size; + +--disable_warnings +SET @@global.max_binlog_stmt_cache_size = @cache_size+4095; +--enable_warnings +--replace_result 4294963200 MAX_BINLOG_STMT_CACHE_SIZE 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE +SELECT @@global.max_binlog_stmt_cache_size; + +--disable_warnings +SET @@global.max_binlog_stmt_cache_size= @cache_size-1; +--enable_warnings +SELECT @@global.max_binlog_stmt_cache_size = @cache_size-4096; + +# Restore @@global.max_binlog_stmt_cache_size to its initial value. +SET @@global.max_binlog_stmt_cache_size= @cache_size; + +--echo # End of test + diff --git a/mysql-test/suite/csv/read_only.result b/mysql-test/suite/csv/read_only.result new file mode 100644 index 00000000000..d6936681f65 --- /dev/null +++ b/mysql-test/suite/csv/read_only.result @@ -0,0 +1,30 @@ +create table t1 (a int not null) engine=csv; +insert t1 values (1),(2); +flush tables; +select * from information_schema.tables where table_schema='test'; +TABLE_CATALOG def +TABLE_SCHEMA test +TABLE_NAME t1 +TABLE_TYPE BASE TABLE +ENGINE NULL +VERSION NULL +ROW_FORMAT NULL +TABLE_ROWS NULL +AVG_ROW_LENGTH NULL +DATA_LENGTH NULL +MAX_DATA_LENGTH NULL +INDEX_LENGTH NULL +DATA_FREE NULL +AUTO_INCREMENT NULL +CREATE_TIME NULL +UPDATE_TIME NULL +CHECK_TIME NULL +TABLE_COLLATION NULL +CHECKSUM NULL +CREATE_OPTIONS NULL +TABLE_COMMENT File './test/t1.CSM' not found (Errcode: 13 "Permission denied") +Warnings: +Level Warning +Code 29 +Message File './test/t1.CSM' not found (Errcode: 13 "Permission denied") +drop table t1; diff --git a/mysql-test/suite/csv/read_only.test b/mysql-test/suite/csv/read_only.test new file mode 100644 index 00000000000..2af209182d0 --- /dev/null +++ b/mysql-test/suite/csv/read_only.test @@ -0,0 +1,19 @@ +# +# MDEV-11883 MariaDB crashes with out-of-memory when query information_schema +# +source include/have_csv.inc; + +let datadir=`select @@datadir`; + +create table t1 (a int not null) engine=csv; +insert t1 values (1),(2); +flush tables; + +chmod 0400 $datadir/test/t1.CSM; +chmod 0400 $datadir/test/t1.CSV; + +--replace_result $datadir ./ +query_vertical select * from information_schema.tables where table_schema='test'; + +drop table t1; + diff --git a/mysql-test/suite/engines/iuds/r/insert_decimal.result b/mysql-test/suite/engines/iuds/r/insert_decimal.result index 50fde80d81d..eab8592c4ee 100644 --- a/mysql-test/suite/engines/iuds/r/insert_decimal.result +++ b/mysql-test/suite/engines/iuds/r/insert_decimal.result @@ -48,7 +48,6 @@ Warning 1264 Out of range value for column 'c2' at row 1 Note 1265 Data truncated for column 'c3' at row 1 insert into t2 values ("0.0","0.0","0.0",7),("-0.0","-0.0","-0.0",8),("+0.0","+0.0","+0.0",9),("01.0","01.0","01.0",10),("+01.0","+01.0","+01.0",11),("-01.0","-01.0","-01.0",12); Warnings: -Warning 1264 Out of range value for column 'c2' at row 2 Warning 1264 Out of range value for column 'c2' at row 6 insert into t2 values ("-.1","-.1","-.1",13),("+.1","+.1","+.1",14),(".1",".1",".1",15); Warnings: diff --git a/mysql-test/suite/federated/federated_bug_35333.test b/mysql-test/suite/federated/federated_bug_35333.test index 47feefd75a1..d43f4e930f8 100644 --- a/mysql-test/suite/federated/federated_bug_35333.test +++ b/mysql-test/suite/federated/federated_bug_35333.test @@ -61,6 +61,7 @@ let $MYSQLD_DATADIR= `SELECT @@datadir`; --echo # --echo # Trigger a MyISAM system error during an INFORMATION_SCHEMA.TABLES query --echo # +--replace_result 20 2 SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, ENGINE, ROW_FORMAT, TABLE_ROWS, DATA_LENGTH, TABLE_COMMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; diff --git a/mysql-test/suite/funcs_2/t/innodb_charset.test b/mysql-test/suite/funcs_2/t/innodb_charset.test index b77bacfc01c..da4dea44ad7 100644 --- a/mysql-test/suite/funcs_2/t/innodb_charset.test +++ b/mysql-test/suite/funcs_2/t/innodb_charset.test @@ -6,6 +6,7 @@ # Checking of other prerequisites is in charset_master.test # ################################################################################ +--source include/no_valgrind_without_big.inc --source include/have_innodb.inc let $engine_type= InnoDB; diff --git a/mysql-test/suite/innodb/r/alter_key_block_size-11757.result b/mysql-test/suite/innodb/r/alter_key_block_size-11757.result new file mode 100644 index 00000000000..400a3d0df3c --- /dev/null +++ b/mysql-test/suite/innodb/r/alter_key_block_size-11757.result @@ -0,0 +1,47 @@ +set global innodb_file_format=barracuda; +create table t1 ( +id1 bigint(20) not null, +id2 bigint(20) not null, +primary key (id1), +unique key id2 (id2) +) engine=innodb row_format=compressed key_block_size=8; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`), + UNIQUE KEY `id2` (`id2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8 +alter table t1 row_format=dynamic; +Warnings: +Warning 1478 InnoDB: ignoring KEY_BLOCK_SIZE=8 unless ROW_FORMAT=COMPRESSED. +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`), + UNIQUE KEY `id2` (`id2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8 +alter table t1 key_block_size=0; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`) KEY_BLOCK_SIZE=8, + UNIQUE KEY `id2` (`id2`) KEY_BLOCK_SIZE=8 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC +alter table t1 drop primary key, add primary key (id1), +drop key id2, add unique (id2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`), + UNIQUE KEY `id2` (`id2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC +drop table t1; +set global innodb_file_format=default; diff --git a/mysql-test/suite/innodb/r/autoinc_debug.result b/mysql-test/suite/innodb/r/autoinc_debug.result new file mode 100644 index 00000000000..b4b5e23e057 --- /dev/null +++ b/mysql-test/suite/innodb/r/autoinc_debug.result @@ -0,0 +1,90 @@ +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY)ENGINE=INNODB; +# SETTING auto_increment_increment IN CONNECTION DEFAULT +SET AUTO_INCREMENT_INCREMENT = 1; +INSERT INTO t1 VALUES(NULL); +SELECT * FROM t1; +id +1 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1 +# SETTING auto_increment_increment IN CONNECTION1 +SET AUTO_INCREMENT_INCREMENT = 2; +SET DEBUG_SYNC= 'ib_after_row_insert SIGNAL opened WAIT_FOR flushed1'; +INSERT INTO t1 VALUES(NULL); +SET AUTO_INCREMENT_INCREMENT = 2; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +SET DEBUG_SYNC= 'ib_after_row_insert_step SIGNAL flushed1 WAIT_FOR opened1'; +insert into t1 values(NULL); +SELECT * FROM t1; +id +1 +3 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=latin1 +SET DEBUG_SYNC= 'now SIGNAL opened1'; +SELECT * FROM t1; +id +1 +3 +5 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1(id INT AUTO_INCREMENT PRIMARY KEY)ENGINE=INNODB; +# SETTING auto_increment_increment IN CONNECTION DEFAULT +SET AUTO_INCREMENT_INCREMENT = 1; +INSERT INTO t1 VALUES(NULL); +SELECT * FROM t1; +id +1 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1 +SET DEBUG_SYNC = 'now SIGNAL flushed'; +# SETTING auto_increment_increment in connection1 +SET AUTO_INCREMENT_INCREMENT = 2; +SET DEBUG_SYNC= 'now WAIT_FOR flushed'; +SET DEBUG_SYNC= 'ib_after_row_insert SIGNAL opened WAIT_FOR flushed1'; +INSERT INTO t1 values(NULL); +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +SET DEBUG_SYNC= 'ib_after_row_insert_step SIGNAL flushed1 WAIT_FOR opened1'; +INSERT INTO t1 VALUES(NULL); +SELECT * FROM t1; +id +1 +3 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=latin1 +SET DEBUG_SYNC= 'now SIGNAL opened1'; +SELECT * FROM t1; +id +1 +3 +5 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=latin1 +DROP TABLE t1; +SET DEBUG_SYNC='RESET'; diff --git a/mysql-test/suite/innodb/r/innodb-alter-debug.result b/mysql-test/suite/innodb/r/innodb-alter-debug.result new file mode 100644 index 00000000000..78976030ac8 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-alter-debug.result @@ -0,0 +1,56 @@ +SET NAMES utf8; +CREATE TABLE â‘ ( +c1 INT PRIMARY KEY, c2 INT DEFAULT 1, ct TEXT, INDEX(c2)) +ENGINE = InnoDB; +CREATE TABLE t1ć (c1 INT PRIMARY KEY, c2 INT, INDEX(c2), +CONSTRAINT t1c2 FOREIGN KEY (c2) REFERENCES â‘ (c2)) +ENGINE=InnoDB; +INSERT INTO â‘ SET c1 = 1; +SET @saved_debug_dbug = @@SESSION.debug_dbug; +SET DEBUG_DBUG = '+d,ib_drop_foreign_error'; +ALTER TABLE t1ć DROP FOREIGN KEY t1c2, RENAME TO â‘¡; +ERROR HY000: The table 't1ć' is full +SET DEBUG_DBUG = @saved_debug_dbug; +SET DEBUG_DBUG = '+d,ib_rename_column_error'; +ALTER TABLE â‘ CHANGE c2 Å¡ INT; +ERROR HY000: The table 'â‘ ' is full +SET DEBUG_DBUG = @saved_debug_dbug; +SHOW CREATE TABLE t1ć; +Table Create Table +t1ć CREATE TABLE `t1ć` ( + `c1` int(11) NOT NULL, + `c2` int(11) DEFAULT NULL, + PRIMARY KEY (`c1`), + KEY `c2` (`c2`), + CONSTRAINT `t1c2` FOREIGN KEY (`c2`) REFERENCES `â‘ ` (`c2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +DROP TABLE t1ć, â‘ ; +# +# Bug #21364096 THE BOGUS DUPLICATE KEY ERROR IN ONLINE DDL +# WITH INCORRECT KEY NAME +create table t1 (id int auto_increment primary key, a int, unique key uk(a)) +engine = innodb; +insert into t1 select 1, 1; +insert into t1 select 2, 2; +SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL s1 WAIT_FOR s2'; +alter table t1 add b int, ALGORITHM=inplace; +/* connection con1 */ +SET DEBUG_SYNC = 'now WAIT_FOR s1'; +insert into t1 select NULL, 1; +ERROR 23000: Duplicate entry '1' for key 'uk' +SET DEBUG_SYNC = 'now SIGNAL s2'; +/* connection default */ +/* reap */ alter table t1 add b int, ALGORITHM=inplace; +ERROR 23000: Duplicate entry '1' for key 'uk' +SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL s1 WAIT_FOR s2'; +alter table t1 add b int, ALGORITHM=inplace;; +/* connection con1 */ +set DEBUG_SYNC = 'now WAIT_FOR s1'; +update t1 set a=1 where id=2; +ERROR 23000: Duplicate entry '1' for key 'uk' +SET DEBUG_SYNC = 'now SIGNAL s2'; +/* connection default */ +/* reap */ alter table t1 add b int, ALGORITHM=inplace; +ERROR 23000: Duplicate entry '1' for key 'uk' +SET DEBUG_SYNC = 'RESET'; +drop table t1; diff --git a/mysql-test/suite/innodb/r/innodb-alter-nullable.result b/mysql-test/suite/innodb/r/innodb-alter-nullable.result new file mode 100644 index 00000000000..e9711b2ac31 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-alter-nullable.result @@ -0,0 +1,53 @@ +CREATE TABLE t (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 INT) ENGINE=InnoDB; +INSERT INTO t VALUES (1,2,3),(4,5,6),(7,8,9); +ALTER TABLE t CHANGE c1 c1 INT NULL FIRST, ALGORITHM=INPLACE; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +set @old_sql_mode = @@sql_mode; +set @@sql_mode = 'STRICT_TRANS_TABLES'; +ALTER TABLE t MODIFY c3 INT NOT NULL, ALGORITHM=INPLACE; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +set @@sql_mode = @old_sql_mode; +ALTER TABLE t CHANGE c2 c2 INT, CHANGE c2 c2 INT NOT NULL; +ERROR 42S22: Unknown column 'c2' in 't' +ALTER TABLE t MODIFY c2 INT, MODIFY c2 INT NOT NULL; +ERROR 42S22: Unknown column 'c2' in 't' +ALTER TABLE t MODIFY c2 INT UNSIGNED, MODIFY c2 INT; +ERROR 42S22: Unknown column 'c2' in 't' +ALTER TABLE t MODIFY c2 CHAR(1) NOT NULL, MODIFY c2 INT NOT NULL; +ERROR 42S22: Unknown column 'c2' in 't' +ALTER TABLE t CHANGE c2 c2 INT NOT NULL; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +ALTER TABLE t MODIFY c2 INT NOT NULL; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SET SQL_MODE='STRICT_ALL_TABLES'; +UPDATE t SET c2=NULL; +ERROR 23000: Column 'c2' cannot be null +SELECT * FROM t; +c1 c2 c3 +1 2 3 +4 5 6 +7 8 9 +ALTER TABLE t MODIFY c2 INT, ALGORITHM=INPLACE; +BEGIN; +UPDATE t SET c2=NULL; +SELECT * FROM t; +c1 c2 c3 +1 NULL 3 +4 NULL 6 +7 NULL 9 +ROLLBACK; +SELECT * FROM t; +c1 c2 c3 +1 2 3 +4 5 6 +7 8 9 +ALTER TABLE t MODIFY c2 INT NULL, ALGORITHM=INPLACE; +SELECT * FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES +WHERE NAME='test/t'; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE +# test/t 1 6 # Antelope Compact 0 +DROP TABLE t; diff --git a/mysql-test/suite/innodb/r/innodb-get-fk.result b/mysql-test/suite/innodb/r/innodb-get-fk.result index 0dca82c2d10..f34fb8bcb67 100644 --- a/mysql-test/suite/innodb/r/innodb-get-fk.result +++ b/mysql-test/suite/innodb/r/innodb-get-fk.result @@ -26,7 +26,8 @@ KEY `fk_crewRoleAssigned_roleCode` (`role_code`), CONSTRAINT `fk_crewRoleAssigned_crewId` FOREIGN KEY (`crew_id`) REFERENCES `repro`.`crew` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT `fk_crewRoleAssigned_pilotId` FOREIGN KEY (`crew_id`) REFERENCES `repro`.`pilot` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB COMMENT="This is a comment about tables"; -# Restart mysqld --innodb_read_only_mode=1 +ALTER TABLE `repro`.`crew_role_assigned` COMMENT = 'innodb_read_only'; +ERROR HY000: Can't lock file (errno: 165 "Table is read only") SHOW CREATE TABLE `repro`.`crew_role_assigned`; Table Create Table crew_role_assigned CREATE TABLE `crew_role_assigned` ( @@ -52,7 +53,6 @@ crew_role_assigned CREATE TABLE `crew_role_assigned` ( CONSTRAINT `fk_crewRoleAssigned_crewId` FOREIGN KEY (`crew_id`) REFERENCES `crew` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT `fk_crewRoleAssigned_pilotId` FOREIGN KEY (`crew_id`) REFERENCES `pilot` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=latin1 COMMENT='This is a new comment about tables' -# Restart mysqld --innodb_read_only_mode=1 SHOW CREATE TABLE `repro`.`crew_role_assigned`; Table Create Table crew_role_assigned CREATE TABLE `crew_role_assigned` ( diff --git a/mysql-test/suite/innodb/r/innodb-index-online-norebuild.result b/mysql-test/suite/innodb/r/innodb-index-online-norebuild.result new file mode 100644 index 00000000000..b9077643870 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-index-online-norebuild.result @@ -0,0 +1,26 @@ +# INPLACE ALTER WITH INPLACE_IGNORE FLAG AND CHANGE CREATE OPTION +# CHANGE THE COLUMN DEFAULT (INPLACE_IGNORE) +# AND TABLE CHARSET(CHANGE CREATE) +CREATE TABLE t1( +id INT PRIMARY KEY, +f1 INT NOT NULL DEFAULT 0)ENGINE=INNODB; +INSERT INTO t1 VALUES(1, 2); +SET SQL_MODE='STRICT_ALL_TABLES'; +ALTER TABLE t1 MODIFY COLUMN f1 INT NOT NULL DEFAULT 0, +DEFAULT CHARSET=latin1, ALGORITHM=INPLACE; +DROP TABLE t1; +# CHANGE THE COMMENT OF COLUMN(INPLACE IGNORE) +# AND TABLE CHARSET(CHANGE CREATE) +CREATE TABLE t1(id INT COMMENT 'independence day')ENGINE=INNODB; +INSERT INTO t1 values(1); +ALTER TABLE t1 MODIFY COLUMN id INT COMMENT 'identifier', +DEFAULT CHARSET=latin1, ALGORITHM=INPLACE; +DROP TABLE t1; +# RENAME THE TABLE(INPLACE IGNORE) +# AND CHANGE TABLE CHARSET(CHANGE CREATE) +CREATE TABLE t1( +f1 INT NOT NULL, +f2 INT NOT NULL)ENGINE=INNODB; +INSERT INTO t1 VALUES(1, 2); +ALTER TABLE t1 RENAME t2, DEFAULT CHARSET=latin1, ALGORITHM=INPLACE; +DROP TABLE t2; diff --git a/mysql-test/suite/innodb/r/innodb-wl5522,xtradb.rdiff b/mysql-test/suite/innodb/r/innodb-wl5522,xtradb.rdiff deleted file mode 100644 index 4aea0b451ec..00000000000 --- a/mysql-test/suite/innodb/r/innodb-wl5522,xtradb.rdiff +++ /dev/null @@ -1,56 +0,0 @@ ---- suite/innodb/r/innodb-wl5522.result -+++ suite/innodb/r/innodb-wl5522.reject -@@ -580,7 +580,7 @@ - ERROR HY000: Tablespace has been discarded for table 't1' - restore: t1 .ibd and .cfg files - ALTER TABLE t1 IMPORT TABLESPACE; --ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x0) -+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x0) - unlink: t1.ibd - unlink: t1.cfg - DROP TABLE t1; -@@ -592,7 +592,7 @@ - ERROR HY000: Tablespace has been discarded for table 't1' - restore: t1 .ibd and .cfg files - ALTER TABLE t1 IMPORT TABLESPACE; --ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x0) -+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x0) - unlink: t1.ibd - unlink: t1.cfg - DROP TABLE t1; -@@ -766,7 +766,7 @@ - ERROR HY000: Tablespace has been discarded for table 't1' - restore: t1 .ibd and .cfg files - ALTER TABLE t1 IMPORT TABLESPACE; --ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x1) -+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x1) - unlink: t1.ibd - unlink: t1.cfg - DROP TABLE t1; -@@ -778,7 +778,7 @@ - ERROR HY000: Tablespace has been discarded for table 't1' - restore: t1 .ibd and .cfg files - ALTER TABLE t1 IMPORT TABLESPACE; --ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x1) -+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x1) - unlink: t1.ibd - unlink: t1.cfg - DROP TABLE t1; -@@ -955,7 +955,7 @@ - ERROR HY000: Tablespace has been discarded for table 't1' - restore: t1 .ibd and .cfg files - ALTER TABLE t1 IMPORT TABLESPACE; --ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x21) -+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x21) - unlink: t1.ibd - unlink: t1.cfg - DROP TABLE t1; -@@ -967,7 +967,7 @@ - ERROR HY000: Tablespace has been discarded for table 't1' - restore: t1 .ibd and .cfg files - ALTER TABLE t1 IMPORT TABLESPACE; --ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x21) -+ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x21) - unlink: t1.ibd - unlink: t1.cfg - DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/innodb-wl5522.result b/mysql-test/suite/innodb/r/innodb-wl5522.result index fb4ac37b9fd..2116dfbf3fa 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522.result @@ -580,7 +580,7 @@ SELECT * FROM t1; ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x0) +ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x0) unlink: t1.ibd unlink: t1.cfg DROP TABLE t1; @@ -592,7 +592,7 @@ SELECT * FROM t1; ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x0) +ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x0) unlink: t1.ibd unlink: t1.cfg DROP TABLE t1; @@ -766,7 +766,7 @@ SELECT * FROM t1; ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x1) +ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x1) unlink: t1.ibd unlink: t1.cfg DROP TABLE t1; @@ -778,7 +778,7 @@ SELECT * FROM t1; ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x1) +ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x21 and the meta-data file has 0x1) unlink: t1.ibd unlink: t1.cfg DROP TABLE t1; @@ -955,7 +955,7 @@ SELECT * FROM t1; ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x21) +ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x1 and the meta-data file has 0x21) unlink: t1.ibd unlink: t1.cfg DROP TABLE t1; @@ -967,7 +967,7 @@ SELECT * FROM t1; ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files ALTER TABLE t1 IMPORT TABLESPACE; -ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x5 and the meta-data file has 0x21) +ERROR HY000: Schema mismatch (Table flags don't match, server table has 0x0 and the meta-data file has 0x21) unlink: t1.ibd unlink: t1.cfg DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/innodb_blob_unrecoverable_crash.result b/mysql-test/suite/innodb/r/innodb_blob_unrecoverable_crash.result index 9f6b7ca6a23..d69359e8978 100644 --- a/mysql-test/suite/innodb/r/innodb_blob_unrecoverable_crash.result +++ b/mysql-test/suite/innodb/r/innodb_blob_unrecoverable_crash.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: The total blob data length"); SET GLOBAL max_allowed_packet = 100*1024*1024; # Connection big_packets: CREATE TABLE t1 (a BIGINT PRIMARY KEY, b LONGBLOB) ENGINE=InnoDB; @@ -10,9 +9,7 @@ INSERT INTO t1 (a, b) VALUES (5, '5'); start transaction; INSERT INTO t1 (a, b) VALUES (6, REPEAT('a', 20*1024*1024)); ERROR 42000: The size of BLOB/TEXT data inserted in one transaction is greater than 10% of redo log size. Increase the redo log size using innodb_log_file_size. -# Connection default: -# Quick shutdown and restart server -# Connection default: +# Kill and restart SELECT a FROM t1; a 1 diff --git a/mysql-test/suite/innodb/r/innodb_bug14676111.result b/mysql-test/suite/innodb/r/innodb_bug14676111.result index c2fdfee5522..fd73ca298fe 100644 --- a/mysql-test/suite/innodb/r/innodb_bug14676111.result +++ b/mysql-test/suite/innodb/r/innodb_bug14676111.result @@ -1,12 +1,13 @@ -drop table if exists t1; -call mtr.add_suppression("option 'innodb-purge-threads': unsigned value 0 adjusted to*"); -set global innodb_stats_persistent = false; -CREATE TABLE t1 (a int not null primary key) engine=InnoDB; +set @old_innodb_limit_optimistic_insert_debug = @@innodb_limit_optimistic_insert_debug; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB STATS_PERSISTENT=0; set global innodb_limit_optimistic_insert_debug = 2; insert into t1 values (1); +begin; insert into t1 values (5); +begin; insert into t1 values (4); insert into t1 values (3); +begin; insert into t1 values (2); analyze table t1; Table Op Msg_type Msg_text @@ -14,48 +15,40 @@ test.t1 analyze status OK select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; CLUST_INDEX_SIZE 10 -delete from t1 where a=4; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; +rollback; analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status OK select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; CLUST_INDEX_SIZE 8 -delete from t1 where a=5; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; +rollback; analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status OK select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; CLUST_INDEX_SIZE 5 -set global innodb_limit_optimistic_insert_debug = 0; -delete from t1 where a=2; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; +set global innodb_limit_optimistic_insert_debug = 10000; +rollback; analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status OK select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; CLUST_INDEX_SIZE 3 +begin; insert into t1 values (2); -delete from t1 where a=2; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; +rollback; analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status OK select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; CLUST_INDEX_SIZE 2 +begin; insert into t1 values (2); -delete from t1 where a=2; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; +rollback; analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status OK @@ -63,3 +56,4 @@ select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME CLUST_INDEX_SIZE 1 drop table t1; +set global innodb_limit_optimistic_insert_debug = @old_innodb_limit_optimistic_insert_debug; diff --git a/mysql-test/suite/innodb/r/innodb_bug59641.result b/mysql-test/suite/innodb/r/innodb_bug59641.result index 5062c69558b..f4c2199ae88 100644 --- a/mysql-test/suite/innodb/r/innodb_bug59641.result +++ b/mysql-test/suite/innodb/r/innodb_bug59641.result @@ -1,5 +1,3 @@ -call mtr.add_suppression("Found 3 prepared XA transactions"); -flush tables; CREATE TABLE t(a INT PRIMARY KEY, b INT)ENGINE=InnoDB; INSERT INTO t VALUES(2,2),(4,4),(8,8),(16,16),(32,32); COMMIT; @@ -16,6 +14,7 @@ XA START '789'; UPDATE t SET b=4*a WHERE a=32; XA END '789'; XA PREPARE '789'; +# Kill and restart SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM t; a b diff --git a/mysql-test/suite/innodb/r/log_file_size.result b/mysql-test/suite/innodb/r/log_file_size.result new file mode 100644 index 00000000000..621176d7879 --- /dev/null +++ b/mysql-test/suite/innodb/r/log_file_size.result @@ -0,0 +1,72 @@ +call mtr.add_suppression("mysqld got signal 11"); +call mtr.add_suppression("Attempting backtrace"); +FLUSH TABLES; +CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +BEGIN; +INSERT INTO t1 VALUES (42); +# Kill and restart: --innodb-log-file-size=6M +SELECT * FROM t1; +a +INSERT INTO t1 VALUES (42); +BEGIN; +DELETE FROM t1; +# Kill and restart: --innodb-log-files-in-group=3 --innodb-log-file-size=5M +SELECT * FROM t1; +a +42 +INSERT INTO t1 VALUES (123); +BEGIN; +DELETE FROM t1; +# Kill the server +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /syntax error in innodb_log_group_home_dir/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Starting crash recovery from checkpoint LSN=/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Cannot create log files in read-only mode/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Setting log file .*ib_logfile[0-9]+ size to/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Setting log file .*ib_logfile[0-9]+ size to/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Only one log file found/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Log file .*ib_logfile1 is of different size 1048576 bytes than other log files/ in mysqld.1.err +SELECT * FROM t1; +ERROR 42000: Unknown storage engine 'InnoDB' +FOUND /InnoDB: Setting log file .*ib_logfile[0-9]+ size to/ in mysqld.1.err +FOUND /InnoDB: Renaming log file .*ib_logfile101 to .*ib_logfile0/ in mysqld.1.err +SELECT * FROM t1; +a +42 +123 +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/read_only_recovery.result b/mysql-test/suite/innodb/r/read_only_recovery.result new file mode 100644 index 00000000000..c54c3b5ab7f --- /dev/null +++ b/mysql-test/suite/innodb/r/read_only_recovery.result @@ -0,0 +1,29 @@ +CREATE TABLE t(a INT PRIMARY KEY) ENGINE=InnoDB; +BEGIN; +INSERT INTO t VALUES(1),(2); +DELETE FROM t WHERE a=2; +# Normal MariaDB shutdown would roll back the above transaction. +# We want the transaction to remain open, so we will kill the server +# after ensuring that any non-transactional files are clean. +FLUSH TABLES; +# Ensure that the above incomplete transaction becomes durable. +SET GLOBAL innodb_flush_log_at_trx_commit=1; +BEGIN; +INSERT INTO t VALUES(0); +ROLLBACK; +# Kill and restart: --innodb-force-recovery=3 +SELECT * FROM t; +a +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT * FROM t; +a +1 +SELECT * FROM t; +a +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT * FROM t; +a +1 +SELECT * FROM t; +a +DROP TABLE t; diff --git a/mysql-test/suite/innodb/r/xa_recovery.result b/mysql-test/suite/innodb/r/xa_recovery.result index 84cb37ef7c9..4441701f961 100644 --- a/mysql-test/suite/innodb/r/xa_recovery.result +++ b/mysql-test/suite/innodb/r/xa_recovery.result @@ -4,7 +4,7 @@ XA START 'x'; UPDATE t1 set a=2; XA END 'x'; XA PREPARE 'x'; -call mtr.add_suppression("Found 1 prepared XA transactions"); +# Kill and restart SELECT * FROM t1 LOCK IN SHARE MODE; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM t1; diff --git a/mysql-test/suite/innodb/t/alter_key_block_size-11757.test b/mysql-test/suite/innodb/t/alter_key_block_size-11757.test new file mode 100644 index 00000000000..50a909870ba --- /dev/null +++ b/mysql-test/suite/innodb/t/alter_key_block_size-11757.test @@ -0,0 +1,23 @@ +# +# MDEV-11757 KEY_BLOCK_SIZE strangeness when UNCOMPRESSing COMPRESSed InnoDB tables +# +source include/have_innodb.inc; +set global innodb_file_format=barracuda; + +create table t1 ( + id1 bigint(20) not null, + id2 bigint(20) not null, + primary key (id1), + unique key id2 (id2) +) engine=innodb row_format=compressed key_block_size=8; +show create table t1; +alter table t1 row_format=dynamic; +show create table t1; +alter table t1 key_block_size=0; +show create table t1; +alter table t1 drop primary key, add primary key (id1), + drop key id2, add unique (id2); +show create table t1; +drop table t1; + +set global innodb_file_format=default; diff --git a/mysql-test/suite/innodb/t/autoinc_debug.test b/mysql-test/suite/innodb/t/autoinc_debug.test new file mode 100644 index 00000000000..2e662565490 --- /dev/null +++ b/mysql-test/suite/innodb/t/autoinc_debug.test @@ -0,0 +1,85 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/not_embedded.inc + +# Two parallel connection with autoinc column after restart. + +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY)ENGINE=INNODB; + +--echo # SETTING auto_increment_increment IN CONNECTION DEFAULT +SET AUTO_INCREMENT_INCREMENT = 1; +INSERT INTO t1 VALUES(NULL); +SELECT * FROM t1; +SHOW CREATE TABLE t1; + +--source include/restart_mysqld.inc + +--echo # SETTING auto_increment_increment IN CONNECTION1 +SET AUTO_INCREMENT_INCREMENT = 2; + +SET DEBUG_SYNC= 'ib_after_row_insert SIGNAL opened WAIT_FOR flushed1'; + +SEND INSERT INTO t1 VALUES(NULL); + +connect(con1, localhost, root,,); +SET AUTO_INCREMENT_INCREMENT = 2; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +SET DEBUG_SYNC= 'ib_after_row_insert_step SIGNAL flushed1 WAIT_FOR opened1'; +send insert into t1 values(NULL); + +connection default; +reap; +SELECT * FROM t1; +SHOW CREATE TABLE t1; +SET DEBUG_SYNC= 'now SIGNAL opened1'; + +connection con1; +reap; +SELECT * FROM t1; +SHOW CREATE TABLE t1; +connection default; +disconnect con1; + +DROP TABLE t1; + +# Two parallel connection with autoinc column without restart. + +CREATE TABLE t1(id INT AUTO_INCREMENT PRIMARY KEY)ENGINE=INNODB; + +--echo # SETTING auto_increment_increment IN CONNECTION DEFAULT +SET AUTO_INCREMENT_INCREMENT = 1; +INSERT INTO t1 VALUES(NULL); +SELECT * FROM t1; +SHOW CREATE TABLE t1; +SET DEBUG_SYNC = 'now SIGNAL flushed'; + +connect(con1, localhost, root,,); + +--echo # SETTING auto_increment_increment in connection1 +SET AUTO_INCREMENT_INCREMENT = 2; + +SET DEBUG_SYNC= 'now WAIT_FOR flushed'; +SET DEBUG_SYNC= 'ib_after_row_insert SIGNAL opened WAIT_FOR flushed1'; + +send INSERT INTO t1 values(NULL); + +connection default; + +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +SET DEBUG_SYNC= 'ib_after_row_insert_step SIGNAL flushed1 WAIT_FOR opened1'; + +send INSERT INTO t1 VALUES(NULL); + +connection con1; +reap; +SELECT * FROM t1; +SHOW CREATE TABLE t1; +SET DEBUG_SYNC= 'now SIGNAL opened1'; +disconnect con1; + +connection default; +reap; +SELECT * FROM t1; +SHOW CREATE TABLE t1; +DROP TABLE t1; +SET DEBUG_SYNC='RESET'; diff --git a/mysql-test/suite/innodb/t/innodb-alter-debug.test b/mysql-test/suite/innodb/t/innodb-alter-debug.test new file mode 100644 index 00000000000..70017ffba35 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-alter-debug.test @@ -0,0 +1,79 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc + +--source include/count_sessions.inc + +SET NAMES utf8; + +CREATE TABLE â‘ ( + c1 INT PRIMARY KEY, c2 INT DEFAULT 1, ct TEXT, INDEX(c2)) +ENGINE = InnoDB; + +CREATE TABLE t1ć (c1 INT PRIMARY KEY, c2 INT, INDEX(c2), + CONSTRAINT t1c2 FOREIGN KEY (c2) REFERENCES â‘ (c2)) +ENGINE=InnoDB; + +INSERT INTO â‘ SET c1 = 1; + +SET @saved_debug_dbug = @@SESSION.debug_dbug; +SET DEBUG_DBUG = '+d,ib_drop_foreign_error'; +--error ER_RECORD_FILE_FULL +ALTER TABLE t1ć DROP FOREIGN KEY t1c2, RENAME TO â‘¡; +SET DEBUG_DBUG = @saved_debug_dbug; + +SET DEBUG_DBUG = '+d,ib_rename_column_error'; +--error ER_RECORD_FILE_FULL +ALTER TABLE â‘ CHANGE c2 Å¡ INT; +SET DEBUG_DBUG = @saved_debug_dbug; + +SHOW CREATE TABLE t1ć; + +DROP TABLE t1ć, â‘ ; + +--echo # +--echo # Bug #21364096 THE BOGUS DUPLICATE KEY ERROR IN ONLINE DDL +--echo # WITH INCORRECT KEY NAME + +create table t1 (id int auto_increment primary key, a int, unique key uk(a)) +engine = innodb; +insert into t1 select 1, 1; +insert into t1 select 2, 2; +SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL s1 WAIT_FOR s2'; +--send alter table t1 add b int, ALGORITHM=inplace + +--echo /* connection con1 */ +connect (con1,localhost,root,,); +SET DEBUG_SYNC = 'now WAIT_FOR s1'; +--error ER_DUP_ENTRY +insert into t1 select NULL, 1; +SET DEBUG_SYNC = 'now SIGNAL s2'; + +--echo /* connection default */ +connection default; +--echo /* reap */ alter table t1 add b int, ALGORITHM=inplace; +--error ER_DUP_ENTRY +--reap + +SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL s1 WAIT_FOR s2'; +--send alter table t1 add b int, ALGORITHM=inplace; + +--echo /* connection con1 */ +connection con1; +set DEBUG_SYNC = 'now WAIT_FOR s1'; +--error ER_DUP_ENTRY +update t1 set a=1 where id=2; +SET DEBUG_SYNC = 'now SIGNAL s2'; +disconnect con1; + +--echo /* connection default */ +connection default; +--echo /* reap */ alter table t1 add b int, ALGORITHM=inplace; +--error ER_DUP_ENTRY +--reap +SET DEBUG_SYNC = 'RESET'; + +drop table t1; + +# Wait till all disconnects are completed +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/suite/innodb/t/innodb-alter-nullable.test b/mysql-test/suite/innodb/t/innodb-alter-nullable.test new file mode 100644 index 00000000000..3f1e82b3183 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-alter-nullable.test @@ -0,0 +1,76 @@ +--source include/have_innodb.inc + +# Save the initial number of concurrent sessions. +--source include/count_sessions.inc + +CREATE TABLE t (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 INT) ENGINE=InnoDB; +INSERT INTO t VALUES (1,2,3),(4,5,6),(7,8,9); + +--enable_info +# This one will be a no-op. +# MySQL should perhaps issue an error, because it refuses to modify +# the PRIMARY KEY column c1 from NOT NULL to NULL. +ALTER TABLE t CHANGE c1 c1 INT NULL FIRST, ALGORITHM=INPLACE; + +# NULL -> NOT NULL only allowed INPLACE if strict sql_mode is on. +--disable_info +set @old_sql_mode = @@sql_mode; +set @@sql_mode = 'STRICT_TRANS_TABLES'; +--enable_info +ALTER TABLE t MODIFY c3 INT NOT NULL, ALGORITHM=INPLACE; +--disable_info +set @@sql_mode = @old_sql_mode; +--enable_info + +# Request some conflicting changes for a single column. +--error ER_BAD_FIELD_ERROR +ALTER TABLE t CHANGE c2 c2 INT, CHANGE c2 c2 INT NOT NULL; +--error ER_BAD_FIELD_ERROR +ALTER TABLE t MODIFY c2 INT, MODIFY c2 INT NOT NULL; +--error ER_BAD_FIELD_ERROR +ALTER TABLE t MODIFY c2 INT UNSIGNED, MODIFY c2 INT; +--error ER_BAD_FIELD_ERROR +ALTER TABLE t MODIFY c2 CHAR(1) NOT NULL, MODIFY c2 INT NOT NULL; + +# No-ops. +ALTER TABLE t CHANGE c2 c2 INT NOT NULL; +ALTER TABLE t MODIFY c2 INT NOT NULL; +--disable_info + +connect (con1,localhost,root,,); +connection con1; + +SET SQL_MODE='STRICT_ALL_TABLES'; + +--error ER_BAD_NULL_ERROR +UPDATE t SET c2=NULL; + +SELECT * FROM t; + +connection default; + +# This should change the column to NULL. +ALTER TABLE t MODIFY c2 INT, ALGORITHM=INPLACE; + +connection con1; +BEGIN; +UPDATE t SET c2=NULL; +SELECT * FROM t; +ROLLBACK; +SELECT * FROM t; + +disconnect con1; +connection default; + +# This should be no-op. +ALTER TABLE t MODIFY c2 INT NULL, ALGORITHM=INPLACE; + +--replace_column 1 # 5 # +SELECT * FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES +WHERE NAME='test/t'; + +DROP TABLE t; + +# Check that all connections opened by test cases in this file are really +# gone so execution of other tests won't be affected by their presence. +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/suite/innodb/t/innodb-get-fk.test b/mysql-test/suite/innodb/t/innodb-get-fk.test index 4245bef289f..c1ab54fab45 100644 --- a/mysql-test/suite/innodb/t/innodb-get-fk.test +++ b/mysql-test/suite/innodb/t/innodb-get-fk.test @@ -33,19 +33,20 @@ CONSTRAINT `fk_crewRoleAssigned_crewId` FOREIGN KEY (`crew_id`) REFERENCES `repr CONSTRAINT `fk_crewRoleAssigned_pilotId` FOREIGN KEY (`crew_id`) REFERENCES `repro`.`pilot` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB COMMENT="This is a comment about tables"; ---echo # Restart mysqld --innodb_read_only_mode=1 --- let $restart_parameters=--innodb-read-only-mode=1 +-- let $restart_parameters=--innodb-read-only -- source include/restart_mysqld.inc +--error ER_CANT_LOCK +ALTER TABLE `repro`.`crew_role_assigned` COMMENT = 'innodb_read_only'; SHOW CREATE TABLE `repro`.`crew_role_assigned`; +-- let $restart_parameters= -- source include/restart_mysqld.inc ALTER TABLE `repro`.`crew_role_assigned` COMMENT = "This is a new comment about tables"; SHOW CREATE TABLE `repro`.`crew_role_assigned`; ---echo # Restart mysqld --innodb_read_only_mode=1 --- let $restart_parameters=--innodb-read-only-mode=1 +-- let $restart_parameters=--innodb-read-only -- source include/restart_mysqld.inc # @@ -53,6 +54,7 @@ SHOW CREATE TABLE `repro`.`crew_role_assigned`; # SHOW CREATE TABLE `repro`.`crew_role_assigned`; +-- let $restart_parameters= -- source include/restart_mysqld.inc DROP TABLE `repro`.`crew_role_assigned`; diff --git a/mysql-test/suite/innodb/t/innodb-index-online-norebuild.opt b/mysql-test/suite/innodb/t/innodb-index-online-norebuild.opt new file mode 100644 index 00000000000..39b93371503 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-index-online-norebuild.opt @@ -0,0 +1 @@ +--loose-innodb-sys-tables diff --git a/mysql-test/suite/innodb/t/innodb-index-online-norebuild.test b/mysql-test/suite/innodb/t/innodb-index-online-norebuild.test new file mode 100644 index 00000000000..518b4efe1df --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-index-online-norebuild.test @@ -0,0 +1,71 @@ +--source include/have_innodb.inc + +--echo # INPLACE ALTER WITH INPLACE_IGNORE FLAG AND CHANGE CREATE OPTION + +--echo # CHANGE THE COLUMN DEFAULT (INPLACE_IGNORE) +--echo # AND TABLE CHARSET(CHANGE CREATE) + +CREATE TABLE t1( + id INT PRIMARY KEY, + f1 INT NOT NULL DEFAULT 0)ENGINE=INNODB; + +INSERT INTO t1 VALUES(1, 2); + +let id_before_alter =`SELECT table_id FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE name="test/t1"`; + +# Allow the following operation to report errors. +SET SQL_MODE='STRICT_ALL_TABLES'; +ALTER TABLE t1 MODIFY COLUMN f1 INT NOT NULL DEFAULT 0, + DEFAULT CHARSET=latin1, ALGORITHM=INPLACE; + +let id_after_alter =`SELECT table_id FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE name="test/t1"`; + +if ($id_before_alter != $id_after_alter) +{ + --echo "Table rebuild happened"; +} + +DROP TABLE t1; + +--echo # CHANGE THE COMMENT OF COLUMN(INPLACE IGNORE) +--echo # AND TABLE CHARSET(CHANGE CREATE) + +CREATE TABLE t1(id INT COMMENT 'independence day')ENGINE=INNODB; + +INSERT INTO t1 values(1); + +let id_before_alter =`SELECT table_id FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE name="test/t1"`; + +ALTER TABLE t1 MODIFY COLUMN id INT COMMENT 'identifier', + DEFAULT CHARSET=latin1, ALGORITHM=INPLACE; + +let id_after_alter =`SELECT table_id FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE name="test/t1"`; + +if ($id_before_alter != $id_after_alter) +{ + --echo "Table rebuild happened"; +} + +DROP TABLE t1; + +--echo # RENAME THE TABLE(INPLACE IGNORE) +--echo # AND CHANGE TABLE CHARSET(CHANGE CREATE) + +CREATE TABLE t1( + f1 INT NOT NULL, + f2 INT NOT NULL)ENGINE=INNODB; + +INSERT INTO t1 VALUES(1, 2); + +let id_before_alter =`SELECT table_id FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE name="test/t1"`; + +ALTER TABLE t1 RENAME t2, DEFAULT CHARSET=latin1, ALGORITHM=INPLACE; + +let id_after_alter =`SELECT table_id FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE name="test/t2"`; + +if ($id_before_alter != $id_after_alter) +{ + --echo "Table rebuild happened"; +} + +DROP TABLE t2; diff --git a/mysql-test/suite/innodb/t/innodb_blob_unrecoverable_crash.test b/mysql-test/suite/innodb/t/innodb_blob_unrecoverable_crash.test index 16fb570737d..dc6d2f6f65c 100644 --- a/mysql-test/suite/innodb/t/innodb_blob_unrecoverable_crash.test +++ b/mysql-test/suite/innodb/t/innodb_blob_unrecoverable_crash.test @@ -2,11 +2,13 @@ --source include/not_crashrep.inc --source include/have_innodb.inc -call mtr.add_suppression("InnoDB: The total blob data length"); - -let $old_max_allowed_packet = `select @@max_allowed_packet`; SET GLOBAL max_allowed_packet = 100*1024*1024; +--disable_query_log +call mtr.add_suppression("InnoDB: The total blob data length"); +FLUSH TABLES; +--enable_query_log + --echo # Connection big_packets: connect(big_packets,localhost,root,,); connection big_packets; @@ -28,28 +30,12 @@ start transaction; --error ER_TOO_BIG_ROWSIZE INSERT INTO t1 (a, b) VALUES (6, REPEAT('a', 20*1024*1024)); ---echo # Connection default: -connection default; - -# We expect a restart. ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect - ---echo # Quick shutdown and restart server ---shutdown_server 0 - -# Wait for the server to come back up, and reconnect. ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # Connection default: connection default; +--source include/kill_and_restart_mysqld.inc +disconnect big_packets; # We should see (1,2,3,4,5) here. SELECT a FROM t1; # Clean up. DROP TABLE t1; - ---disable_query_log -eval set global max_allowed_packet = $old_max_allowed_packet; ---enable_query_log diff --git a/mysql-test/suite/innodb/t/innodb_bug14676111-master.opt b/mysql-test/suite/innodb/t/innodb_bug14676111-master.opt deleted file mode 100644 index e16b9b0b895..00000000000 --- a/mysql-test/suite/innodb/t/innodb_bug14676111-master.opt +++ /dev/null @@ -1 +0,0 @@ ---loose-innodb-purge-threads=0 diff --git a/mysql-test/suite/innodb/t/innodb_bug14676111.test b/mysql-test/suite/innodb/t/innodb_bug14676111.test index ba04d421fde..3abc574a8d2 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14676111.test +++ b/mysql-test/suite/innodb/t/innodb_bug14676111.test @@ -2,50 +2,40 @@ -- source include/have_innodb.inc -- source include/have_debug.inc +-- source include/count_sessions.inc -if (`select count(*)=0 from information_schema.global_variables where variable_name = 'INNODB_LIMIT_OPTIMISTIC_INSERT_DEBUG'`) -{ - --skip Test requires InnoDB built with UNIV_DEBUG definition. -} - ---disable_query_log set @old_innodb_limit_optimistic_insert_debug = @@innodb_limit_optimistic_insert_debug; -set @old_innodb_stats_persistent = @@innodb_stats_persistent; -set @old_innodb_undo_logs = @@innodb_undo_logs; -# Limit undo segments for stable progress of purge. -set global innodb_undo_logs = 1; ---enable_query_log ---disable_warnings -drop table if exists t1; ---enable_warnings - -call mtr.add_suppression("option 'innodb-purge-threads': unsigned value 0 adjusted to*"); -set global innodb_stats_persistent = false; - -CREATE TABLE t1 (a int not null primary key) engine=InnoDB; +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB STATS_PERSISTENT=0; # # make 4 leveled straight tree # set global innodb_limit_optimistic_insert_debug = 2; insert into t1 values (1); +--connect (con5,localhost,root) +begin; insert into t1 values (5); #current tree form # (1, 5) +--connect (con4,localhost,root) +begin; insert into t1 values (4); #records in a page is limited to 2 artificially. root rise occurs #current tree form # (1, 5) #(1, 4) (5) +--connection default insert into t1 values (3); #current tree form # (1, 5) # (1, 4) (5) #(1, 3) (4) (5) +--connect (con2,localhost,root) +begin; insert into t1 values (2); #current tree form # (1, 5) @@ -53,13 +43,15 @@ insert into t1 values (2); # (1, 3) (4) (5) #(1, 2) (3) (4) (5) +--connection default analyze table t1; select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; -delete from t1 where a=4; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; ---source include/wait_innodb_all_purged.inc +--connection con4 +rollback; +--disconnect con4 +--connection default + #deleting 1 record of 2 records don't cause merge artificially. #current tree form # (1, 5) @@ -70,10 +62,11 @@ set global innodb_purge_run_now=ON; analyze table t1; select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; -delete from t1 where a=5; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; ---source include/wait_innodb_all_purged.inc +--connection con5 +rollback; +--disconnect con5 +--connection default + #deleting 1 record of 2 records don't cause merge artificially. #current tree form # (1) @@ -89,11 +82,12 @@ select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME # #disable the artificial limitation of records in a page -set global innodb_limit_optimistic_insert_debug = 0; -delete from t1 where a=2; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; ---source include/wait_innodb_all_purged.inc +set global innodb_limit_optimistic_insert_debug = 10000; +--connection con2 +rollback; +--disconnect con2 +--connection default + #merge page occurs. and lift up occurs. #current tree form # (1) @@ -103,16 +97,14 @@ set global innodb_purge_run_now=ON; analyze table t1; select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; +begin; insert into t1 values (2); #current tree form # (1) # (1) <- lift up this level next, because it is not root # (1, 2, 3) +rollback; -delete from t1 where a=2; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; ---source include/wait_innodb_all_purged.inc #current tree form # (1) # (1, 3) @@ -120,15 +112,12 @@ set global innodb_purge_run_now=ON; analyze table t1; select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME = 'test/t1'; +begin; insert into t1 values (2); #current tree form # (1) # (1, 2, 3) <- lift up this level next, because the father is root - -delete from t1 where a=2; -set global innodb_purge_stop_now=ON; -set global innodb_purge_run_now=ON; ---source include/wait_innodb_all_purged.inc +rollback; #current tree form # (1, 3) @@ -137,8 +126,5 @@ select CLUST_INDEX_SIZE from information_schema.INNODB_SYS_TABLESTATS where NAME drop table t1; ---disable_query_log set global innodb_limit_optimistic_insert_debug = @old_innodb_limit_optimistic_insert_debug; -set global innodb_stats_persistent = @old_innodb_stats_persistent; -set global innodb_undo_logs = @old_innodb_undo_logs; ---enable_query_log +-- source include/wait_until_count_sessions.inc diff --git a/mysql-test/suite/innodb/t/innodb_bug59641.test b/mysql-test/suite/innodb/t/innodb_bug59641.test index a8d35cd1029..7fa32d2ec35 100644 --- a/mysql-test/suite/innodb/t/innodb_bug59641.test +++ b/mysql-test/suite/innodb/t/innodb_bug59641.test @@ -2,11 +2,11 @@ # Bug #59641 Prepared XA transaction causes shutdown hang after a crash -- source include/not_embedded.inc -# The server would issue this warning on restart. -call mtr.add_suppression("Found 3 prepared XA transactions"); -# Close tables used by other tests (to not get crashed myisam tables) -flush tables; +--disable_query_log +call mtr.add_suppression("Found 3 prepared XA transactions"); +FLUSH TABLES; +--enable_query_log CREATE TABLE t(a INT PRIMARY KEY, b INT)ENGINE=InnoDB; INSERT INTO t VALUES(2,2),(4,4),(8,8),(16,16),(32,32); @@ -17,7 +17,6 @@ XA END '123'; XA PREPARE '123'; CONNECT (con1,localhost,root,,); -CONNECTION con1; XA START '456'; INSERT INTO t VALUES(3,47),(5,67); @@ -26,7 +25,6 @@ XA END '456'; XA PREPARE '456'; CONNECT (con2,localhost,root,,); -CONNECTION con2; XA START '789'; UPDATE t SET b=4*a WHERE a=32; @@ -34,30 +32,13 @@ XA END '789'; XA PREPARE '789'; CONNECT (con3,localhost,root,,); -CONNECTION con3; -# Kill the server without sending a shutdown command --- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- shutdown_server 0 --- source include/wait_until_disconnected.inc - -# Restart the server. --- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- enable_reconnect --- source include/wait_until_connected_again.inc +--source include/kill_and_restart_mysqld.inc SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM t; COMMIT; -# Shut down the server. This would hang because of the bug. --- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- shutdown_server --- source include/wait_until_disconnected.inc - -# Restart the server. --- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- enable_reconnect --- source include/wait_until_connected_again.inc +--source include/restart_mysqld.inc SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM t; diff --git a/mysql-test/suite/innodb/t/log_file_size.test b/mysql-test/suite/innodb/t/log_file_size.test new file mode 100644 index 00000000000..4bae93957e8 --- /dev/null +++ b/mysql-test/suite/innodb/t/log_file_size.test @@ -0,0 +1,215 @@ +# Test resizing the InnoDB redo log. + +--source include/have_innodb.inc + +# Embedded server does not support crashing +--source include/not_embedded.inc +# Avoid CrashReporter popup on Mac +--source include/not_crashrep.inc +# DBUG_EXECUTE_IF is needed +--source include/have_debug.inc + +if (`SELECT @@innodb_log_file_size = 1048576`) { + --skip Test requires innodb_log_file_size>1M. +} + +--disable_query_log +call mtr.add_suppression("InnoDB: Resizing redo log"); +call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files"); +call mtr.add_suppression("InnoDB: New log files created"); +# This message is output by 10.0 and 10.1, not by 10.2 +call mtr.add_suppression("InnoDB: The log sequence number in the ibdata files is higher than the log sequence number in the ib_logfiles"); +call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles"); +call mtr.add_suppression("syntax error in innodb_log_group_home_dir"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); +call mtr.add_suppression("InnoDB: innodb_read_only prevents crash recovery"); +call mtr.add_suppression("InnoDB: Are you sure you are using the right ib_logfiles"); +call mtr.add_suppression("InnoDB: Cannot create log files in read-only mode"); +call mtr.add_suppression("InnoDB: Only one log file found"); +call mtr.add_suppression("InnoDB: Log file .*ib_logfile[01].* size"); +call mtr.add_suppression("InnoDB: Unable to open .*ib_logfile0. to check native AIO read support"); +# InnoDB shutdown after refused startup is not clean in 10.0 or 10.1! +--source include/not_valgrind.inc +call mtr.add_suppression("mysqld got signal 11"); +call mtr.add_suppression("Attempting backtrace"); +FLUSH TABLES; +--enable_query_log + +CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; +BEGIN; +INSERT INTO t1 VALUES (42); + +let $restart_parameters = --innodb-log-file-size=6M; +--source include/kill_and_restart_mysqld.inc + +SELECT * FROM t1; + +INSERT INTO t1 VALUES (42); +BEGIN; +DELETE FROM t1; + +let $restart_parameters = --innodb-log-files-in-group=3 --innodb-log-file-size=5M; +--source include/kill_and_restart_mysqld.inc + +SELECT * FROM t1; + +INSERT INTO t1 VALUES (123); + +let MYSQLD_DATADIR= `select @@datadir`; +let SEARCH_RANGE= -50000; +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; + +BEGIN; +DELETE FROM t1; + +--source include/kill_mysqld.inc + +--let $restart_parameters= --innodb-log-group-home-dir=foo\;bar +--source include/start_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= syntax error in innodb_log_group_home_dir; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_1 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= InnoDB: Starting crash recovery from checkpoint LSN=; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_3 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +--let $restart_parameters= --innodb-read-only +--source include/restart_mysqld.inc + +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_4 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_5 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --innodb-read-only +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_6 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_7 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +# this aborts right after deleting all log files + +--let $restart_parameters= --innodb-read-only +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= InnoDB: Cannot create log files in read-only mode; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_8 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= InnoDB: Setting log file .*ib_logfile[0-9]+ size to; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= --debug=d,innodb_log_abort_9 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= InnoDB: Setting log file .*ib_logfile[0-9]+ size to; +--source include/search_pattern_in_file.inc +--source include/shutdown_mysqld.inc + +# We should have perfectly synced files here. +# Rename the log files, and trigger an error in recovery. +--move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0 +--move_file $MYSQLD_DATADIR/ib_logfile1 $MYSQLD_DATADIR/ib_logfile1_hidden + +--let $restart_parameters= +--source include/start_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= InnoDB: Only one log file found; +--source include/search_pattern_in_file.inc +--move_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile101 + +perl; +die unless open(FILE, ">$ENV{MYSQLD_DATADIR}/ib_logfile0"); +print FILE "garbage"; +close(FILE); +EOF + +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; +let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size; +--source include/search_pattern_in_file.inc +--remove_file $MYSQLD_DATADIR/ib_logfile0 +--move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0 + +perl; +die unless open(FILE, ">$ENV{MYSQLD_DATADIR}/ib_logfile1"); +print FILE "junkfill" x 131072; +close(FILE); +EOF + +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile1 is of different size 1048576 bytes than other log files; +--source include/search_pattern_in_file.inc +--remove_file $MYSQLD_DATADIR/ib_logfile1 +--move_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile101 +--move_file $MYSQLD_DATADIR/ib_logfile1_hidden $MYSQLD_DATADIR/ib_logfile1 + +--let $restart_parameters= --debug=d,innodb_log_abort_10 +--source include/restart_mysqld.inc +--error ER_UNKNOWN_STORAGE_ENGINE +SELECT * FROM t1; + +let SEARCH_PATTERN= InnoDB: Setting log file .*ib_logfile[0-9]+ size to; +--source include/search_pattern_in_file.inc +let SEARCH_PATTERN= InnoDB: Renaming log file .*ib_logfile101 to .*ib_logfile0; +--source include/search_pattern_in_file.inc + +--let $restart_parameters= +--source include/restart_mysqld.inc + +SELECT * FROM t1; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/read_only_recovery.test b/mysql-test/suite/innodb/t/read_only_recovery.test new file mode 100644 index 00000000000..f41081e5a94 --- /dev/null +++ b/mysql-test/suite/innodb/t/read_only_recovery.test @@ -0,0 +1,36 @@ +--source include/have_innodb.inc +# need to restart server +--source include/not_embedded.inc + +--connect(con1, localhost, root) +CREATE TABLE t(a INT PRIMARY KEY) ENGINE=InnoDB; +BEGIN; +# Generate insert_undo log. +INSERT INTO t VALUES(1),(2); +# Generate update_undo log. +DELETE FROM t WHERE a=2; +--connection default +--echo # Normal MariaDB shutdown would roll back the above transaction. +--echo # We want the transaction to remain open, so we will kill the server +--echo # after ensuring that any non-transactional files are clean. +FLUSH TABLES; +--echo # Ensure that the above incomplete transaction becomes durable. +SET GLOBAL innodb_flush_log_at_trx_commit=1; +BEGIN; +INSERT INTO t VALUES(0); +ROLLBACK; +--let $restart_parameters= --innodb-force-recovery=3 +--source include/kill_and_restart_mysqld.inc +--disconnect con1 +SELECT * FROM t; +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT * FROM t; +--let $restart_parameters= --innodb-read-only +--source include/restart_mysqld.inc +SELECT * FROM t; +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT * FROM t; +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; diff --git a/mysql-test/suite/innodb/t/xa_recovery.test b/mysql-test/suite/innodb/t/xa_recovery.test index 2c1034f3c4d..f5c2b655545 100644 --- a/mysql-test/suite/innodb/t/xa_recovery.test +++ b/mysql-test/suite/innodb/t/xa_recovery.test @@ -1,7 +1,3 @@ -if (`select plugin_auth_version <= "5.6.24" from information_schema.plugins where plugin_name='innodb'`) -{ - --skip Not fixed in InnoDB as of 5.6.24 or earlier -} --source include/have_innodb.inc # Embedded server does not support restarting. --source include/not_embedded.inc @@ -9,6 +5,7 @@ if (`select plugin_auth_version <= "5.6.24" from information_schema.plugins wher # MDEV-8841 - close tables opened by previous tests, # so they don't get marked crashed when the server gets crashed --disable_query_log +call mtr.add_suppression("Found 1 prepared XA transactions"); FLUSH TABLES; --enable_query_log @@ -18,17 +15,7 @@ connect (con1,localhost,root); XA START 'x'; UPDATE t1 set a=2; XA END 'x'; XA PREPARE 'x'; connection default; -call mtr.add_suppression("Found 1 prepared XA transactions"); - -# Kill and restart the server. --- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- shutdown_server 0 --- source include/wait_until_disconnected.inc - --- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- enable_reconnect --- source include/wait_until_connected_again.inc --- disable_reconnect +--source include/kill_and_restart_mysqld.inc disconnect con1; connect (con1,localhost,root); diff --git a/mysql-test/suite/innodb_fts/r/create.result b/mysql-test/suite/innodb_fts/r/create.result new file mode 100644 index 00000000000..c537aa81efd --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/create.result @@ -0,0 +1,168 @@ +SET NAMES utf8mb4; +# +# MDEV-11233 CREATE FULLTEXT INDEX with a token +# longer than 127 bytes crashes server +# +CREATE TABLE t(t TEXT CHARACTER SET utf8mb3) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +Warnings: +Warning 124 InnoDB rebuilding table to add column FTS_DOC_ID +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +# The data below is not 3-byte UTF-8, but 4-byte chars. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +Warnings: +Warning 1366 Incorrect string value: '\xF0\x9F\x96\x95\xF0\x9F...' for column 't' at row 1 +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +Warnings: +Warning 1366 Incorrect string value: '\xF0\x9F\x96\x96\xF0\x9F...' for column 't' at row 1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST +(REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +COUNT(*) +0 +SELECT * FROM t; +t +็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้ +ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ +beforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbefor +BeforeTheIndexCreation +this was inserted after creating the index +１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１ +afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文 +???????????????????????????????????????????????????????????????????????????????????? +????????????????????????????????????????????????????????????????????????????????????? +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +len COUNT(*) +252 6 +DROP TABLE t; +CREATE TABLE t(t TEXT CHARACTER SET utf8mb4) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +Warnings: +Warning 124 InnoDB rebuilding table to add column FTS_DOC_ID +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); +INSERT INTO t SET t=REPEAT(concat(repeat(_utf8mb3 0xE0B987, 4), repeat(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +# The token below exceeds the 84-character limit. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +COUNT(*) +1 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +COUNT(*) +0 +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +COUNT(*) +0 +SELECT * FROM t; +t +甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文甲骨文 +ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ï¼ +beforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbeforbefor +BeforeTheIndexCreation +this was inserted after creating the index +１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１１ +afterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafterafter +็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้็็็็้้้้้ +🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕🖕 +🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖🖖 +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +len COUNT(*) +336 6 +DROP TABLE t; +CREATE TABLE t(t TEXT CHARACTER SET latin1, FULLTEXT INDEX(t)) +ENGINE=InnoDB; +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +len COUNT(*) +84 6 +DROP TABLE t; diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_stopword_charset.result b/mysql-test/suite/innodb_fts/r/innodb_fts_stopword_charset.result index 5f8d5e37680..d8954e73e33 100644 --- a/mysql-test/suite/innodb_fts/r/innodb_fts_stopword_charset.result +++ b/mysql-test/suite/innodb_fts/r/innodb_fts_stopword_charset.result @@ -316,6 +316,3 @@ id title 13 lòve 14 LÃ’VE DROP TABLE articles; -SET SESSION innodb_ft_enable_stopword=1; -SET GLOBAL innodb_ft_server_stopword_table=default; -SET SESSION innodb_ft_user_stopword_table=default; diff --git a/mysql-test/suite/innodb_fts/t/create.opt b/mysql-test/suite/innodb_fts/t/create.opt new file mode 100644 index 00000000000..3ad568c816e --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/create.opt @@ -0,0 +1 @@ +--loose-innodb-sys-columns diff --git a/mysql-test/suite/innodb_fts/t/create.test b/mysql-test/suite/innodb_fts/t/create.test new file mode 100644 index 00000000000..f0329602ed1 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/create.test @@ -0,0 +1,92 @@ +--source include/have_innodb.inc +SET NAMES utf8mb4; + +--echo # +--echo # MDEV-11233 CREATE FULLTEXT INDEX with a token +--echo # longer than 127 bytes crashes server +--echo # + +# This bug is the result of merging the Oracle MySQL follow-up fix +# BUG#22963169 MYSQL CRASHES ON CREATE FULLTEXT INDEX +# without merging a fix of Bug#79475 Insert a token of 84 4-bytes +# chars into fts index causes server crash. + +# Oracle did not publish tests for either of the above MySQL bugs. +# The tests below were developed for MariaDB Server. +# The maximum length of a fulltext-indexed word is 84 characters. + +CREATE TABLE t(t TEXT CHARACTER SET utf8mb3) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); # too long, will not be indexed +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); # too long, will not be indexed +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +--echo # The data below is not 3-byte UTF-8, but 4-byte chars. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST +(REPEAT(CONCAT(REPEAT(_utf8mb3 0xE0B987, 4), REPEAT(_utf8mb3 0xE0B989, 5)), 5)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +SELECT * FROM t; + +# The column length should be 252 bytes (84 characters * 3 bytes/character). +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +DROP TABLE t; + +CREATE TABLE t(t TEXT CHARACTER SET utf8mb4) ENGINE=InnoDB; +INSERT INTO t SET t=REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15); +INSERT INTO t SET t=REPEAT(_utf8 0xefbc90,84); +INSERT INTO t SET t=REPEAT('befor',17); # too long, will not be indexed +INSERT INTO t SET t='BeforeTheIndexCreation'; +CREATE FULLTEXT INDEX ft ON t(t); +INSERT INTO t SET t='this was inserted after creating the index'; +INSERT INTO t SET t=REPEAT(_utf8 0xefbc91,84); +INSERT INTO t SET t=REPEAT('after',17); # too long, will not be indexed +INSERT INTO t SET t=REPEAT(concat(repeat(_utf8mb3 0xE0B987, 4), repeat(_utf8mb3 0xE0B989, 5)), 5); +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9695, 84); +--echo # The token below exceeds the 84-character limit. +INSERT INTO t SET t=REPEAT(_utf8mb4 0xf09f9696, 85); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb3 0xe794b2e9aaa8e69687, 15)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('BeforeTheIndexCreation'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('befor',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST ('after'); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT('after',17)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc90, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8 0xefbc91, 85)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 83)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9695, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 84)); +SELECT COUNT(*) FROM t WHERE MATCH t AGAINST (REPEAT(_utf8mb4 0xf09f9696, 85)); +SELECT * FROM t; + +# The column length should be 336 bytes (84 characters * 4 bytes/character). +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +DROP TABLE t; + +CREATE TABLE t(t TEXT CHARACTER SET latin1, FULLTEXT INDEX(t)) +ENGINE=InnoDB; + +# The column length should be 84 bytes (84 characters * 1 byte/character). +SELECT len,COUNT(*) FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS where name='word' GROUP BY len; +DROP TABLE t; diff --git a/mysql-test/suite/innodb_fts/t/innodb_fts_stopword_charset.test b/mysql-test/suite/innodb_fts/t/innodb_fts_stopword_charset.test index cb49ca0e39f..4092c324bf9 100644 --- a/mysql-test/suite/innodb_fts/t/innodb_fts_stopword_charset.test +++ b/mysql-test/suite/innodb_fts/t/innodb_fts_stopword_charset.test @@ -2,16 +2,8 @@ -- source include/have_innodb.inc -# Valgrind would complain about memory leaks when we crash on purpose. ---source include/not_valgrind.inc -# Embedded server does not support crashing +# Embedded server tests do not support restarting --source include/not_embedded.inc -# Avoid CrashReporter popup on Mac ---source include/not_crashrep.inc - -let $innodb_ft_server_stopword_table_orig=`SELECT @@innodb_ft_server_stopword_table`; -let $innodb_ft_enable_stopword_orig=`SELECT @@innodb_ft_enable_stopword`; -let $innodb_ft_user_stopword_table_orig=`SELECT @@innodb_ft_user_stopword_table`; SELECT @@innodb_ft_server_stopword_table; SELECT @@innodb_ft_enable_stopword; @@ -414,8 +406,3 @@ SELECT * FROM articles WHERE MATCH (title) AGAINST ('love' IN NATURAL LANGUAGE MODE); DROP TABLE articles; - -# Restore Values -eval SET SESSION innodb_ft_enable_stopword=$innodb_ft_enable_stopword_orig; -eval SET GLOBAL innodb_ft_server_stopword_table=default; -eval SET SESSION innodb_ft_user_stopword_table=default; diff --git a/mysql-test/suite/maria/icp.result b/mysql-test/suite/maria/icp.result index 1c11072cbde..cc03898f054 100644 --- a/mysql-test/suite/maria/icp.result +++ b/mysql-test/suite/maria/icp.result @@ -438,7 +438,7 @@ WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10 id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 Using where 2 DEPENDENT SUBQUERY it eq_ref PRIMARY PRIMARY 4 func 1 Using index condition -2 DEPENDENT SUBQUERY t2 index NULL PRIMARY 4 NULL 3 Using where; Using index; Using join buffer (flat, BNL join) +2 DEPENDENT SUBQUERY t2 index NULL PRIMARY 4 NULL 3 Using index; Using join buffer (flat, BNL join) SELECT * FROM t1 WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10); pk i @@ -802,7 +802,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t ALL PRIMARY,c NULL NULL NULL 64 Using where 1 PRIMARY t2 ref g g 5 test.t.c 19 Using where 2 DEPENDENT SUBQUERY t1 index PRIMARY PRIMARY 4 NULL 64 Using where; Using index -2 DEPENDENT SUBQUERY t2 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 Using index condition; Using where +2 DEPENDENT SUBQUERY t2 eq_ref PRIMARY PRIMARY 4 test.t1.a 1 Using where SELECT COUNT(*) FROM t1 AS t, t2 WHERE c = g AND (EXISTS (SELECT * FROM t1, t2 WHERE a = f AND h <= t.e AND a > t.b) diff --git a/mysql-test/suite/parts/r/partition_bigint_innodb.result b/mysql-test/suite/parts/r/partition_bigint_innodb.result new file mode 100644 index 00000000000..bb0f08d9356 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_bigint_innodb.result @@ -0,0 +1,121 @@ +create table t1 (a bigint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` bigint(20) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ +insert into t1 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612), (1), (2), (65535); +select * from t1; +a +1 +18446744073709551612 +18446744073709551613 +18446744073709551614 +18446744073709551615 +2 +65535 +select * from t1 where a=-2; +a +delete from t1 where a=-2; +select * from t1; +a +1 +18446744073709551612 +18446744073709551613 +18446744073709551614 +18446744073709551615 +2 +65535 +select * from t1 where a=18446744073709551615; +a +18446744073709551615 +delete from t1 where a=18446744073709551615; +select * from t1; +a +1 +18446744073709551612 +18446744073709551613 +18446744073709551614 +2 +65535 +drop table t1; +create table t2 (a bigint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` bigint(20) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612); +select * from t2; +a +18446744073709551612 +18446744073709551613 +18446744073709551614 +18446744073709551615 +select * from t2 where a=18446744073709551615; +a +18446744073709551615 +delete from t2 where a=18446744073709551615; +select * from t2; +a +18446744073709551612 +18446744073709551613 +18446744073709551614 +delete from t2; +1024 inserts; +select count(*) from t2; +count(*) +1024 +drop table t2; +create table t3 (a bigint not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` bigint(20) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (9223372036854775807), (9223372036854775806), (9223372036854775805), (9223372036854775804), (-9223372036854775808), (-9223372036854775807), (1), (-1), (0); +select * from t3; +a +-1 +-9223372036854775807 +-9223372036854775808 +0 +1 +9223372036854775804 +9223372036854775805 +9223372036854775806 +9223372036854775807 +select * from t3 where a=9223372036854775806; +a +9223372036854775806 +delete from t3 where a=9223372036854775806; +select * from t3; +a +-1 +-9223372036854775807 +-9223372036854775808 +0 +1 +9223372036854775804 +9223372036854775805 +9223372036854775807 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_bigint_myisam.result b/mysql-test/suite/parts/r/partition_bigint_myisam.result new file mode 100644 index 00000000000..5938bcaf7d8 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_bigint_myisam.result @@ -0,0 +1,121 @@ +create table t1 (a bigint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` bigint(20) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ +insert into t1 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612), (1), (2), (65535); +select * from t1; +a +1 +18446744073709551612 +18446744073709551613 +18446744073709551614 +18446744073709551615 +2 +65535 +select * from t1 where a=-2; +a +delete from t1 where a=-2; +select * from t1; +a +1 +18446744073709551612 +18446744073709551613 +18446744073709551614 +18446744073709551615 +2 +65535 +select * from t1 where a=18446744073709551615; +a +18446744073709551615 +delete from t1 where a=18446744073709551615; +select * from t1; +a +1 +18446744073709551612 +18446744073709551613 +18446744073709551614 +2 +65535 +drop table t1; +create table t2 (a bigint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` bigint(20) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612); +select * from t2; +a +18446744073709551612 +18446744073709551613 +18446744073709551614 +18446744073709551615 +select * from t2 where a=18446744073709551615; +a +18446744073709551615 +delete from t2 where a=18446744073709551615; +select * from t2; +a +18446744073709551612 +18446744073709551613 +18446744073709551614 +delete from t2; +65535 inserts; +select count(*) from t2; +count(*) +65535 +drop table t2; +create table t3 (a bigint not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` bigint(20) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (9223372036854775807), (9223372036854775806), (9223372036854775805), (9223372036854775804), (-9223372036854775808), (-9223372036854775807), (1), (-1), (0); +select * from t3; +a +-1 +-9223372036854775807 +-9223372036854775808 +0 +1 +9223372036854775804 +9223372036854775805 +9223372036854775806 +9223372036854775807 +select * from t3 where a=9223372036854775806; +a +9223372036854775806 +delete from t3 where a=9223372036854775806; +select * from t3; +a +-1 +-9223372036854775807 +-9223372036854775808 +0 +1 +9223372036854775804 +9223372036854775805 +9223372036854775807 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_double_innodb.result b/mysql-test/suite/parts/r/partition_double_innodb.result new file mode 100644 index 00000000000..8c0daf929dd --- /dev/null +++ b/mysql-test/suite/parts/r/partition_double_innodb.result @@ -0,0 +1,82 @@ +create table t1 (a double not null, primary key(a)) engine='InnoDB' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` double NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ +insert into t1 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); +select * from t1; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1.5 +1234.567 +2.2250738585072016e208 +select * from t1 where a=1.5; +a +1.5 +delete from t1 where a=1.5; +select * from t1; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1234.567 +2.2250738585072016e208 +drop table t1; +create table t2 (a double not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 10; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` double NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 10 */ +insert into t2 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); +select * from t2; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1.5 +1234.567 +2.2250738585072016e208 +select * from t2 where a=1234.567; +a +1234.567 +delete from t2 where a=1234.567; +select * from t2; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1.5 +2.2250738585072016e208 +delete from t2; +1024*3 inserts; +select count(*) from t2; +count(*) +3072 +drop table t2; diff --git a/mysql-test/suite/parts/r/partition_double_myisam.result b/mysql-test/suite/parts/r/partition_double_myisam.result new file mode 100644 index 00000000000..045763e5ef9 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_double_myisam.result @@ -0,0 +1,82 @@ +create table t1 (a double not null, primary key(a)) engine='MYISAM' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` double NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ +insert into t1 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); +select * from t1; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1.5 +1234.567 +2.2250738585072016e208 +select * from t1 where a=1.5; +a +1.5 +delete from t1 where a=1.5; +select * from t1; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1234.567 +2.2250738585072016e208 +drop table t1; +create table t2 (a double not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 10; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` double NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 10 */ +insert into t2 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); +select * from t2; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1.5 +1234.567 +2.2250738585072016e208 +select * from t2 where a=1234.567; +a +1234.567 +delete from t2 where a=1234.567; +select * from t2; +a +-2.2250738585072016e208 +-1.5 +-1 +-2.2250738585072014e-208 +0 +1.5 +2.2250738585072016e208 +delete from t2; +16384*3 inserts; +select count(*) from t2; +count(*) +49152 +drop table t2; diff --git a/mysql-test/suite/parts/r/partition_float_innodb.result b/mysql-test/suite/parts/r/partition_float_innodb.result index d2f04a68629..b0870992c41 100644 --- a/mysql-test/suite/parts/r/partition_float_innodb.result +++ b/mysql-test/suite/parts/r/partition_float_innodb.result @@ -88,85 +88,3 @@ select count(*) from t2; count(*) 3072 drop table t2; -create table t1 (a double not null, primary key(a)) engine='InnoDB' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` double NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ -insert into t1 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); -select * from t1; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1.5 -1234.567 -2.2250738585072016e208 -select * from t1 where a=1.5; -a -1.5 -delete from t1 where a=1.5; -select * from t1; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1234.567 -2.2250738585072016e208 -drop table t1; -create table t2 (a double not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 10; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` double NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 10 */ -insert into t2 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); -select * from t2; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1.5 -1234.567 -2.2250738585072016e208 -select * from t2 where a=1234.567; -a -1234.567 -delete from t2 where a=1234.567; -select * from t2; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1.5 -2.2250738585072016e208 -delete from t2; -1024*3 inserts; -select count(*) from t2; -count(*) -3072 -drop table t2; diff --git a/mysql-test/suite/parts/r/partition_float_myisam.result b/mysql-test/suite/parts/r/partition_float_myisam.result index 2d52d095989..931c4ef0394 100644 --- a/mysql-test/suite/parts/r/partition_float_myisam.result +++ b/mysql-test/suite/parts/r/partition_float_myisam.result @@ -88,85 +88,3 @@ select count(*) from t2; count(*) 49152 drop table t2; -create table t1 (a double not null, primary key(a)) engine='MYISAM' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` double NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ -insert into t1 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); -select * from t1; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1.5 -1234.567 -2.2250738585072016e208 -select * from t1 where a=1.5; -a -1.5 -delete from t1 where a=1.5; -select * from t1; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1234.567 -2.2250738585072016e208 -drop table t1; -create table t2 (a double not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 10; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` double NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 10 */ -insert into t2 values (-2.2250738585072014E+208), (-2.2250738585072014E-208), (-1.5), (-1), (0), (1.5), (1234.567), (2.2250738585072014E+208); -select * from t2; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1.5 -1234.567 -2.2250738585072016e208 -select * from t2 where a=1234.567; -a -1234.567 -delete from t2 where a=1234.567; -select * from t2; -a --2.2250738585072016e208 --1.5 --1 --2.2250738585072014e-208 -0 -1.5 -2.2250738585072016e208 -delete from t2; -16384*3 inserts; -select count(*) from t2; -count(*) -49152 -drop table t2; diff --git a/mysql-test/suite/parts/r/partition_int_innodb.result b/mysql-test/suite/parts/r/partition_int_innodb.result index 7a51b80d5d7..c1798e5f711 100644 --- a/mysql-test/suite/parts/r/partition_int_innodb.result +++ b/mysql-test/suite/parts/r/partition_int_innodb.result @@ -1,221 +1,3 @@ -create table t1 (a tinyint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` tinyint(3) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ -insert into t1 values (255), (254), (253), (252), (1), (2), (128); -select * from t1; -a -1 -128 -2 -252 -253 -254 -255 -select * from t1 where a=253; -a -253 -delete from t1 where a=253; -select * from t1; -a -1 -128 -2 -252 -254 -255 -drop table t1; -create table t2 (a tinyint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` tinyint(3) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (255), (254), (253), (252); -select * from t2; -a -252 -253 -254 -255 -select * from t2 where a=253; -a -253 -delete from t2 where a=253; -select * from t2; -a -252 -254 -255 -delete from t2; -255 inserts; -select count(*) from t2; -count(*) -255 -drop table t2; -create table t3 (a tinyint not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` tinyint(4) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (127), (126), (125), (124), (-128), (-127), (1), (-1), (0); -select * from t3; -a --1 --127 --128 -0 -1 -124 -125 -126 -127 -select * from t3 where a=125; -a -125 -delete from t3 where a=125; -select * from t3; -a --1 --127 --128 -0 -1 -124 -126 -127 -drop table t3; -create table t1 (a smallint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` smallint(5) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ -insert into t1 values (65535), (65534), (65533), (65532), (1), (2), (256); -select * from t1; -a -1 -2 -256 -65532 -65533 -65534 -65535 -select * from t1 where a=65533; -a -65533 -delete from t1 where a=65533; -select * from t1; -a -1 -2 -256 -65532 -65534 -65535 -drop table t1; -create table t2 (a smallint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` smallint(5) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (65535), (65534), (65533), (65532); -select * from t2; -a -65532 -65533 -65534 -65535 -select * from t2 where a=65533; -a -65533 -delete from t2 where a=65533; -select * from t2; -a -65532 -65534 -65535 -delete from t2; -1024 inserts; -select count(*) from t2; -count(*) -1024 -drop table t2; -create table t3 (a smallint not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` smallint(6) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (32767), (32766), (32765), (32764), (-32768), (-32767), (1), (-1), (0); -select * from t3; -a --1 --32767 --32768 -0 -1 -32764 -32765 -32766 -32767 -select * from t3 where a=32765; -a -32765 -delete from t3 where a=32765; -select * from t3; -a --1 --32767 --32768 -0 -1 -32764 -32766 -32767 -drop table t3; create table t1 (a int unsigned not null, primary key(a)) engine='InnoDB' partition by key (a) ( partition pa1 max_rows=20 min_rows=2, @@ -325,233 +107,3 @@ a 2147483646 2147483647 drop table t3; -create table t1 (a mediumint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` mediumint(8) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ -insert into t1 values (16777215), (16777214), (16777213), (16777212), (1), (2), (65535); -select * from t1; -a -1 -16777212 -16777213 -16777214 -16777215 -2 -65535 -select * from t1 where a=16777213; -a -16777213 -delete from t1 where a=16777213; -select * from t1; -a -1 -16777212 -16777214 -16777215 -2 -65535 -drop table t1; -create table t2 (a mediumint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` mediumint(8) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (16777215), (16777214), (16777213), (16777212); -select * from t2; -a -16777212 -16777213 -16777214 -16777215 -select * from t2 where a=16777213; -a -16777213 -delete from t2 where a=16777213; -select * from t2; -a -16777212 -16777214 -16777215 -delete from t2; -1024 inserts; -select count(*) from t2; -count(*) -1024 -drop table t2; -create table t3 (a mediumint not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` mediumint(9) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (8388607), (8388606), (8388605), (8388604), (-8388608), (-8388607), (1), (-1), (0); -select * from t3; -a --1 --8388607 --8388608 -0 -1 -8388604 -8388605 -8388606 -8388607 -select * from t3 where a=8388605; -a -8388605 -delete from t3 where a=8388605; -select * from t3; -a --1 --8388607 --8388608 -0 -1 -8388604 -8388606 -8388607 -drop table t3; -create table t1 (a bigint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` bigint(20) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ -insert into t1 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612), (1), (2), (65535); -select * from t1; -a -1 -18446744073709551612 -18446744073709551613 -18446744073709551614 -18446744073709551615 -2 -65535 -select * from t1 where a=-2; -a -delete from t1 where a=-2; -select * from t1; -a -1 -18446744073709551612 -18446744073709551613 -18446744073709551614 -18446744073709551615 -2 -65535 -select * from t1 where a=18446744073709551615; -a -18446744073709551615 -delete from t1 where a=18446744073709551615; -select * from t1; -a -1 -18446744073709551612 -18446744073709551613 -18446744073709551614 -2 -65535 -drop table t1; -create table t2 (a bigint unsigned not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` bigint(20) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612); -select * from t2; -a -18446744073709551612 -18446744073709551613 -18446744073709551614 -18446744073709551615 -select * from t2 where a=18446744073709551615; -a -18446744073709551615 -delete from t2 where a=18446744073709551615; -select * from t2; -a -18446744073709551612 -18446744073709551613 -18446744073709551614 -delete from t2; -1024 inserts; -select count(*) from t2; -count(*) -1024 -drop table t2; -create table t3 (a bigint not null, primary key(a)) engine='InnoDB' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` bigint(20) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (9223372036854775807), (9223372036854775806), (9223372036854775805), (9223372036854775804), (-9223372036854775808), (-9223372036854775807), (1), (-1), (0); -select * from t3; -a --1 --9223372036854775807 --9223372036854775808 -0 -1 -9223372036854775804 -9223372036854775805 -9223372036854775806 -9223372036854775807 -select * from t3 where a=9223372036854775806; -a -9223372036854775806 -delete from t3 where a=9223372036854775806; -select * from t3; -a --1 --9223372036854775807 --9223372036854775808 -0 -1 -9223372036854775804 -9223372036854775805 -9223372036854775807 -drop table t3; diff --git a/mysql-test/suite/parts/r/partition_int_myisam.result b/mysql-test/suite/parts/r/partition_int_myisam.result index 4387bbfdd78..8b8352ebc38 100644 --- a/mysql-test/suite/parts/r/partition_int_myisam.result +++ b/mysql-test/suite/parts/r/partition_int_myisam.result @@ -1,221 +1,3 @@ -create table t1 (a tinyint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` tinyint(3) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ -insert into t1 values (255), (254), (253), (252), (1), (2), (128); -select * from t1; -a -1 -128 -2 -252 -253 -254 -255 -select * from t1 where a=253; -a -253 -delete from t1 where a=253; -select * from t1; -a -1 -128 -2 -252 -254 -255 -drop table t1; -create table t2 (a tinyint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` tinyint(3) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (255), (254), (253), (252); -select * from t2; -a -252 -253 -254 -255 -select * from t2 where a=253; -a -253 -delete from t2 where a=253; -select * from t2; -a -252 -254 -255 -delete from t2; -255 inserts; -select count(*) from t2; -count(*) -255 -drop table t2; -create table t3 (a tinyint not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` tinyint(4) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (127), (126), (125), (124), (-128), (-127), (1), (-1), (0); -select * from t3; -a --1 --127 --128 -0 -1 -124 -125 -126 -127 -select * from t3 where a=125; -a -125 -delete from t3 where a=125; -select * from t3; -a --1 --127 --128 -0 -1 -124 -126 -127 -drop table t3; -create table t1 (a smallint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` smallint(5) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ -insert into t1 values (65535), (65534), (65533), (65532), (1), (2), (256); -select * from t1; -a -1 -2 -256 -65532 -65533 -65534 -65535 -select * from t1 where a=65533; -a -65533 -delete from t1 where a=65533; -select * from t1; -a -1 -2 -256 -65532 -65534 -65535 -drop table t1; -create table t2 (a smallint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` smallint(5) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (65535), (65534), (65533), (65532); -select * from t2; -a -65532 -65533 -65534 -65535 -select * from t2 where a=65533; -a -65533 -delete from t2 where a=65533; -select * from t2; -a -65532 -65534 -65535 -delete from t2; -65535 inserts; -select count(*) from t2; -count(*) -65535 -drop table t2; -create table t3 (a smallint not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` smallint(6) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (32767), (32766), (32765), (32764), (-32768), (-32767), (1), (-1), (0); -select * from t3; -a --1 --32767 --32768 -0 -1 -32764 -32765 -32766 -32767 -select * from t3 where a=32765; -a -32765 -delete from t3 where a=32765; -select * from t3; -a --1 --32767 --32768 -0 -1 -32764 -32766 -32767 -drop table t3; create table t1 (a int unsigned not null, primary key(a)) engine='MYISAM' partition by key (a) ( partition pa1 max_rows=20 min_rows=2, @@ -325,233 +107,3 @@ a 2147483646 2147483647 drop table t3; -create table t1 (a mediumint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` mediumint(8) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ -insert into t1 values (16777215), (16777214), (16777213), (16777212), (1), (2), (65535); -select * from t1; -a -1 -16777212 -16777213 -16777214 -16777215 -2 -65535 -select * from t1 where a=16777213; -a -16777213 -delete from t1 where a=16777213; -select * from t1; -a -1 -16777212 -16777214 -16777215 -2 -65535 -drop table t1; -create table t2 (a mediumint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` mediumint(8) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (16777215), (16777214), (16777213), (16777212); -select * from t2; -a -16777212 -16777213 -16777214 -16777215 -select * from t2 where a=16777213; -a -16777213 -delete from t2 where a=16777213; -select * from t2; -a -16777212 -16777214 -16777215 -delete from t2; -65535 inserts; -select count(*) from t2; -count(*) -65535 -drop table t2; -create table t3 (a mediumint not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` mediumint(9) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (8388607), (8388606), (8388605), (8388604), (-8388608), (-8388607), (1), (-1), (0); -select * from t3; -a --1 --8388607 --8388608 -0 -1 -8388604 -8388605 -8388606 -8388607 -select * from t3 where a=8388605; -a -8388605 -delete from t3 where a=8388605; -select * from t3; -a --1 --8388607 --8388608 -0 -1 -8388604 -8388606 -8388607 -drop table t3; -create table t1 (a bigint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) ( -partition pa1 max_rows=20 min_rows=2, -partition pa2 max_rows=30 min_rows=3, -partition pa3 max_rows=30 min_rows=4, -partition pa4 max_rows=40 min_rows=2); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` bigint(20) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, - PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, - PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, - PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ -insert into t1 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612), (1), (2), (65535); -select * from t1; -a -1 -18446744073709551612 -18446744073709551613 -18446744073709551614 -18446744073709551615 -2 -65535 -select * from t1 where a=-2; -a -delete from t1 where a=-2; -select * from t1; -a -1 -18446744073709551612 -18446744073709551613 -18446744073709551614 -18446744073709551615 -2 -65535 -select * from t1 where a=18446744073709551615; -a -18446744073709551615 -delete from t1 where a=18446744073709551615; -select * from t1; -a -1 -18446744073709551612 -18446744073709551613 -18446744073709551614 -2 -65535 -drop table t1; -create table t2 (a bigint unsigned not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 8; -show create table t2; -Table Create Table -t2 CREATE TABLE `t2` ( - `a` bigint(20) unsigned NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 8 */ -insert into t2 values (18446744073709551615), (0xFFFFFFFFFFFFFFFE), (18446744073709551613), (18446744073709551612); -select * from t2; -a -18446744073709551612 -18446744073709551613 -18446744073709551614 -18446744073709551615 -select * from t2 where a=18446744073709551615; -a -18446744073709551615 -delete from t2 where a=18446744073709551615; -select * from t2; -a -18446744073709551612 -18446744073709551613 -18446744073709551614 -delete from t2; -65535 inserts; -select count(*) from t2; -count(*) -65535 -drop table t2; -create table t3 (a bigint not null, primary key(a)) engine='MYISAM' -partition by key (a) partitions 7; -show create table t3; -Table Create Table -t3 CREATE TABLE `t3` ( - `a` bigint(20) NOT NULL, - PRIMARY KEY (`a`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (a) -PARTITIONS 7 */ -insert into t3 values (9223372036854775807), (9223372036854775806), (9223372036854775805), (9223372036854775804), (-9223372036854775808), (-9223372036854775807), (1), (-1), (0); -select * from t3; -a --1 --9223372036854775807 --9223372036854775808 -0 -1 -9223372036854775804 -9223372036854775805 -9223372036854775806 -9223372036854775807 -select * from t3 where a=9223372036854775806; -a -9223372036854775806 -delete from t3 where a=9223372036854775806; -select * from t3; -a --1 --9223372036854775807 --9223372036854775808 -0 -1 -9223372036854775804 -9223372036854775805 -9223372036854775807 -drop table t3; diff --git a/mysql-test/suite/parts/r/partition_mediumint_innodb.result b/mysql-test/suite/parts/r/partition_mediumint_innodb.result new file mode 100644 index 00000000000..8e3e5543ddc --- /dev/null +++ b/mysql-test/suite/parts/r/partition_mediumint_innodb.result @@ -0,0 +1,109 @@ +create table t1 (a mediumint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` mediumint(8) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ +insert into t1 values (16777215), (16777214), (16777213), (16777212), (1), (2), (65535); +select * from t1; +a +1 +16777212 +16777213 +16777214 +16777215 +2 +65535 +select * from t1 where a=16777213; +a +16777213 +delete from t1 where a=16777213; +select * from t1; +a +1 +16777212 +16777214 +16777215 +2 +65535 +drop table t1; +create table t2 (a mediumint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` mediumint(8) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (16777215), (16777214), (16777213), (16777212); +select * from t2; +a +16777212 +16777213 +16777214 +16777215 +select * from t2 where a=16777213; +a +16777213 +delete from t2 where a=16777213; +select * from t2; +a +16777212 +16777214 +16777215 +delete from t2; +1024 inserts; +select count(*) from t2; +count(*) +1024 +drop table t2; +create table t3 (a mediumint not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` mediumint(9) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (8388607), (8388606), (8388605), (8388604), (-8388608), (-8388607), (1), (-1), (0); +select * from t3; +a +-1 +-8388607 +-8388608 +0 +1 +8388604 +8388605 +8388606 +8388607 +select * from t3 where a=8388605; +a +8388605 +delete from t3 where a=8388605; +select * from t3; +a +-1 +-8388607 +-8388608 +0 +1 +8388604 +8388606 +8388607 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_mediumint_myisam.result b/mysql-test/suite/parts/r/partition_mediumint_myisam.result new file mode 100644 index 00000000000..4853680610b --- /dev/null +++ b/mysql-test/suite/parts/r/partition_mediumint_myisam.result @@ -0,0 +1,109 @@ +create table t1 (a mediumint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` mediumint(8) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ +insert into t1 values (16777215), (16777214), (16777213), (16777212), (1), (2), (65535); +select * from t1; +a +1 +16777212 +16777213 +16777214 +16777215 +2 +65535 +select * from t1 where a=16777213; +a +16777213 +delete from t1 where a=16777213; +select * from t1; +a +1 +16777212 +16777214 +16777215 +2 +65535 +drop table t1; +create table t2 (a mediumint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` mediumint(8) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (16777215), (16777214), (16777213), (16777212); +select * from t2; +a +16777212 +16777213 +16777214 +16777215 +select * from t2 where a=16777213; +a +16777213 +delete from t2 where a=16777213; +select * from t2; +a +16777212 +16777214 +16777215 +delete from t2; +65535 inserts; +select count(*) from t2; +count(*) +65535 +drop table t2; +create table t3 (a mediumint not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` mediumint(9) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (8388607), (8388606), (8388605), (8388604), (-8388608), (-8388607), (1), (-1), (0); +select * from t3; +a +-1 +-8388607 +-8388608 +0 +1 +8388604 +8388605 +8388606 +8388607 +select * from t3 where a=8388605; +a +8388605 +delete from t3 where a=8388605; +select * from t3; +a +-1 +-8388607 +-8388608 +0 +1 +8388604 +8388606 +8388607 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_smallint_innodb.result b/mysql-test/suite/parts/r/partition_smallint_innodb.result new file mode 100644 index 00000000000..fbf23fe582c --- /dev/null +++ b/mysql-test/suite/parts/r/partition_smallint_innodb.result @@ -0,0 +1,109 @@ +create table t1 (a smallint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` smallint(5) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ +insert into t1 values (65535), (65534), (65533), (65532), (1), (2), (256); +select * from t1; +a +1 +2 +256 +65532 +65533 +65534 +65535 +select * from t1 where a=65533; +a +65533 +delete from t1 where a=65533; +select * from t1; +a +1 +2 +256 +65532 +65534 +65535 +drop table t1; +create table t2 (a smallint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` smallint(5) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (65535), (65534), (65533), (65532); +select * from t2; +a +65532 +65533 +65534 +65535 +select * from t2 where a=65533; +a +65533 +delete from t2 where a=65533; +select * from t2; +a +65532 +65534 +65535 +delete from t2; +1024 inserts; +select count(*) from t2; +count(*) +1024 +drop table t2; +create table t3 (a smallint not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (32767), (32766), (32765), (32764), (-32768), (-32767), (1), (-1), (0); +select * from t3; +a +-1 +-32767 +-32768 +0 +1 +32764 +32765 +32766 +32767 +select * from t3 where a=32765; +a +32765 +delete from t3 where a=32765; +select * from t3; +a +-1 +-32767 +-32768 +0 +1 +32764 +32766 +32767 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_smallint_myisam.result b/mysql-test/suite/parts/r/partition_smallint_myisam.result new file mode 100644 index 00000000000..a405d025919 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_smallint_myisam.result @@ -0,0 +1,109 @@ +create table t1 (a smallint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` smallint(5) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ +insert into t1 values (65535), (65534), (65533), (65532), (1), (2), (256); +select * from t1; +a +1 +2 +256 +65532 +65533 +65534 +65535 +select * from t1 where a=65533; +a +65533 +delete from t1 where a=65533; +select * from t1; +a +1 +2 +256 +65532 +65534 +65535 +drop table t1; +create table t2 (a smallint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` smallint(5) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (65535), (65534), (65533), (65532); +select * from t2; +a +65532 +65533 +65534 +65535 +select * from t2 where a=65533; +a +65533 +delete from t2 where a=65533; +select * from t2; +a +65532 +65534 +65535 +delete from t2; +65535 inserts; +select count(*) from t2; +count(*) +65535 +drop table t2; +create table t3 (a smallint not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` smallint(6) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (32767), (32766), (32765), (32764), (-32768), (-32767), (1), (-1), (0); +select * from t3; +a +-1 +-32767 +-32768 +0 +1 +32764 +32765 +32766 +32767 +select * from t3 where a=32765; +a +32765 +delete from t3 where a=32765; +select * from t3; +a +-1 +-32767 +-32768 +0 +1 +32764 +32766 +32767 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_tinyint_innodb.result b/mysql-test/suite/parts/r/partition_tinyint_innodb.result new file mode 100644 index 00000000000..d7138539b78 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_tinyint_innodb.result @@ -0,0 +1,109 @@ +create table t1 (a tinyint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` tinyint(3) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = InnoDB, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = InnoDB, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = InnoDB, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = InnoDB) */ +insert into t1 values (255), (254), (253), (252), (1), (2), (128); +select * from t1; +a +1 +128 +2 +252 +253 +254 +255 +select * from t1 where a=253; +a +253 +delete from t1 where a=253; +select * from t1; +a +1 +128 +2 +252 +254 +255 +drop table t1; +create table t2 (a tinyint unsigned not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` tinyint(3) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (255), (254), (253), (252); +select * from t2; +a +252 +253 +254 +255 +select * from t2 where a=253; +a +253 +delete from t2 where a=253; +select * from t2; +a +252 +254 +255 +delete from t2; +255 inserts; +select count(*) from t2; +count(*) +255 +drop table t2; +create table t3 (a tinyint not null, primary key(a)) engine='InnoDB' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` tinyint(4) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (127), (126), (125), (124), (-128), (-127), (1), (-1), (0); +select * from t3; +a +-1 +-127 +-128 +0 +1 +124 +125 +126 +127 +select * from t3 where a=125; +a +125 +delete from t3 where a=125; +select * from t3; +a +-1 +-127 +-128 +0 +1 +124 +126 +127 +drop table t3; diff --git a/mysql-test/suite/parts/r/partition_tinyint_myisam.result b/mysql-test/suite/parts/r/partition_tinyint_myisam.result new file mode 100644 index 00000000000..08a688e8f36 --- /dev/null +++ b/mysql-test/suite/parts/r/partition_tinyint_myisam.result @@ -0,0 +1,109 @@ +create table t1 (a tinyint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) ( +partition pa1 max_rows=20 min_rows=2, +partition pa2 max_rows=30 min_rows=3, +partition pa3 max_rows=30 min_rows=4, +partition pa4 max_rows=40 min_rows=2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` tinyint(3) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +(PARTITION pa1 MAX_ROWS = 20 MIN_ROWS = 2 ENGINE = MyISAM, + PARTITION pa2 MAX_ROWS = 30 MIN_ROWS = 3 ENGINE = MyISAM, + PARTITION pa3 MAX_ROWS = 30 MIN_ROWS = 4 ENGINE = MyISAM, + PARTITION pa4 MAX_ROWS = 40 MIN_ROWS = 2 ENGINE = MyISAM) */ +insert into t1 values (255), (254), (253), (252), (1), (2), (128); +select * from t1; +a +1 +128 +2 +252 +253 +254 +255 +select * from t1 where a=253; +a +253 +delete from t1 where a=253; +select * from t1; +a +1 +128 +2 +252 +254 +255 +drop table t1; +create table t2 (a tinyint unsigned not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 8; +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` tinyint(3) unsigned NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 8 */ +insert into t2 values (255), (254), (253), (252); +select * from t2; +a +252 +253 +254 +255 +select * from t2 where a=253; +a +253 +delete from t2 where a=253; +select * from t2; +a +252 +254 +255 +delete from t2; +255 inserts; +select count(*) from t2; +count(*) +255 +drop table t2; +create table t3 (a tinyint not null, primary key(a)) engine='MYISAM' +partition by key (a) partitions 7; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` tinyint(4) NOT NULL, + PRIMARY KEY (`a`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (a) +PARTITIONS 7 */ +insert into t3 values (127), (126), (125), (124), (-128), (-127), (1), (-1), (0); +select * from t3; +a +-1 +-127 +-128 +0 +1 +124 +125 +126 +127 +select * from t3 where a=125; +a +125 +delete from t3 where a=125; +select * from t3; +a +-1 +-127 +-128 +0 +1 +124 +126 +127 +drop table t3; diff --git a/mysql-test/suite/parts/t/partition_bigint_innodb.test b/mysql-test/suite/parts/t/partition_bigint_innodb.test new file mode 100644 index 00000000000..348ee0add05 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_bigint_innodb.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_bigint_innodb.test # +# # +# Purpose: # +# Tests around integer type # +# INNODB branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_innodb # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT INNODB SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +--source include/have_innodb.inc +let $engine= 'InnoDB'; + +##### max rows to be inserted +let $maxrows=1024; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_bigint.inc diff --git a/mysql-test/suite/parts/t/partition_bigint_myisam.test b/mysql-test/suite/parts/t/partition_bigint_myisam.test new file mode 100644 index 00000000000..f427ffce08d --- /dev/null +++ b/mysql-test/suite/parts/t/partition_bigint_myisam.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_bigint_myisam.test # +# # +# Purpose: # +# Tests around integer type # +# MyISAM branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_myisam # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +--source include/long_test.inc + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +let $engine= 'MYISAM'; +##### number of rows to be inserted +let $maxrows=65535; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_bigint.inc diff --git a/mysql-test/suite/parts/t/partition_double_innodb.test b/mysql-test/suite/parts/t/partition_double_innodb.test new file mode 100644 index 00000000000..e31f7049502 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_double_innodb.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_double_innodb.test # +# # +# Purpose: # +# Tests around float type # +# INNODB branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_float_innodb # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT INNODB SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +--source include/have_innodb.inc +let $engine= 'InnoDB'; + +##### Number of row to be inserted. +let $maxrows=1024; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_double.inc diff --git a/mysql-test/suite/parts/t/partition_double_myisam.test b/mysql-test/suite/parts/t/partition_double_myisam.test new file mode 100644 index 00000000000..6228d657c48 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_double_myisam.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_double_myisam.test # +# # +# Purpose: # +# Tests around float type # +# MyISAM branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_float_myisam # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +--source include/long_test.inc + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +let $engine= 'MYISAM'; +##### Number of row to be inserted. +let $maxrows=16384; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_double.inc diff --git a/mysql-test/suite/parts/t/partition_float_innodb.test b/mysql-test/suite/parts/t/partition_float_innodb.test index 2f1fe723dad..8d96eafbb06 100644 --- a/mysql-test/suite/parts/t/partition_float_innodb.test +++ b/mysql-test/suite/parts/t/partition_float_innodb.test @@ -8,9 +8,9 @@ #------------------------------------------------------------------------------# # Original Author: HH # # Original Date: 2006-08-01 # -# Change Author: # -# Change Date: # -# Change: # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test for double type has been spawned into a separate test file # ################################################################################ # @@ -44,4 +44,3 @@ let $maxrows=1024; #------------------------------------------------------------------------------# # Execute the tests to be applied to all storage engines --source suite/parts/inc/partition_float.inc ---source suite/parts/inc/partition_double.inc diff --git a/mysql-test/suite/parts/t/partition_float_myisam.test b/mysql-test/suite/parts/t/partition_float_myisam.test index f15e6ad3636..bdc0edd41bb 100644 --- a/mysql-test/suite/parts/t/partition_float_myisam.test +++ b/mysql-test/suite/parts/t/partition_float_myisam.test @@ -8,9 +8,9 @@ #------------------------------------------------------------------------------# # Original Author: HH # # Original Date: 2006-08-01 # -# Change Author: # -# Change Date: # -# Change: # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test for double type has been spawned into a separate test file # ################################################################################ # @@ -44,4 +44,3 @@ let $maxrows=16384; #------------------------------------------------------------------------------# # Execute the tests to be applied to all storage engines --source suite/parts/inc/partition_float.inc ---source suite/parts/inc/partition_double.inc diff --git a/mysql-test/suite/parts/t/partition_int_innodb.test b/mysql-test/suite/parts/t/partition_int_innodb.test index 698a2c93c22..fd00e6a0d80 100644 --- a/mysql-test/suite/parts/t/partition_int_innodb.test +++ b/mysql-test/suite/parts/t/partition_int_innodb.test @@ -8,9 +8,9 @@ #------------------------------------------------------------------------------# # Original Author: HH # # Original Date: 2006-08-01 # -# Change Author: # -# Change Date: # -# Change: # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: Int subtypes (tinyint etc.) have been spawned into separate tests # ################################################################################ # @@ -43,8 +43,4 @@ let $maxrows=1024; #------------------------------------------------------------------------------# # Execute the tests to be applied to all storage engines ---source suite/parts/inc/partition_tinyint.inc ---source suite/parts/inc/partition_smallint.inc --source suite/parts/inc/partition_int.inc ---source suite/parts/inc/partition_mediumint.inc ---source suite/parts/inc/partition_bigint.inc diff --git a/mysql-test/suite/parts/t/partition_int_myisam.test b/mysql-test/suite/parts/t/partition_int_myisam.test index 5f29b575244..e8de09f1bf3 100644 --- a/mysql-test/suite/parts/t/partition_int_myisam.test +++ b/mysql-test/suite/parts/t/partition_int_myisam.test @@ -8,9 +8,9 @@ #------------------------------------------------------------------------------# # Original Author: HH # # Original Date: 2006-08-01 # -# Change Author: # -# Change Date: # -# Change: # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: Int subtypes (tinyint etc.) have been spawned into separate tests # ################################################################################ # @@ -43,8 +43,4 @@ let $maxrows=65535; #------------------------------------------------------------------------------# # Execute the tests to be applied to all storage engines ---source suite/parts/inc/partition_tinyint.inc ---source suite/parts/inc/partition_smallint.inc --source suite/parts/inc/partition_int.inc ---source suite/parts/inc/partition_mediumint.inc ---source suite/parts/inc/partition_bigint.inc diff --git a/mysql-test/suite/parts/t/partition_mediumint_innodb.test b/mysql-test/suite/parts/t/partition_mediumint_innodb.test new file mode 100644 index 00000000000..9218b55fa78 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_mediumint_innodb.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_mediumint_innodb.test # +# # +# Purpose: # +# Tests around integer type # +# INNODB branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_innodb # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT INNODB SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +--source include/have_innodb.inc +let $engine= 'InnoDB'; + +##### max rows to be inserted +let $maxrows=1024; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_mediumint.inc diff --git a/mysql-test/suite/parts/t/partition_mediumint_myisam.test b/mysql-test/suite/parts/t/partition_mediumint_myisam.test new file mode 100644 index 00000000000..bbf1775ba97 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_mediumint_myisam.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_mediumint_myisam.test # +# # +# Purpose: # +# Tests around integer type # +# MyISAM branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_myisam # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +--source include/long_test.inc + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +let $engine= 'MYISAM'; +##### number of rows to be inserted +let $maxrows=65535; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_mediumint.inc diff --git a/mysql-test/suite/parts/t/partition_smallint_innodb.test b/mysql-test/suite/parts/t/partition_smallint_innodb.test new file mode 100644 index 00000000000..22d16cf9d55 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_smallint_innodb.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_smallint_innodb.test # +# # +# Purpose: # +# Tests around integer type # +# INNODB branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_innodb # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT INNODB SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +--source include/have_innodb.inc +let $engine= 'InnoDB'; + +##### max rows to be inserted +let $maxrows=1024; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_smallint.inc diff --git a/mysql-test/suite/parts/t/partition_smallint_myisam.test b/mysql-test/suite/parts/t/partition_smallint_myisam.test new file mode 100644 index 00000000000..f473a6772f8 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_smallint_myisam.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_smallint_myisam.test # +# # +# Purpose: # +# Tests around integer type # +# MyISAM branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_myisam # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +--source include/long_test.inc + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +let $engine= 'MYISAM'; +##### number of rows to be inserted +let $maxrows=65535; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_smallint.inc diff --git a/mysql-test/suite/parts/t/partition_tinyint_innodb.test b/mysql-test/suite/parts/t/partition_tinyint_innodb.test new file mode 100644 index 00000000000..aec10c1aea5 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_tinyint_innodb.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_tinyint_innodb.test # +# # +# Purpose: # +# Tests around integer type # +# INNODB branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_innodb # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT INNODB SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +--source include/have_innodb.inc +let $engine= 'InnoDB'; + +##### max rows to be inserted +let $maxrows=1024; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_tinyint.inc diff --git a/mysql-test/suite/parts/t/partition_tinyint_myisam.test b/mysql-test/suite/parts/t/partition_tinyint_myisam.test new file mode 100644 index 00000000000..9807bffb1da --- /dev/null +++ b/mysql-test/suite/parts/t/partition_tinyint_myisam.test @@ -0,0 +1,46 @@ +################################################################################ +# t/partition_tinyint_myisam.test # +# # +# Purpose: # +# Tests around integer type # +# MyISAM branch # +# # +#------------------------------------------------------------------------------# +# Original Author: HH # +# Original Date: 2006-08-01 # +# Change Author: Elena Stepanova # +# Change Date: 2017-02-18 # +# Change: The test file is spawned from the mega-test partition_int_myisam # +################################################################################ + +# +# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# +# Please read the README at the end of inc/partition.pre before changing +# any of the variables. +# + +--source include/long_test.inc + +#------------------------------------------------------------------------------# +# General not engine specific settings and requirements + +##### Options, for debugging support ##### +let $debug= 0; + +# The server must support partitioning. +--source include/have_partition.inc + +#------------------------------------------------------------------------------# +# Engine specific settings and requirements + +##### Storage engine to be tested +let $engine= 'MYISAM'; +##### number of rows to be inserted +let $maxrows=65535; + +#------------------------------------------------------------------------------# +# Execute the tests to be applied to all storage engines +--source suite/parts/inc/partition_tinyint.inc diff --git a/mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc b/mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc new file mode 100644 index 00000000000..eff3d7df854 --- /dev/null +++ b/mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc @@ -0,0 +1,10 @@ +# threads are removed from: +# - information_schema.processlist +# - performance_schema.threads +# at different times, so we may have to wait a little more +# for the event_scheduler to shutdown +# +let $wait_condition= + SELECT COUNT(*) = 0 FROM performance_schema.threads + WHERE name like 'thread/sql/event%'; +--source include/wait_condition.inc diff --git a/mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc b/mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc new file mode 100644 index 00000000000..219a41051fb --- /dev/null +++ b/mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc @@ -0,0 +1,10 @@ +# threads are removed from: +# - information_schema.processlist +# - performance_schema.threads +# at different times, so we may have to wait a little more +# for the event_scheduler to shutdown +# +let $wait_condition= + SELECT COUNT(*) = 1 FROM performance_schema.threads + WHERE name like 'thread/sql/event%'; +--source include/wait_condition.inc diff --git a/mysql-test/suite/perfschema/r/start_server_1_digest.result b/mysql-test/suite/perfschema/r/start_server_1_digest.result new file mode 100644 index 00000000000..cf07022d344 --- /dev/null +++ b/mysql-test/suite/perfschema/r/start_server_1_digest.result @@ -0,0 +1,7 @@ +SELECT "Digest table has a size 1 and is full already." as use_case; +use_case +Digest table has a size 1 and is full already. +select SCHEMA_NAME, DIGEST, DIGEST_TEXT +from performance_schema.events_statements_summary_by_digest; +SCHEMA_NAME DIGEST DIGEST_TEXT +NULL NULL NULL diff --git a/mysql-test/suite/perfschema/t/start_server_1_digest-master.opt b/mysql-test/suite/perfschema/t/start_server_1_digest-master.opt new file mode 100644 index 00000000000..e59be3eb02f --- /dev/null +++ b/mysql-test/suite/perfschema/t/start_server_1_digest-master.opt @@ -0,0 +1 @@ +--performance-schema-digests-size=1 diff --git a/mysql-test/suite/perfschema/t/start_server_1_digest.test b/mysql-test/suite/perfschema/t/start_server_1_digest.test new file mode 100644 index 00000000000..998d9a5eebe --- /dev/null +++ b/mysql-test/suite/perfschema/t/start_server_1_digest.test @@ -0,0 +1,15 @@ +# ----------------------------------------------------------------------- +# Tests for the performance schema statement Digests. +# ----------------------------------------------------------------------- + +--source include/not_embedded.inc +--source include/have_perfschema.inc +--source include/no_protocol.inc + +SELECT "Digest table has a size 1 and is full already." as use_case; + +select SCHEMA_NAME, DIGEST, DIGEST_TEXT + from performance_schema.events_statements_summary_by_digest; + + + diff --git a/mysql-test/suite/perfschema/t/table_name.test b/mysql-test/suite/perfschema/t/table_name.test index a8179f2d1f8..5fb8ccd0f7f 100644 --- a/mysql-test/suite/perfschema/t/table_name.test +++ b/mysql-test/suite/perfschema/t/table_name.test @@ -31,6 +31,7 @@ INSERT INTO `sql_1` VALUES(1,'one'); --echo --echo # Verify that the tables are treated as normal tables . --echo +--sorted_result SELECT object_type, object_schema, object_name FROM performance_schema.objects_summary_global_by_type WHERE object_schema="test"; diff --git a/mysql-test/suite/perfschema/t/threads_mysql.test b/mysql-test/suite/perfschema/t/threads_mysql.test index 8576c8767d6..c33f421863e 100644 --- a/mysql-test/suite/perfschema/t/threads_mysql.test +++ b/mysql-test/suite/perfschema/t/threads_mysql.test @@ -9,22 +9,10 @@ # Ensure that the event scheduler (started via threads_mysql-master.opt) # is really running. ---source include/running_event_scheduler.inc +--source include/pfs_running_event_scheduler.inc SET GLOBAL event_scheduler = OFF; ---source include/no_running_event_scheduler.inc - -# threads are removed from: -# - information_schema.processlist -# - performance_schema.threads -# at different times, so we may have to wait a little more -# for the event_scheduler to shutdown -# -let $wait_timeout= 1; -let $wait_condition= - SELECT COUNT(*) = 0 FROM performance_schema.threads - WHERE name like 'thread/sql/event%'; ---source include/wait_condition.inc +--source include/pfs_no_running_event_scheduler.inc --vertical_results @@ -59,7 +47,7 @@ WHERE name LIKE 'thread/sql%'; SET GLOBAL event_scheduler = ON; ---source include/running_event_scheduler.inc +--source include/pfs_running_event_scheduler.inc # Show entries belonging to the just started event scheduler SELECT name, type, processlist_user, processlist_host, processlist_db, diff --git a/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result new file mode 100644 index 00000000000..84915073d87 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result @@ -0,0 +1,220 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +connection slave; +select * from t1 order by a; +a +1 +2 +3 +4 +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +connection master; +insert into t1 values(5); +insert into t1 values(6); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#UPDATE query +connection master; +update t1 set a = a+10; +select * from t1 order by a; +a +11 +12 +13 +14 +15 +16 +connection slave; +select * from t1 order by a; +a z1 z2 +11 12 13 +12 13 14 +13 14 15 +14 15 16 +15 16 17 +16 17 18 +connection master; +update t1 set a = a-10; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#DELETE quert +connection master; +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +#REPLACE query +connection master; +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#SELECT query +connection master; +select * from t1 where a > 2 and a < 4; +a +3 +connection slave; +select * from t1 where a > 2 and a < 4; +a z1 z2 +3 4 5 +#UPDATE with SELECT query +connection master; +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +13 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +13 14 15 +connection master; +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; +connection master; +#entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +connection slave; +include/wait_for_slave_sql_error.inc [errno=1062] +connection slave; +connection slave; +select * from t1 order by a; +a z1 z2 z4 +1 2 3 1 +2 3 4 2 +3 4 5 3 +4 5 6 4 +5 6 7 5 +6 7 8 0 +alter table t1 drop column z4; +start slave; +include/wait_for_slave_sql_to_start.inc +connection slave; +connection slave; +connection master; +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +7 8 9 +connection master; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +drop table t1; +include/rpl_end.inc +connection server_2; +connection server_2; +connection server_2; +connection server_2; +connection server_1; +connection server_1; +connection server_1; +connection server_2; +connection server_1; +connection server_2; +connection server_2; +connection server_1; +connection server_1; diff --git a/mysql-test/suite/rpl/r/rpl_heartbeat_basic.result b/mysql-test/suite/rpl/r/rpl_heartbeat_basic.result index 91ed6d7a0cb..0cb1aed9905 100644 --- a/mysql-test/suite/rpl/r/rpl_heartbeat_basic.result +++ b/mysql-test/suite/rpl/r/rpl_heartbeat_basic.result @@ -7,7 +7,6 @@ RESET SLAVE; SET @restore_slave_net_timeout=@@global.slave_net_timeout; RESET MASTER; SET @restore_slave_net_timeout=@@global.slave_net_timeout; -SET @restore_event_scheduler=@@global.event_scheduler; *** Default value *** CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=MASTER_PORT, MASTER_USER='root'; @@ -221,7 +220,7 @@ RESET SLAVE; CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=MASTER_PORT, MASTER_USER='root', MASTER_CONNECT_RETRY=20, MASTER_HEARTBEAT_PERIOD=5; include/start_slave.inc SET @@global.event_scheduler=1; -Number of received heartbeat events: 0 +Received heartbeats meet expectations: TRUE DELETE FROM t1; DROP EVENT e1; diff --git a/mysql-test/suite/rpl/r/rpl_mdev6386.result b/mysql-test/suite/rpl/r/rpl_mdev6386.result index fa49d9a9c03..c8bd6f51822 100644 --- a/mysql-test/suite/rpl/r/rpl_mdev6386.result +++ b/mysql-test/suite/rpl/r/rpl_mdev6386.result @@ -1,7 +1,6 @@ include/master-slave.inc [connection master] ALTER TABLE mysql.gtid_slave_pos ENGINE = InnoDB; -FLUSH LOGS; CREATE TABLE t1 (a INT PRIMARY KEY, b INT) Engine=InnoDB; include/stop_slave.inc SET sql_log_bin= 0; @@ -22,6 +21,7 @@ INSERT INTO t2 VALUE (4, 1); INSERT INTO t2 VALUE (5, 1); INSERT INTO t1 SELECT * FROM t2; DROP TEMPORARY TABLE t2; +INSERT INTO t1 VALUE (6, 3); include/save_master_gtid.inc Contents on master: SELECT * FROM t1 ORDER BY a; @@ -31,6 +31,7 @@ a b 3 1 4 1 5 1 +6 3 START SLAVE; include/wait_for_slave_sql_error.inc [errno=1062] STOP SLAVE IO_THREAD; @@ -51,6 +52,7 @@ a b 3 1 4 1 5 1 +6 3 DROP TABLE t1; include/stop_slave.inc SET GLOBAL slave_parallel_threads= @old_parallel; diff --git a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result index 1489af830cc..19611ac52b5 100644 --- a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result +++ b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result @@ -1,14 +1,21 @@ include/master-slave.inc [connection master] - ---Setup Section -- set timestamp=1000000000; -DROP TABLE IF EXISTS t1,t2,t3; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); - ----Test1 check table load -- +INSERT INTO t1 VALUES ("abirvalg"); +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +set @d1 = 'dd1'; +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +---Test 1 check table load -- SELECT COUNT(*) from t1; COUNT(*) 351 @@ -71,9 +78,7 @@ c1 c3 c4 c5 5 2006-02-22 00:00:00 Tested in Texas 11 insert into t1 values ("Alas"); flush logs; - --- Test 1 Dump binlog to file -- - --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; @@ -84,9 +89,7 @@ reset master; reset slave; start slave; include/wait_for_slave_to_start.inc - --- Test 1 Load from Dump binlog file -- - --- Test 1 Check Load Results -- SELECT COUNT(*) from t1; COUNT(*) @@ -148,7 +151,6 @@ c1 c3 c4 c5 3 2006-02-22 00:00:00 Tested in Texas 6.6 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 - --- Test 2 position test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -172,7 +174,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 3 First Remote test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -189,9 +190,6 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -205,7 +203,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 4 Second Remote test -- DROP TABLE t1; DROP TABLE t2; @@ -276,7 +273,6 @@ c1 c3 c4 c5 3 2006-02-22 00:00:00 Tested in Texas 6.6 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 - --- Test 5 LOAD DATA -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -287,7 +283,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 6 reading stdin -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -304,9 +299,6 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -320,7 +312,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 7 reading stdin w/position -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -344,7 +335,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 8 switch internal charset -- stop slave; include/wait_for_slave_to_stop.inc @@ -380,14 +370,13 @@ HEX(f) select HEX(f) from t5; HEX(f) 835C - --- Test cleanup -- -DROP TABLE IF EXISTS t1; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; CREATE TABLE t1 (a INT NOT NULL KEY, b INT); INSERT INTO t1 VALUES(1,1); SELECT * FROM t1; a b 1 1 FLUSH LOGS; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; +DROP TABLE t1; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_special_charset.result b/mysql-test/suite/rpl/r/rpl_special_charset.result index 99b8d3a21d1..6efe81bf6cb 100644 --- a/mysql-test/suite/rpl/r/rpl_special_charset.result +++ b/mysql-test/suite/rpl/r/rpl_special_charset.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); include/diff_tables.inc [master:t1, slave:t1] diff --git a/mysql-test/suite/rpl/r/sec_behind_master-5114.result b/mysql-test/suite/rpl/r/sec_behind_master-5114.result index 5554b17347d..17f50f0b612 100644 --- a/mysql-test/suite/rpl/r/sec_behind_master-5114.result +++ b/mysql-test/suite/rpl/r/sec_behind_master-5114.result @@ -3,7 +3,8 @@ include/master-slave.inc call mtr.add_suppression("Unsafe statement written to the binary log"); CREATE TABLE t1 (a int); INSERT INTO t1 VALUES(SLEEP(2)); -Seconds_Behind_Master: 1 +Seconds_Behind_Master_is_less_than_100 +1 Warnings: Note 1592 Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. drop table t1; diff --git a/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test new file mode 100644 index 00000000000..3b2fff1cb13 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test @@ -0,0 +1,106 @@ +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--enable_connect_log +--connection master +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); + +--sync_slave_with_master +select * from t1 order by a; +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; + +--connection master +insert into t1 values(5); +insert into t1 values(6); + +--sync_slave_with_master +select * from t1 order by a; + + +--echo #UPDATE query + +--connection master +update t1 set a = a+10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a-10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #DELETE quert +--connection master +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #REPLACE query +--connection master +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); + +--sync_slave_with_master +select * from t1 order by a; + +--echo #SELECT query +--connection master +select * from t1 where a > 2 and a < 4; + +--connection slave +select * from t1 where a > 2 and a < 4; + +--echo #UPDATE with SELECT query +--connection master +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; + +--connection master + +--echo #entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; + +--connection slave +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +select * from t1 order by a; +alter table t1 drop column z4; +start slave; + +--source include/wait_for_slave_sql_to_start.inc + +--connection master +--sync_slave_with_master +select * from t1 order by a; + +--connection master +select * from t1 order by a; +drop table t1; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_heartbeat_basic.test b/mysql-test/suite/rpl/t/rpl_heartbeat_basic.test index 5b55f11da85..4c8d3a1fedb 100644 --- a/mysql-test/suite/rpl/t/rpl_heartbeat_basic.test +++ b/mysql-test/suite/rpl/t/rpl_heartbeat_basic.test @@ -34,7 +34,6 @@ eval SET @restore_slave_heartbeat_timeout=$slave_heartbeat_timeout; --connection master RESET MASTER; SET @restore_slave_net_timeout=@@global.slave_net_timeout; -SET @restore_event_scheduler=@@global.event_scheduler; --echo # @@ -352,21 +351,54 @@ eval CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=$MASTER_MYPORT, MASTE --connection master # Enable scheduler SET @@global.event_scheduler=1; + --sync_slave_with_master let $rcvd_heartbeats_before= query_get_value(SHOW STATUS LIKE 'slave_received_heartbeats', Value, 1); -# Wait some updates for table t1 from master -let $wait_condition= SELECT COUNT(*)=1 FROM t1 WHERE a > 5; ---source include/wait_condition.inc + +--connection master + +# Whether or not to send a heartbeat is decided on the master, based on +# whether the binlog was updated during the period or not. +# Even with the 1-second event, we cannot make the master to write binary +# logs (or execute SQL) in a timely manner. We can only check that they +# were executed in a timely manner, and if they were not, neutralize the +# heartbeat check on the slave. +# We will wait for 5 events, and keep checking 'Binlog_commits' on master. +# Time interval between consequent events will be measured. +# We can only expect that no heartbeats have been sent if the interval +# between events never exceeded MASTER_HEARTBEAT_PERIOD. +# If it has exceeded the value at least once, the slave can legitimately +# receive a heartbeat (but we cannot require it, because the delay +# could have occurred somewhere else, e.g. upon checking the status). +# So, if the delay is detected, we will signal slave to ignore possible +# heartbeats. + +let $possible_heartbeats= 0; +let $commits_to_wait= 5; +while ($commits_to_wait) +{ + let $tm= `SELECT UNIX_TIMESTAMP(NOW(3))`; + let $binlog_commits= query_get_value(SHOW STATUS LIKE 'Binlog_commits', Value, 1); + let $wait_condition= SELECT VARIABLE_VALUE > $binlog_commits FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME= 'BINLOG_COMMITS'; + --source include/wait_condition.inc + dec $commits_to_wait; + if (`SELECT UNIX_TIMESTAMP(NOW(3)) > $tm + 5`) + { + let $possible_heartbeats= 1; + let $commits_to_wait= 0; + } +} + +--connection slave let $rcvd_heartbeats_after= query_get_value(SHOW STATUS LIKE 'slave_received_heartbeats', Value, 1); -let $result= query_get_value(SELECT ($rcvd_heartbeats_after - $rcvd_heartbeats_before) > 0 AS Result, Result, 1); ---echo Number of received heartbeat events: $result +let $result= `SELECT CASE WHEN $possible_heartbeats THEN 'TRUE' WHEN $rcvd_heartbeats_after - $rcvd_heartbeats_before > 0 THEN 'FALSE' ELSE 'TRUE' END`; +--echo Received heartbeats meet expectations: $result --connection master DELETE FROM t1; DROP EVENT e1; --sync_slave_with_master --echo - # Check received heartbeat events while logs flushed on slave --echo *** Flush logs on slave *** STOP SLAVE; diff --git a/mysql-test/suite/rpl/t/rpl_mdev6386.test b/mysql-test/suite/rpl/t/rpl_mdev6386.test index 3e4e79ea5a3..e85b1ae0132 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev6386.test +++ b/mysql-test/suite/rpl/t/rpl_mdev6386.test @@ -2,9 +2,7 @@ --source include/master-slave.inc --connection master -# ToDo: Remove this FLUSH LOGS when MDEV-6403 is fixed. ALTER TABLE mysql.gtid_slave_pos ENGINE = InnoDB; -FLUSH LOGS; CREATE TABLE t1 (a INT PRIMARY KEY, b INT) Engine=InnoDB; --sync_slave_with_master @@ -31,6 +29,7 @@ INSERT INTO t2 VALUE (4, 1); INSERT INTO t2 VALUE (5, 1); INSERT INTO t1 SELECT * FROM t2; DROP TEMPORARY TABLE t2; +INSERT INTO t1 VALUE (6, 3); --source include/save_master_gtid.inc --echo Contents on master: SELECT * FROM t1 ORDER BY a; diff --git a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test index ed0f31b75be..678679f0cf1 100644 --- a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test +++ b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test @@ -4,43 +4,27 @@ # Purpose: To test changes to mysqlbinlog for row based bin logs # # We are using .opt file since we need small binlog size # ################################################################## -# Include Section -# Make sure that we have row based bin log -- source include/have_binlog_format_row.inc -# Embedded server doesn't support binlogging -- source include/not_embedded.inc -# This test requires the cp932 charset compiled in -- source include/have_cp932.inc -# Slow test, don't run during staging part --- source include/not_staging.inc - -- source include/master-slave.inc -# Setup Section -# we need this for getting fixed timestamps inside of this test - ---disable_query_log -select "---Setup Section --" as ""; ---enable_query_log +--echo ---Setup Section -- +# we need this for getting fixed timestamps inside of this test set timestamp=1000000000; ---disable_warnings -DROP TABLE IF EXISTS t1,t2,t3; ---enable_warnings - -connection master; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); ---let $position= query_get_value(SHOW MASTER STATUS, Position, 1) +--let position= query_get_value(SHOW MASTER STATUS, Position, 1) CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); ---let $stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) ---let $stop_position1=`select $stop_position - 1` ---let $binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) +--let stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) +--let stop_position1=`select $stop_position - 1` +--let binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) + # Test Section # Lets start by putting some data into the tables. ---disable_query_log INSERT INTO t1 VALUES ("abirvalg"); LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; @@ -54,7 +38,8 @@ set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); -let $count=500; +--disable_query_log +let count=500; while ($count) { INSERT INTO t2 VALUES (NULL); @@ -63,10 +48,7 @@ while ($count) } --enable_query_log - ---disable_query_log -select "---Test1 check table load --" as ""; ---enable_query_log +--echo ---Test 1 check table load -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -95,34 +77,26 @@ insert into t1 values ("Alas"); flush logs; # delimiters are for easier debugging in future ---disable_query_log -select "--- Test 1 Dump binlog to file --" as ""; ---enable_query_log +--echo --- Test 1 Dump binlog to file -- # # Prepare local temporary file to recreate what we have currently. -let $MYSQLD_DATADIR= `select @@datadir;`; +let MYSQLD_DATADIR= `select @@datadir;`; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/master.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/master.sql # Now that we have our file, lets get rid of the current database. # Cleanup the master and the slave and try to recreate. ---disable_query_log -select "--- Test 1 delete tables, clean master and slave --" as ""; ---enable_query_log +--echo --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -132,15 +106,11 @@ start slave; connection master; # We should be clean at this point, now we will run in the file from above. ---disable_query_log -select "--- Test 1 Load from Dump binlog file --" as ""; ---enable_query_log +--echo --- Test 1 Load from Dump binlog file -- --exec $MYSQL -e "source $MYSQLTEST_VARDIR/tmp/master.sql" ---disable_query_log -select "--- Test 1 Check Load Results --" as ""; ---enable_query_log +--echo --- Test 1 Check Load Results -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -168,28 +138,20 @@ remove_file $MYSQLTEST_VARDIR/tmp/master.sql; # this test for start-position option # By setting this position to 416, we should only get the create of t3 ---disable_query_log -select "--- Test 2 position test --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 2 position test -- --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --start-position=$position --stop-position=$stop_position $MYSQLD_DATADIR/master-bin.000001 # These are tests for remote binlog. # They should return the same as previous test. ---disable_query_log -select "--- Test 3 First Remote test --" as ""; ---enable_query_log +--echo --- Test 3 First Remote test -- # This is broken now --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$stop_position --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 ---disable_query_log -select "--- Test 4 Second Remote test --" as ""; ---enable_query_log +--echo --- Test 4 Second Remote test -- --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql - --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/remote.sql # Now that we have our file, lets get rid of the current database. @@ -201,13 +163,8 @@ DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -251,40 +208,26 @@ connection master; # transactions. /Matz # LOAD DATA ---disable_query_log -select "--- Test 5 LOAD DATA --" as ""; ---enable_query_log +--echo --- Test 5 LOAD DATA -- --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$binlog_start_pos --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 # Bug#7853 (mysqlbinlog does not accept input from stdin) ---disable_query_log -select "--- Test 6 reading stdin --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 6 reading stdin -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --stop-position=$stop_position1 - < $MYSQLD_DATADIR/master-bin.000001 ---disable_query_log -select "--- Test 7 reading stdin w/position --" as ""; ---enable_query_log +--echo --- Test 7 reading stdin w/position -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --start-position=$position --stop-position=$stop_position - < $MYSQLD_DATADIR/master-bin.000001 # Bug#16217 (mysql client did not know how not switch its internal charset) ---disable_query_log -select "--- Test 8 switch internal charset --" as ""; ---enable_query_log +--echo --- Test 8 switch internal charset -- sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -297,7 +240,6 @@ create table t4 (f text character set utf8); create table t5 (f text character set cp932); --exec $MYSQL --default-character-set=utf8 test -e "insert into t4 values(_utf8'ソ')" --exec $MYSQL --default-character-set=cp932 test -e "insert into t5 values(_cp932'ƒ\');" -let $MYSQLD_DATADIR= `select @@datadir;`; flush logs; rename table t4 to t04, t5 to t05; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 | $MYSQL --default-character-set=utf8 @@ -314,42 +256,30 @@ select HEX(f) from t4; select HEX(f) from t05; select HEX(f) from t5; ---disable_query_log -select "--- Test cleanup --" as ""; ---enable_query_log +--echo --- Test cleanup -- # clean up connection master; sync_slave_with_master; connection master; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; # BUG#17654 also test mysqlbinlog to ensure it can read the binlog from a remote server # and ensure that the results are the same as if read from a file (the same file). ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings - CREATE TABLE t1 (a INT NOT NULL KEY, b INT); - INSERT INTO t1 VALUES(1,1); - SELECT * FROM t1; - -let $MYSQLD_DATADIR= `select @@datadir;`; - FLUSH LOGS; --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/local.sql --diff_files $MYSQLTEST_VARDIR/tmp/local.sql $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/local.sql +DROP TABLE t1; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; sync_slave_with_master; # End of 4.1 tests diff --git a/mysql-test/suite/rpl/t/rpl_special_charset.test b/mysql-test/suite/rpl/t/rpl_special_charset.test index 8ccb1b4183f..3a947d413bb 100644 --- a/mysql-test/suite/rpl/t/rpl_special_charset.test +++ b/mysql-test/suite/rpl/t/rpl_special_charset.test @@ -14,7 +14,7 @@ # then set default's client character set(latin1) as client's character set. ############################################################################### --source include/master-slave.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); --sync_slave_with_master diff --git a/mysql-test/suite/rpl/t/sec_behind_master-5114.test b/mysql-test/suite/rpl/t/sec_behind_master-5114.test index 8c70da4f0a8..ff8cab54c4f 100644 --- a/mysql-test/suite/rpl/t/sec_behind_master-5114.test +++ b/mysql-test/suite/rpl/t/sec_behind_master-5114.test @@ -3,24 +3,60 @@ source include/have_binlog_format_statement.inc; call mtr.add_suppression("Unsafe statement written to the binary log"); + +# Make sure that the start time of the first event is certainly different +# from the next event +--let $timestamp= `SELECT @@timestamp` +--disable_query_log +eval SET TIMESTAMP= $timestamp-100; +--enable_query_log CREATE TABLE t1 (a int); + +# Make sure that the slave is done with the first event, and all checks +# that we'll perform later will be really against the second event +sync_slave_with_master; + +connection master; + +# Restore the timestamp now. It doesn't matter that it's not precise, +# it just needs to be very different from the earlier event +--disable_query_log +eval SET TIMESTAMP= $timestamp; +--enable_query_log + send INSERT INTO t1 VALUES(SLEEP(2)); connection slave; -let $run = 10; + +# When the slave starts executing the event, Seconds_Behind_Master +# should start growing steadilly. The bugfix ensures that they are +# calculated based on the start time of the current event, rather +# than the start time of the previous event. To check it, we only need +# the first non-zero value + +let $run = 20; while ($run) { dec $run; let $sbm=query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1); # for debugging uncomment echo and remove the if() - #echo Seconds_Behind_Master: $sbm; +# echo Seconds_Behind_Master: $sbm; if ($sbm) { let $run = 0; } sleep 0.5; } -echo Seconds_Behind_Master: $sbm; + +# Normally the first non-zero value should be 1. However, due to race +# conditions on slow servers, sometimes the check might miss the value 1, +# and only catch a higher one. It does not matter, we just need to make +# sure it didn't start with 100+, as it would have with bug MDEV-5114 + +--disable_query_log +eval SELECT $sbm > 0 and $sbm < 99 AS Seconds_Behind_Master_is_less_than_100; +--enable_query_log + connection master; reap; drop table t1; diff --git a/mysql-test/suite/sys_vars/r/innodb_force_recovery_crash_basic.result b/mysql-test/suite/sys_vars/r/innodb_force_recovery_crash_basic.result deleted file mode 100644 index 5af00f21c74..00000000000 --- a/mysql-test/suite/sys_vars/r/innodb_force_recovery_crash_basic.result +++ /dev/null @@ -1,33 +0,0 @@ -select @@global.innodb_force_recovery_crash in (0, 1); -@@global.innodb_force_recovery_crash in (0, 1) -1 -select @@global.innodb_force_recovery_crash; -@@global.innodb_force_recovery_crash -0 -select @@session.innodb_force_recovery_crash; -ERROR HY000: Variable 'innodb_force_recovery_crash' is a GLOBAL variable -show global variables like 'innodb_force_recovery_crash'; -Variable_name Value -innodb_force_recovery_crash 0 -show session variables like 'innodb_force_recovery_crash'; -Variable_name Value -innodb_force_recovery_crash 0 -select * from information_schema.global_variables where variable_name='innodb_force_recovery_crash'; -VARIABLE_NAME VARIABLE_VALUE -INNODB_FORCE_RECOVERY_CRASH 0 -select * from information_schema.session_variables where variable_name='innodb_force_recovery_crash'; -VARIABLE_NAME VARIABLE_VALUE -INNODB_FORCE_RECOVERY_CRASH 0 -set global innodb_force_recovery_crash=1; -ERROR HY000: Variable 'innodb_force_recovery_crash' is a read only variable -set global innodb_force_recovery_crash=0; -ERROR HY000: Variable 'innodb_force_recovery_crash' is a read only variable -select @@global.innodb_force_recovery_crash; -@@global.innodb_force_recovery_crash -0 -set session innodb_force_recovery_crash='some'; -ERROR HY000: Variable 'innodb_force_recovery_crash' is a read only variable -set @@session.innodb_force_recovery_crash='some'; -ERROR HY000: Variable 'innodb_force_recovery_crash' is a read only variable -set global innodb_force_recovery_crash='some'; -ERROR HY000: Variable 'innodb_force_recovery_crash' is a read only variable diff --git a/mysql-test/suite/sys_vars/r/innodb_stats_include_delete_marked_basic.result b/mysql-test/suite/sys_vars/r/innodb_stats_include_delete_marked_basic.result new file mode 100644 index 00000000000..ffd208e7927 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_stats_include_delete_marked_basic.result @@ -0,0 +1,25 @@ +SELECT @@innodb_stats_include_delete_marked; +@@innodb_stats_include_delete_marked +0 +SET GLOBAL innodb_stats_include_delete_marked=1; +SELECT @@innodb_stats_include_delete_marked; +@@innodb_stats_include_delete_marked +1 +SET SESSION innodb_stats_include_delete_marked=1; +ERROR HY000: Variable 'innodb_stats_include_delete_marked' is a GLOBAL variable and should be set with SET GLOBAL +SET GLOBAL innodb_stats_include_delete_marked=100; +ERROR 42000: Variable 'innodb_stats_include_delete_marked' can't be set to the value of '100' +SET GLOBAL innodb_stats_include_delete_marked=foo; +ERROR 42000: Variable 'innodb_stats_include_delete_marked' can't be set to the value of 'foo' +SET GLOBAL innodb_stats_include_delete_marked=OFF ; +SELECT @@innodb_stats_include_delete_marked; +@@innodb_stats_include_delete_marked +0 +SET GLOBAL innodb_stats_include_delete_marked=ON ; +SELECT @@innodb_stats_include_delete_marked; +@@innodb_stats_include_delete_marked +1 +SET GLOBAL innodb_stats_include_delete_marked=Default ; +SELECT @@innodb_stats_include_delete_marked; +@@innodb_stats_include_delete_marked +0 diff --git a/mysql-test/suite/sys_vars/t/innodb_force_recovery_crash_basic.test b/mysql-test/suite/sys_vars/t/innodb_force_recovery_crash_basic.test deleted file mode 100644 index 5eefe1b9219..00000000000 --- a/mysql-test/suite/sys_vars/t/innodb_force_recovery_crash_basic.test +++ /dev/null @@ -1,28 +0,0 @@ ---source include/have_innodb.inc ---source include/have_debug.inc - -# -# exists as global only -# -select @@global.innodb_force_recovery_crash in (0, 1); -select @@global.innodb_force_recovery_crash; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -select @@session.innodb_force_recovery_crash; -show global variables like 'innodb_force_recovery_crash'; -show session variables like 'innodb_force_recovery_crash'; -select * from information_schema.global_variables where variable_name='innodb_force_recovery_crash'; -select * from information_schema.session_variables where variable_name='innodb_force_recovery_crash'; - -# show that it's read-only -# ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set global innodb_force_recovery_crash=1; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set global innodb_force_recovery_crash=0; -select @@global.innodb_force_recovery_crash; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set session innodb_force_recovery_crash='some'; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set @@session.innodb_force_recovery_crash='some'; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set global innodb_force_recovery_crash='some'; diff --git a/mysql-test/suite/sys_vars/t/innodb_stats_include_delete_marked_basic.test b/mysql-test/suite/sys_vars/t/innodb_stats_include_delete_marked_basic.test new file mode 100644 index 00000000000..2a3a0f9b44e --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_stats_include_delete_marked_basic.test @@ -0,0 +1,53 @@ +############################################################################### +# # +# Variable Name: innodb_stats_include_delete_marked # +# Scope: Global # +# Access Type: Dynamic # +# Data Type: numeric # +# # +# # +# Creation Date: 2016-08-29 # +# Author : Aditya # +# # +# # +# Description: # +# * Value check # +# * Scope check # +# # +############################################################################### + +--source include/have_innodb.inc + +#################################################################### +# Display default value # +#################################################################### +SELECT @@innodb_stats_include_delete_marked; + +SET GLOBAL innodb_stats_include_delete_marked=1; + +SELECT @@innodb_stats_include_delete_marked; + +# check error +--error ER_GLOBAL_VARIABLE +SET SESSION innodb_stats_include_delete_marked=1; + +# check error +--error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL innodb_stats_include_delete_marked=100; + +# check error +--error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL innodb_stats_include_delete_marked=foo; + +SET GLOBAL innodb_stats_include_delete_marked=OFF ; + +SELECT @@innodb_stats_include_delete_marked; + +SET GLOBAL innodb_stats_include_delete_marked=ON ; + +SELECT @@innodb_stats_include_delete_marked; + +# Check with default setting +SET GLOBAL innodb_stats_include_delete_marked=Default ; + +SELECT @@innodb_stats_include_delete_marked; diff --git a/mysql-test/suite/sys_vars/t/secure_file_priv.test b/mysql-test/suite/sys_vars/t/secure_file_priv.test index 5c53da58275..a5a465d8c98 100644 --- a/mysql-test/suite/sys_vars/t/secure_file_priv.test +++ b/mysql-test/suite/sys_vars/t/secure_file_priv.test @@ -21,6 +21,9 @@ SHOW VARIABLES LIKE 'secure_file_priv'; --perl use File::Basename; my $protected_file= dirname($ENV{MYSQLTEST_VARDIR}).'/bug50373.txt'; +# Ensure bug50373.txt does not exist (e.g. leftover from previous +# test runs). +unlink $protected_file; open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/bug50373.inc") or die; print FILE "SELECT * FROM t1 INTO OUTFILE '".$protected_file."';\n"; print FILE "DELETE FROM t1;\n"; diff --git a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc index eb7e6ad32b9..b9a6ba28a5f 100644 --- a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc +++ b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc @@ -110,7 +110,7 @@ drop table t1,t2; drop procedure p1; --echo # ---echo # Bug mdev-3845: values of virtual columns are not computed for triggers +--echo # MDEV-3845 values of virtual columns are not computed for triggers --echo # CREATE TABLE t1 ( @@ -149,3 +149,10 @@ DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +# +# MDEV-11706 Assertion `is_stat_field || !table || (!table->write_set || bitmap_is_set(table->write_set, field_index) || (table->vcol_set && bitmap_is_set(table->vcol_set, field_index)))' failed in Field_time::store_TIME_with_warning +# +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +drop table t1; diff --git a/mysql-test/suite/vcol/r/vcol_misc.result b/mysql-test/suite/vcol/r/vcol_misc.result index 0aaed59ed6c..699b6d4efe3 100644 --- a/mysql-test/suite/vcol/r/vcol_misc.result +++ b/mysql-test/suite/vcol/r/vcol_misc.result @@ -330,3 +330,10 @@ t1 CREATE TABLE `t1` ( `c1` varchar(50) COLLATE latin1_general_ci DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci drop table t1; +set sql_mode='no_zero_date'; +create table t1 ( +ts timestamp not null default current_timestamp, +tsv timestamp as (adddate(ts, interval 1 day)) virtual +); +drop table t1; +set sql_mode=default; diff --git a/mysql-test/suite/vcol/r/vcol_select_myisam.result b/mysql-test/suite/vcol/r/vcol_select_myisam.result index 934d047f6bf..6dee132b3e5 100644 --- a/mysql-test/suite/vcol/r/vcol_select_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_select_myisam.result @@ -295,3 +295,112 @@ Note 1003 select `test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a` from `test`.`t1` SELECT * FROM t1 NATURAL JOIN t2; b a DROP TABLE t1,t2; +create table t1 ( +pk integer auto_increment, +bi integer not null, +vi integer generated always as (bi) persistent, +bc varchar(1) not null, +vc varchar(2) generated always as (concat(bc, bc)) persistent, +primary key (pk), +key (vi, vc)); +insert t1 (bi, bc) values (0, 'x'), (0, 'n'), (1, 'w'), (7, 's'), (0, 'a'), (4, 'd'), (1, 'w'), (1, 'j'), (1, 'm'), (4, 'k'), (7, 't'), (4, 'k'), (2, 'e'), (0, 'i'), (1, 't'), (6, 'z'), (3, 'c'), (6, 'i'), (8, 'v'); +create table t2 ( +pk integer auto_increment, +bi integer not null, +vi integer generated always as (bi) persistent, +bc varchar(257) not null, +vc varchar(2) generated always as (concat(bc, bc)) persistent, +primary key (pk), +key (vi, vc)); +insert t2 (bi, bc) values (1, 'c'), (8, 'm'), (9, 'd'), (6, 'y'), (1, 't'), (6, 'd'), (2, 's'), (4, 'r'), (8, 'm'), (4, 'b'), (4, 'x'), (7, 'g'), (4, 'p'), (1, 'q'), (9, 'w'), (4, 'd'), (8, 'e'), (4, 'b'), (8, 'y'); +explain # should be using join buffer +select t2.vi from (t2 as t3 right join (t2 left join t1 on (t1.bi = t2.vi)) on (t1.vc = t2.vc)); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 index NULL vi 10 NULL 19 Using index +1 SIMPLE t1 ALL NULL NULL NULL NULL 19 Using where; Using join buffer (flat, BNL join) +1 SIMPLE t3 index NULL PRIMARY 4 NULL 19 Using where; Using index; Using join buffer (incremental, BNL join) +select t2.vi from (t2 as t3 right join (t2 left join t1 on (t1.bi = t2.vi)) on (t1.vc = t2.vc)); +vi +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +2 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +4 +6 +6 +6 +6 +7 +7 +8 +8 +8 +8 +9 +9 +drop table t2,t1; diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result index 1d78bbf50e4..034a67a9bb0 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result @@ -86,7 +86,7 @@ a b c drop table t1,t2; drop procedure p1; # -# Bug mdev-3845: values of virtual columns are not computed for triggers +# MDEV-3845 values of virtual columns are not computed for triggers # CREATE TABLE t1 ( a INTEGER UNSIGNED NULL DEFAULT NULL, @@ -125,3 +125,9 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +Warnings: +Warning 1364 Field 't' doesn't have a default value +drop table t1; diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result index 77efa8fe6b9..07d011ac64c 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result @@ -86,7 +86,7 @@ a b c drop table t1,t2; drop procedure p1; # -# Bug mdev-3845: values of virtual columns are not computed for triggers +# MDEV-3845 values of virtual columns are not computed for triggers # CREATE TABLE t1 ( a INTEGER UNSIGNED NULL DEFAULT NULL, @@ -125,3 +125,9 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +Warnings: +Warning 1364 Field 't' doesn't have a default value +drop table t1; diff --git a/mysql-test/suite/vcol/r/wrong_arena.result b/mysql-test/suite/vcol/r/wrong_arena.result new file mode 100644 index 00000000000..172b59d6c4c --- /dev/null +++ b/mysql-test/suite/vcol/r/wrong_arena.result @@ -0,0 +1,61 @@ +create table t1 (a datetime, +# get_datetime_value +b int as (a > 1), # Arg_comparator +c int as (a in (1,2,3)), # in_datetime +d int as ((a,a) in ((1,1),(2,1),(NULL,1))), # cmp_item_datetime +# other issues +e int as ((a,1) in ((1,1),(2,1),(NULL,1))) # cmp_item_row::alloc_comparators() +); +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '3' +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` datetime DEFAULT NULL, + `b` int(11) AS (a > 1) VIRTUAL, + `c` int(11) AS (a in (1,2,3)) VIRTUAL, + `d` int(11) AS ((a,a) in ((1,1),(2,1),(NULL,1))) VIRTUAL, + `e` int(11) AS ((a,1) in ((1,1),(2,1),(NULL,1))) VIRTUAL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '3' +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +a b c d e +2010-10-10 10:10:10 1 0 0 NULL +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +select * from t1; +a b c d e +2010-10-10 10:10:10 1 0 0 NULL +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +drop table t1; +create table t1 (a datetime, +b datetime as (least(a,1)) # Item_func_min_max::get_date +); +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +a b +2010-10-10 10:10:10 0000-00-00 00:00:00 +Warnings: +Warning 1292 Incorrect datetime value: '1' +select * from t1; +a b +2010-10-10 10:10:10 0000-00-00 00:00:00 +Warnings: +Warning 1292 Incorrect datetime value: '1' +drop table t1; diff --git a/mysql-test/suite/vcol/t/vcol_misc.test b/mysql-test/suite/vcol/t/vcol_misc.test index 12f46e9b002..80a36d9c623 100644 --- a/mysql-test/suite/vcol/t/vcol_misc.test +++ b/mysql-test/suite/vcol/t/vcol_misc.test @@ -290,3 +290,14 @@ create table t1 (a int, b int as (b is null) virtual); create table t1 (v1 varchar(255) as (c1) persistent, c1 varchar(50)) collate=latin1_general_ci; show create table t1; drop table t1; + +# +# MDEV-11527 Virtual columns do not get along well with NO_ZERO_DATE +# +set sql_mode='no_zero_date'; +create table t1 ( + ts timestamp not null default current_timestamp, + tsv timestamp as (adddate(ts, interval 1 day)) virtual +); +drop table t1; +set sql_mode=default; diff --git a/mysql-test/suite/vcol/t/vcol_select_myisam.test b/mysql-test/suite/vcol/t/vcol_select_myisam.test index c14faba576d..b392b74c2d9 100644 --- a/mysql-test/suite/vcol/t/vcol_select_myisam.test +++ b/mysql-test/suite/vcol/t/vcol_select_myisam.test @@ -68,3 +68,35 @@ SELECT * FROM t1 NATURAL JOIN t2; SELECT * FROM t1 NATURAL JOIN t2; DROP TABLE t1,t2; + +# +# MDEV-11525 Assertion `cp + len <= buff + buff_size' failed in JOIN_CACHE::write_record_data +# + +create table t1 ( + pk integer auto_increment, + bi integer not null, + vi integer generated always as (bi) persistent, + bc varchar(1) not null, + vc varchar(2) generated always as (concat(bc, bc)) persistent, + primary key (pk), + key (vi, vc)); +insert t1 (bi, bc) values (0, 'x'), (0, 'n'), (1, 'w'), (7, 's'), (0, 'a'), (4, 'd'), (1, 'w'), (1, 'j'), (1, 'm'), (4, 'k'), (7, 't'), (4, 'k'), (2, 'e'), (0, 'i'), (1, 't'), (6, 'z'), (3, 'c'), (6, 'i'), (8, 'v'); +create table t2 ( + pk integer auto_increment, + bi integer not null, + vi integer generated always as (bi) persistent, + bc varchar(257) not null, + vc varchar(2) generated always as (concat(bc, bc)) persistent, + primary key (pk), + key (vi, vc)); +insert t2 (bi, bc) values (1, 'c'), (8, 'm'), (9, 'd'), (6, 'y'), (1, 't'), (6, 'd'), (2, 's'), (4, 'r'), (8, 'm'), (4, 'b'), (4, 'x'), (7, 'g'), (4, 'p'), (1, 'q'), (9, 'w'), (4, 'd'), (8, 'e'), (4, 'b'), (8, 'y'); +explain # should be using join buffer +select t2.vi from (t2 as t3 right join (t2 left join t1 on (t1.bi = t2.vi)) on (t1.vc = t2.vc)); +--sorted_result +select t2.vi from (t2 as t3 right join (t2 left join t1 on (t1.bi = t2.vi)) on (t1.vc = t2.vc)); +drop table t2,t1; + +# +# End of 5.5 tests +# diff --git a/mysql-test/suite/vcol/t/wrong_arena.test b/mysql-test/suite/vcol/t/wrong_arena.test new file mode 100644 index 00000000000..484f1fe685d --- /dev/null +++ b/mysql-test/suite/vcol/t/wrong_arena.test @@ -0,0 +1,35 @@ +# +# This tests various issues when vcol items allocate memory (e.g. more items) +# not in the TABLE::expr_arena. +# + +# +# MDEV-9690 concurrent queries with virtual columns crash in temporal code +# +create table t1 (a datetime, + # get_datetime_value + b int as (a > 1), # Arg_comparator + c int as (a in (1,2,3)), # in_datetime + d int as ((a,a) in ((1,1),(2,1),(NULL,1))), # cmp_item_datetime + # other issues + e int as ((a,1) in ((1,1),(2,1),(NULL,1))) # cmp_item_row::alloc_comparators() +); +show create table t1; +connect con1, localhost, root; +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +disconnect con1; +connection default; +select * from t1; +drop table t1; + +connect con1, localhost, root; +create table t1 (a datetime, + b datetime as (least(a,1)) # Item_func_min_max::get_date +); +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +disconnect con1; +connection default; +select * from t1; +drop table t1; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index d2b8a6082a6..3670e871bb0 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1737,3 +1737,33 @@ SHOW CREATE TABLE `tab1`; ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT; SHOW CREATE TABLE `tab1`; DROP TABLE `tab1`; + +--echo # +--echo # MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS +--echo # + +CREATE TABLE t1 (id INT UNSIGNED NOT NULL PRIMARY KEY); +CREATE TABLE t2 (id1 INT UNSIGNED NOT NULL); + +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) + REFERENCES t1 (id); + +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); + +DROP TABLE t2; +DROP TABLE t1; + + +--echo # +--echo # MDEV-6390 CONVERT TO CHARACTER SET utf8 doesn't change DEFAULT CHARSET. +--echo # + +CREATE TABLE t1 (id int(11) NOT NULL, a int(11) NOT NULL, b int(11)) + ENGINE=InnoDB DEFAULT CHARSET=latin1; +SHOW CREATE TABLE t1; +ALTER TABLE t1 CONVERT TO CHARACTER SET utf8; +SHOW CREATE TABLE t1; +DROP TABLE t1; diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test index 999a50f3acb..a51fd0b8a8f 100644 --- a/mysql-test/t/create.test +++ b/mysql-test/t/create.test @@ -2081,6 +2081,15 @@ DELIMITER ;| create table t1 as select f1(); drop function f1; +--echo # +--echo # MDEV-10274 Bundling insert with create statement +--echo # for table with unsigned Decimal primary key issues warning 1194 +--echo # + +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory + select 2.1 ID; +drop table t1; + --echo End of 5.5 tests # diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test index 33d41e9bc4c..165d945944e 100644 --- a/mysql-test/t/ctype_ucs.test +++ b/mysql-test/t/ctype_ucs.test @@ -811,6 +811,14 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)); +--echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=ucs2; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; --echo # --echo # End of 5.5 tests diff --git a/mysql-test/t/ctype_ucs2_def.test b/mysql-test/t/ctype_ucs2_def.test index be8e044f2e4..e297fa5ccf1 100644 --- a/mysql-test/t/ctype_ucs2_def.test +++ b/mysql-test/t/ctype_ucs2_def.test @@ -1,6 +1,6 @@ -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # MySQL Bug#15276: MySQL ignores collation-server diff --git a/mysql-test/t/ctype_ucs2_query_cache.test b/mysql-test/t/ctype_ucs2_query_cache.test index acb39419751..ace826aec44 100644 --- a/mysql-test/t/ctype_ucs2_query_cache.test +++ b/mysql-test/t/ctype_ucs2_query_cache.test @@ -1,7 +1,7 @@ -- source include/have_query_cache.inc -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); --echo # --echo # Start of 5.5 tests diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test index e4305ed9879..cda85239512 100644 --- a/mysql-test/t/ctype_utf16.test +++ b/mysql-test/t/ctype_utf16.test @@ -796,6 +796,15 @@ DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); --echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=utf16; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/ctype_utf16_def.test b/mysql-test/t/ctype_utf16_def.test index fad61b057c3..0829cd53285 100644 --- a/mysql-test/t/ctype_utf16_def.test +++ b/mysql-test/t/ctype_utf16_def.test @@ -1,5 +1,5 @@ --source include/have_utf16.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); # # Bug #32391 Character sets: crash with --character-set-server diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test index 600880d6be5..2f5952045c5 100644 --- a/mysql-test/t/ctype_utf32.test +++ b/mysql-test/t/ctype_utf32.test @@ -895,6 +895,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061)); select hex(lower(cast(0xffff0000 as char character set utf32))) as c; --echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=utf32; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/derived.test b/mysql-test/t/derived.test index b517dfd29d1..28e48bf03c0 100644 --- a/mysql-test/t/derived.test +++ b/mysql-test/t/derived.test @@ -875,4 +875,27 @@ SELECT Customer, Success, SUM(OrderSize) DROP TABLE example1463; set sql_mode= @save_sql_mode; +--echo # +--echo # MDEV-9028: SELECT DISTINCT constant column of derived table +--echo # used as the second operand of LEFT JOIN +--echo # + +create table t1 (id int, data varchar(255)); +insert into t1 values (1,'yes'),(2,'yes'); + +select distinct t1.id, tt.id, tt.data + from t1 + left join + (select t1.id, 'yes' as data from t1) as tt + on t1.id = tt.id; + +select distinct t1.id, tt.id, tt.data + from t1 + left join + (select t1.id, 'yes' as data from t1 where id > 1) as tt + on t1.id = tt.id; + +drop table t1; + + --echo # end of 5.5 diff --git a/mysql-test/t/events_2.test b/mysql-test/t/events_2.test index 5443f76d1a1..bc906e37457 100644 --- a/mysql-test/t/events_2.test +++ b/mysql-test/t/events_2.test @@ -13,7 +13,7 @@ use events_test; # mysql.event intact checking end # -create event e_26 on schedule at '2027-01-01 00:00:00' disable do set @a = 5; +create event e_26 on schedule at '2037-01-01 00:00:00' disable do set @a = 5; select db, name, body, definer, convert_tz(execute_at, 'UTC', 'SYSTEM'), on_completion from mysql.event; drop event e_26; --error ER_WRONG_VALUE diff --git a/mysql-test/t/events_slowlog.test b/mysql-test/t/events_slowlog.test new file mode 100644 index 00000000000..9679714dba3 --- /dev/null +++ b/mysql-test/t/events_slowlog.test @@ -0,0 +1,28 @@ +--source include/not_embedded.inc +# +# MDEV-11552 Queries executed by event scheduler are written to slow log incorrectly or not written at all +# +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; + +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; + +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); + +--let wait_condition= select i from t1 where i > 0 +--source include/wait_condition.inc + +--let SEARCH_FILE = `SELECT @@slow_query_log_file` +--let SEARCH_PATTERN= update t1 set i=1 +--let SEARCH_RANGE= -1000 +--source include/search_pattern_in_file.inc + +drop table t1; + +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/t/func_regexp_pcre.test b/mysql-test/t/func_regexp_pcre.test index c9b4c10007d..8e3adad0037 100644 --- a/mysql-test/t/func_regexp_pcre.test +++ b/mysql-test/t/func_regexp_pcre.test @@ -426,3 +426,7 @@ SELECT 0xE001 REGEXP @regCheck; SET NAMES latin1; SET @regCheck= '\\xE0\\x01'; SELECT CAST(0xE001 AS BINARY) REGEXP @regCheck; + +--echo # MDEV-12420: Testing recursion overflow +--replace_regex /[0-9]+ exceeded/NUM exceeded/ +SELECT 1 FROM dual WHERE ('Alpha,Bravo,Charlie,Delta,Echo,Foxtrot,StrataCentral,Golf,Hotel,India,Juliet,Kilo,Lima,Mike,StrataL3,November,Oscar,StrataL2,Sand,P3,P4SwitchTest,Arsys,Poppa,ExtensionMgr,Arp,Quebec,Romeo,StrataApiV2,PtReyes,Sierra,SandAcl,Arrow,Artools,BridgeTest,Tango,SandT,PAlaska,Namespace,Agent,Qos,PatchPanel,ProjectReport,Ark,Gimp,Agent,SliceAgent,Arnet,Bgp,Ale,Tommy,Central,AsicPktTestLib,Hsc,SandL3,Abuild,Pca9555,Standby,ControllerDut,CalSys,SandLib,Sb820,PointV2,BfnLib,Evpn,BfnSdk,Sflow,ManagementActive,AutoTest,GatedTest,Bgp,Sand,xinetd,BfnAgentLib,bf-utils,Hello,BfnState,Eos,Artest,Qos,Scd,ThermoMgr,Uniform,EosUtils,Eb,FanController,Central,BfnL3,BfnL2,tcp_wrappers,Victor,Environment,Route,Failover,Whiskey,Xray,Gimp,BfnFixed,Strata,SoCal,XApi,Msrp,XpProfile,tcpdump,PatchPanel,ArosTest,FhTest,Arbus,XpAcl,MacConc,XpApi,telnet,QosTest,Alpha2,BfnVlan,Stp,VxlanControllerTest,MplsAgent,Bravo2,Lanz,BfnMbb,Intf,XCtrl,Unicast,SandTunnel,L3Unicast,Ipsec,MplsTest,Rsvp,EthIntf,StageMgr,Sol,MplsUtils,Nat,Ira,P4NamespaceDut,Counters,Charlie2,Aqlc,Mlag,Power,OpenFlow,Lag,RestApi,BfdTest,strongs,Sfa,CEosUtils,Adt746,MaintenanceMode,MlagDut,EosImage,IpEth,MultiProtocol,Launcher,Max3179,Snmp,Acl,IpEthTest,PhyEee,bf-syslibs,tacc,XpL2,p4-ar-switch,p4-bf-switch,LdpTest,BfnPhy,Mirroring,Phy6,Ptp' REGEXP '^((?!\b(Strata|StrataApi|StrataApiV2)\b).)*$'); diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 7f0f8132eac..e807132ecc7 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1668,6 +1668,11 @@ INSERT INTO t1 VALUES (18, '2010-10-13'); SELECT a.id,a.date1,FROM_DAYS(TO_DAYS(a.date1)-10) as date2, DATE_ADD(a.date1,INTERVAL -10 DAY),TO_DAYS(a.date1)-10 FROM t1 a ORDER BY a.id; DROP TABLE t1; +--echo # +--echo # MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +--echo # +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; + --echo # --echo # Start of 10.0 tests diff --git a/mysql-test/t/gis.test b/mysql-test/t/gis.test index 9378ea55377..f689902533a 100644 --- a/mysql-test/t/gis.test +++ b/mysql-test/t/gis.test @@ -1492,3 +1492,16 @@ SELECT ASTEXT(p) FROM v1; DROP VIEW v1; # --echo End of 5.5 tests + +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-12495 Conditional jump depends on uninitialised value for: SELECT NULL UNION geom_expression +--echo # +SELECT AsText(g) FROM (SELECT NULL AS g UNION SELECT Point(1,1)) AS t1; + +--echo # +--echo # End 10.0 tests +--echo # diff --git a/mysql-test/t/grant.test b/mysql-test/t/grant.test index 548fd0f810e..0be4c254269 100644 --- a/mysql-test/t/grant.test +++ b/mysql-test/t/grant.test @@ -2176,6 +2176,68 @@ DROP USER mysqltest_u1@localhost; --echo # End of Bug#38347. --echo + +--echo # +--echo # BUG#11759114 - '51401: GRANT TREATS NONEXISTENT FUNCTIONS/PRIVILEGES +--echo # DIFFERENTLY'. +--echo # +--disable_warnings +drop database if exists mysqltest_db1; +--enable_warnings +create database mysqltest_db1; +create user mysqltest_u1; +--echo # Both GRANT statements below should fail with the same error. +--error ER_SP_DOES_NOT_EXIST +grant execute on function mysqltest_db1.f1 to mysqltest_u1; +--error ER_SP_DOES_NOT_EXIST +grant execute on procedure mysqltest_db1.p1 to mysqltest_u1; +--echo # Let us show that GRANT behaviour for routines is consistent +--echo # with GRANT behaviour for tables. Attempt to grant privilege +--echo # on non-existent table also results in an error. +--error ER_NO_SUCH_TABLE +grant select on mysqltest_db1.t1 to mysqltest_u1; +show grants for mysqltest_u1; +drop database mysqltest_db1; +drop user mysqltest_u1; + + +--echo # +--echo # Bug#12766319 - 61865: RENAME USER DOES NOT WORK CORRECTLY - +--echo # REQUIRES FLUSH PRIVILEGES +--echo # + +CREATE USER foo@'127.0.0.1'; +GRANT ALL ON *.* TO foo@'127.0.0.1'; + +--echo # First attempt, should connect successfully +connect (conn1, '127.0.0.1', foo,,test); +SELECT user(), current_user(); + +--echo # Rename the user +RENAME USER foo@'127.0.0.1' to foo@'127.0.0.0/255.0.0.0'; + +--echo # Second attempt, should connect successfully as its valid mask +--echo # This was failing without fix +connect (conn2, '127.0.0.1', foo,,test); +SELECT user(), current_user(); + +--echo # Rename the user back to original +RENAME USER foo@'127.0.0.0/255.0.0.0' to foo@'127.0.0.1'; + +--echo # Third attempt, should connect successfully +connect (conn3, '127.0.0.1', foo,,test); +SELECT user(), current_user(); + +--echo # Clean-up +connection default; +disconnect conn1; +disconnect conn2; +disconnect conn3; +DROP USER foo@'127.0.0.1'; + +--echo # End of Bug#12766319 + + --echo # --echo # Bug#11756966 - 48958: STORED PROCEDURES CAN BE LEVERAGED TO BYPASS --echo # DATABASE SECURITY @@ -2210,26 +2272,3 @@ DROP DATABASE secret; # Wait till we reached the initial number of concurrent sessions --source include/wait_until_count_sessions.inc - ---echo # ---echo # BUG#11759114 - '51401: GRANT TREATS NONEXISTENT FUNCTIONS/PRIVILEGES ---echo # DIFFERENTLY'. ---echo # ---disable_warnings -drop database if exists mysqltest_db1; ---enable_warnings -create database mysqltest_db1; -create user mysqltest_u1; ---echo # Both GRANT statements below should fail with the same error. ---error ER_SP_DOES_NOT_EXIST -grant execute on function mysqltest_db1.f1 to mysqltest_u1; ---error ER_SP_DOES_NOT_EXIST -grant execute on procedure mysqltest_db1.p1 to mysqltest_u1; ---echo # Let us show that GRANT behaviour for routines is consistent ---echo # with GRANT behaviour for tables. Attempt to grant privilege ---echo # on non-existent table also results in an error. ---error ER_NO_SUCH_TABLE -grant select on mysqltest_db1.t1 to mysqltest_u1; -show grants for mysqltest_u1; -drop database mysqltest_db1; -drop user mysqltest_u1; diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test index 6a1cb53dc40..fb56e44b5ae 100644 --- a/mysql-test/t/index_merge_innodb.test +++ b/mysql-test/t/index_merge_innodb.test @@ -171,6 +171,37 @@ WHERE ( tb.b != ta.b OR tb.a = ta.a ) AND ( tb.b = ta.c OR tb.b = ta.b ); DROP TABLE t1; - set optimizer_switch= @optimizer_switch_save; +--echo # +--echo # MDEV-10927: Crash When Using sort_union Optimization +--echo # + +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; + +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; + +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; + +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/t/information_schema_part.test b/mysql-test/t/information_schema_part.test index f1415d12f79..ea88f364c07 100644 --- a/mysql-test/t/information_schema_part.test +++ b/mysql-test/t/information_schema_part.test @@ -131,3 +131,10 @@ drop table if exists t1; create table t1 (f1 int key) partition by key(f1) partitions 2; select create_options from information_schema.tables where table_schema="test"; drop table t1; + +--echo # +--echo # MDEV-11353 - Identical logical conditions +--echo # +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; diff --git a/mysql-test/t/join_cache.test b/mysql-test/t/join_cache.test index 7d873c555f6..aecc934aa2b 100644 --- a/mysql-test/t/join_cache.test +++ b/mysql-test/t/join_cache.test @@ -3708,9 +3708,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; set join_cache_level=2; @@ -3743,9 +3745,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; EXPLAIN SELECT t.* @@ -3776,6 +3780,8 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; diff --git a/mysql-test/t/join_nested.test b/mysql-test/t/join_nested.test index 7b7d9236835..e60b7827f75 100644 --- a/mysql-test/t/join_nested.test +++ b/mysql-test/t/join_nested.test @@ -1309,5 +1309,74 @@ LEFT JOIN t4 AS alias5 JOIN t5 ON alias5.f5 ON alias2.f3 ON alias1.f2; DROP TABLE t1,t2,t3,t4,t5; -set optimizer_search_depth= @tmp_mdev621; +--echo # +--echo # MDEV-7992: Nested left joins + 'not exists' optimization +--echo # + +CREATE TABLE t1( + K1 INT PRIMARY KEY, + Name VARCHAR(15) +); + +INSERT INTO t1 VALUES + (1,'T1Row1'), (2,'T1Row2'); + + +CREATE TABLE t2( + K2 INT PRIMARY KEY, + K1r INT, + rowTimestamp DATETIME, + Event VARCHAR(15) +); + +INSERT INTO t2 VALUES + (1, 1, '2015-04-13 10:42:11' ,'T1Row1Event1'), + (2, 1, '2015-04-13 10:42:12' ,'T1Row1Event2'), + (3, 1, '2015-04-13 10:42:12' ,'T1Row1Event3'); + +let $q1= +SELECT t1a.*, t2a.*, + t2i.K2 AS K2B, t2i.K1r AS K1rB, + t2i.rowTimestamp AS rowTimestampB, t2i.Event AS EventB +FROM + t1 t1a JOIN t2 t2a ON t2a.K1r = t1a.K1 + LEFT JOIN + ( t1 t1i LEFT JOIN t2 t2i ON t2i.K1r = t1i.K1) + ON (t1i.K1 = 1) AND + (((t2i.K1r = t1a.K1 AND t2i.rowTimestamp > t2a.rowTimestamp ) OR + (t2i.rowTimestamp = t2a.rowTimestamp AND t2i.K2 > t2a.K2)) + OR (t2i.K2 IS NULL)) +WHERE +t2a.K1r = 1 AND t2i.K2 IS NULL; + +eval $q1; +eval EXPLAIN EXTENDED $q1; + +CREATE VIEW v1 AS + SELECT t2i.* + FROM t1 as t1i LEFT JOIN t2 as t2i ON t2i.K1r = t1i.K1 + WHERE t1i.K1 = 1 ; + +let $q2= +SELECT + t1a.*, t2a.*, t2b.K2 as K2B, t2b.K1r as K1rB, + t2b.rowTimestamp as rowTimestampB, t2b.Event as EventB +FROM + t1 as t1a JOIN t2 as t2a ON t2a.K1r = t1a.K1 + LEFT JOIN + v1 as t2b + ON ((t2b.K1r = t1a.K1 AND t2b.rowTimestamp > t2a.rowTimestamp) OR + (t2b.rowTimestamp = t2a.rowTimestamp AND t2b.K2 > t2a.K2)) + OR (t2b.K2 IS NULL) +WHERE + t1a.K1 = 1 AND + t2b.K2 IS NULL; + +eval $q2; +eval EXPLAIN EXTENDED $q2; + +DROP VIEW v1; +DROP TABLE t1,t2; + +set optimizer_search_depth= @tmp_mdev621; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 9f2aafc8efd..8f8ff3520a2 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -294,7 +294,7 @@ SELECT * FROM v2; DELETE FROM t1; --echo ---error ER_LOAD_DATA_INVALID_COLUMN +--error ER_NONUPDATEABLE_COLUMN LOAD DATA INFILE '../../std_data/bug35469.dat' INTO TABLE v2 FIELDS ESCAPED BY '\\' TERMINATED BY ',' @@ -612,7 +612,7 @@ disconnect con1; --echo # CREATE TABLE t1(f1 INT); -EVAL SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +EVAL SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; --disable_warnings LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; --enable_warnings @@ -658,25 +658,21 @@ SET @@sql_mode= @old_mode; --remove_file $MYSQLTEST_VARDIR/mysql DROP TABLE t1; ---echo + +--echo # +--echo # MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files +--echo # + +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +DROP TABLE t1; + --echo # ---echo # Bug#23080148 - Backport of Bug#20683959. ---echo # Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY ---echo # UNDER DB CHARSET IS UTF8. +--echo # MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character --echo # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -SELECT HEX(val) FROM t1; - -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; - -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +DROP TABLE t1; diff --git a/mysql-test/t/log_slow.test b/mysql-test/t/log_slow.test index 8d5a09d7a94..56e35bd5a20 100644 --- a/mysql-test/t/log_slow.test +++ b/mysql-test/t/log_slow.test @@ -50,7 +50,6 @@ set global slow_query_log=1; set global log_output='TABLE'; select sleep(0.5); select count(*) FROM mysql.slow_log; -truncate mysql.slow_log; # Reset used variables set @@long_query_time=default; @@ -58,3 +57,4 @@ set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/t/log_tables-big.test b/mysql-test/t/log_tables-big.test index 8c956fa6f55..8936a163d73 100644 --- a/mysql-test/t/log_tables-big.test +++ b/mysql-test/t/log_tables-big.test @@ -7,6 +7,8 @@ # check that CSV engine was compiled in --source include/have_csv.inc +set @@global.log_output = 'TABLE'; + connect (con1,localhost,root,,); connect (con2,localhost,root,,); @@ -18,18 +20,20 @@ set session long_query_time=10; select get_lock('bug27638', 1); connection con2; set session long_query_time=1; -truncate table mysql.slow_log; select get_lock('bug27638', 2); -select if (query_time between '00:00:01' and '00:00:10', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log; -truncate table mysql.slow_log; +select if (query_time >= '00:00:01', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log + where sql_text = 'select get_lock(\'bug27638\', 2)'; select get_lock('bug27638', 60); -select if (query_time between '00:00:59' and '00:01:10', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log; -truncate table mysql.slow_log; +select if (query_time >= '00:00:59', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log + where sql_text = 'select get_lock(\'bug27638\', 60)'; select get_lock('bug27638', 101); -select if (query_time between '00:01:40' and '00:01:50', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log; +select if (query_time >= '00:01:40', 'OK', 'WRONG') as qt, sql_text from mysql.slow_log + where sql_text = 'select get_lock(\'bug27638\', 101)'; connection con1; select release_lock('bug27638'); connection default; disconnect con1; disconnect con2; + +set @@global.log_output=default; diff --git a/mysql-test/t/mdev-504.test b/mysql-test/t/mdev-504.test index fb5c7666d33..b96c8779c68 100644 --- a/mysql-test/t/mdev-504.test +++ b/mysql-test/t/mdev-504.test @@ -1,3 +1,4 @@ +--source include/not_valgrind.inc --disable_ps_protocol SET GLOBAL net_write_timeout = 900; diff --git a/mysql-test/t/myisam_debug.test b/mysql-test/t/myisam_debug.test index 5b5d006bd22..57d423d88c9 100644 --- a/mysql-test/t/myisam_debug.test +++ b/mysql-test/t/myisam_debug.test @@ -59,3 +59,16 @@ KILL QUERY @thread_id; CHECK TABLE t1; DROP TABLE t1,t2; DISCONNECT insertConn; + +# +# MDEV-12761 Error return from external_lock make the server crash +# +call mtr.add_suppression('Incorrect key file for table'); +create table t1 (a int, index(a)); +lock tables t1 write; +insert t1 values (1),(2),(1); +set @old_dbug=@@debug_dbug; +set debug_dbug='+d,mi_lock_database_failure'; +unlock tables; +set debug_dbug=@old_dbug; +drop table t1; diff --git a/mysql-test/t/mysqldump.test b/mysql-test/t/mysqldump.test index fd50896b71f..bb5a9ae1846 100644 --- a/mysql-test/t/mysqldump.test +++ b/mysql-test/t/mysqldump.test @@ -2483,6 +2483,56 @@ DROP TABLE t1; DROP TABLE t2; DROP DATABASE db_20772273; +--echo # +--echo # Bug #25717383: MYSQLDUMP MAY EXECUTE ANY ARBITRARY QUERY +--echo # + + +CREATE DATABASE bug25717383; +use bug25717383; + +CREATE TABLE `tab +one` (a int); +CREATE VIEW `view +one` as SELECT * FROM `tab +one`; + +CREATE PROCEDURE `proc +one`() SELECT * from `tab +one`; + +CREATE TEMPORARY TABLE `temp +one` (id INT); + +CREATE TRIGGER `trig +one` BEFORE INSERT ON `tab +one` FOR EACH ROW SET NEW.a = 1; + +CREATE EVENT `event +one` ON SCHEDULE AT '2030-01-01 00:00:00' DO SET @a=5; + +SHOW TABLES FROM bug25717383; +SHOW TRIGGERS FROM bug25717383; +--replace_column 6 # +SHOW EVENTS FROM bug25717383; + +SELECT ROUTINE_NAME FROM INFORMATION_SCHEMA.ROUTINES + WHERE ROUTINE_SCHEMA='bug25717383' AND ROUTINE_TYPE= 'PROCEDURE' + ORDER BY ROUTINE_NAME; + +--exec $MYSQL_DUMP --triggers --events --routines --add-drop-database --databases bug25717383 > $MYSQLTEST_VARDIR/tmp/bug25717383.sql + +SHOW TABLES FROM bug25717383; +SHOW TRIGGERS FROM bug25717383; +--replace_column 6 # +SHOW EVENTS FROM bug25717383; + +SELECT ROUTINE_NAME FROM INFORMATION_SCHEMA.ROUTINES + WHERE ROUTINE_SCHEMA='bug25717383' AND ROUTINE_TYPE= 'PROCEDURE' + ORDER BY ROUTINE_NAME; + +DROP DATABASE bug25717383; + # # MDEV-6091 mysqldump goes in a loop and segfaults if --dump-slave is specified and it cannot connect to the server # @@ -2512,3 +2562,11 @@ if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Wind } --exec $MYSQL_DUMP --routines --compact $shell_ready_db_name DROP DATABASE `a\"'``b`; + +#" +# MDEV-11505 wrong databasename in mysqldump comment +# +let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/bug11505.sql; +let SEARCH_PATTERN=Database: mysql; +exec $MYSQL_DUMP mysql func > $SEARCH_FILE; +source include/search_pattern_in_file.inc; diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test index cf6a4d473c3..a86405889b1 100644 --- a/mysql-test/t/order_by.test +++ b/mysql-test/t/order_by.test @@ -1945,6 +1945,16 @@ order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +--echo # +--echo # mdev-10705 : long order by list that can be skipped +--echo # + +SELECT 1 +UNION +( SELECT 2 + ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, + @a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); + --echo End of 5.5 tests --echo # diff --git a/mysql-test/t/partition_innodb.test b/mysql-test/t/partition_innodb.test index 6845cca10fb..f6faa4cb0e6 100644 --- a/mysql-test/t/partition_innodb.test +++ b/mysql-test/t/partition_innodb.test @@ -820,6 +820,104 @@ INSERT INTO t1 (d) VALUES ('1991-01-01'); SELECT * FROM t1 WHERE d = '1991-01-01'; DROP TABLE t1; +set global default_storage_engine=default; + +--echo # +--echo # MDEV-9455: [ERROR] mysqld got signal 11 +--echo # + +CREATE TABLE `t1` ( + `DIARY_TOTAL_DAY_SEQ` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `IMORY_ID` bigint(20) NOT NULL, + `NAME` varchar(75) DEFAULT NULL, + `DATETIME` varchar(10) NOT NULL DEFAULT '', + `DAILY_CALL_CNT` int(11) DEFAULT NULL, + `DAILY_SMS_CNT` int(11) DEFAULT NULL, + `NUMBER` varchar(64) DEFAULT NULL, + `DURATION` varchar(16) DEFAULT NULL, + PRIMARY KEY (`DIARY_TOTAL_DAY_SEQ`,`DATETIME`), + KEY `IDX_t1_01` (`IMORY_ID`,`DATETIME`) +) AUTO_INCREMENT=328702514 DEFAULT CHARSET=utf8mb4 +PARTITION BY RANGE COLUMNS(`DATETIME`) +(PARTITION p0 VALUES LESS THAN ('2015-10-01') ENGINE = InnoDB, + PARTITION p1 VALUES LESS THAN ('2015-11-01') ENGINE = InnoDB, + PARTITION p2 VALUES LESS THAN ('2015-12-01') ENGINE = InnoDB, + PARTITION p3 VALUES LESS THAN ('2016-01-01') ENGINE = InnoDB, + PARTITION p4 VALUES LESS THAN ('2016-02-01') ENGINE = InnoDB, + PARTITION p5 VALUES LESS THAN ('2016-03-01') ENGINE = InnoDB, + PARTITION p6 VALUES LESS THAN ('2016-04-01') ENGINE = InnoDB, + PARTITION p7 VALUES LESS THAN ('2016-05-01') ENGINE = InnoDB, + PARTITION p8 VALUES LESS THAN ('2016-06-01') ENGINE = InnoDB, + PARTITION p9 VALUES LESS THAN ('2016-07-01') ENGINE = InnoDB, + PARTITION p10 VALUES LESS THAN ('2016-08-01') ENGINE = InnoDB) +; + +CREATE TABLE `t2` ( + `DIARY_SEQ` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `IMORY_ID` bigint(20) NOT NULL, + `CALL_TYPE` varchar(1) DEFAULT NULL, + `DATA_TYPE` varchar(1) DEFAULT NULL, + `FEATURES` varchar(1) DEFAULT NULL, + `NAME` varchar(75) DEFAULT NULL, + `NUMBER` varchar(64) DEFAULT NULL, + `DATETIME` datetime NOT NULL, + `REG_DATE` datetime NOT NULL, + `TITLE` varchar(50) DEFAULT NULL, + `BODY` varchar(4200) DEFAULT NULL, + `MIME_TYPE` varchar(32) DEFAULT NULL, + `DURATION` varchar(16) DEFAULT NULL, + `DEVICE_ID` varchar(64) DEFAULT NULL, + `DEVICE_NAME` varchar(32) DEFAULT NULL, + PRIMARY KEY (`DIARY_SEQ`,`DATETIME`,`REG_DATE`), + KEY `IDX_TB_DIARY_01` (`IMORY_ID`,`DATETIME`,`CALL_TYPE`,`NUMBER`), + KEY `IDX_TB_DIARY_02` (`REG_DATE`) +) AUTO_INCREMENT=688799006 DEFAULT CHARSET=utf8mb4 +PARTITION BY RANGE COLUMNS(REG_DATE) +(PARTITION p0 VALUES LESS THAN ('2015-10-01') ENGINE = InnoDB, + PARTITION p1 VALUES LESS THAN ('2015-11-01') ENGINE = InnoDB, + PARTITION p2 VALUES LESS THAN ('2015-12-01') ENGINE = InnoDB, + PARTITION p3 VALUES LESS THAN ('2016-01-01') ENGINE = InnoDB, + PARTITION p4 VALUES LESS THAN ('2016-02-01') ENGINE = InnoDB, + PARTITION p5 VALUES LESS THAN ('2016-03-01') ENGINE = InnoDB, + PARTITION p6 VALUES LESS THAN ('2016-04-01') ENGINE = InnoDB, + PARTITION p7 VALUES LESS THAN ('2016-05-01') ENGINE = InnoDB, + PARTITION p8 VALUES LESS THAN ('2016-06-01') ENGINE = InnoDB, + PARTITION p9 VALUES LESS THAN ('2016-07-01') ENGINE = InnoDB, + PARTITION p10 VALUES LESS THAN ('2016-08-01') ENGINE = InnoDB) +; + +SELECT + A.IMORY_ID, + A.NUMBER, + A.NAME, + DATE_FORMAT(A.DATETIME, '%Y-%m-%d') AS TARGET_DATE, + SUM( CASE WHEN A.DATA_TYPE='1' THEN 1 ELSE 0 END) AS CALL_CNT, + SUM( CASE WHEN A.DATA_TYPE IN ('2', '3') THEN 1 ELSE 0 END) AS SMS_CNT, + SUM(CAST(A.DURATION AS INT)) AS DURATION, + ( SELECT COUNT(*) + FROM t1 + WHERE IMORY_ID=A.IMORY_ID + AND NUMBER=A.NUMBER + AND NAME=A.NAME + AND DATETIME = DATE_FORMAT(A.DATETIME, '%Y-%m-%d') + ) STATS_COUNT +FROM t2 A +WHERE A.IMORY_ID = 55094102 + AND A.DATETIME LIKE ( + SELECT CONCAT (DATE_FORMAT(DATETIME, '%Y-%m-%d') ,'%') + FROM t2 + WHERE IMORY_ID=55094102 + AND DIARY_SEQ IN ( 608351221, 608351225, 608351229 ) + group by DATE_FORMAT(DATETIME, '%Y-%m-%d') + ) +GROUP BY A.IMORY_ID, A.NUMBER, A.NAME, DATE_FORMAT(A.DATETIME, '%Y-%m-%d') +; + +drop table t2, t1; + + +set global default_storage_engine='innodb'; + --echo # --echo # MDEV-5963: InnoDB: Assertion failure in file row0sel.cc line 2503, --echo # Failing assertion: 0 with "key ptr now exceeds key end by 762 bytes" diff --git a/mysql-test/t/partition_myisam.test b/mysql-test/t/partition_myisam.test index d07637057e0..4d083c37b68 100644 --- a/mysql-test/t/partition_myisam.test +++ b/mysql-test/t/partition_myisam.test @@ -216,6 +216,28 @@ PARTITION BY RANGE (a) PARTITION pMax VALUES LESS THAN MAXVALUE); INSERT INTO t1 VALUES (1, "Partition p1, first row"); DROP TABLE t1; + +--echo # +--echo # MDEV-10418 Assertion `m_extra_cache' failed +--echo # in ha_partition::late_extra_cache(uint) +--echo # + +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (f2 INT) ENGINE=MyISAM PARTITION BY RANGE(f2) (PARTITION pmax VALUES LESS THAN MAXVALUE); +INSERT INTO t2 VALUES (8); + +CREATE ALGORITHM = MERGE VIEW v AS SELECT f2 FROM t2, t1; + +UPDATE v SET f2 = 1; + +SELECT * FROM t2; + +DROP VIEW v; +DROP TABLE t2; +DROP TABLE t1; + --echo # --echo # bug#11760213-52599: ALTER TABLE REMOVE PARTITIONING ON NON-PARTITIONED --echo # TABLE CORRUPTS MYISAM diff --git a/mysql-test/t/pool_of_threads.cnf b/mysql-test/t/pool_of_threads.cnf index c03e1da6450..f6651c878de 100644 --- a/mysql-test/t/pool_of_threads.cnf +++ b/mysql-test/t/pool_of_threads.cnf @@ -7,8 +7,5 @@ loose-thread_pool_max_threads= 2 extra-port= @ENV.MASTER_EXTRA_PORT extra-max-connections=1 -[client] -connect-timeout= 2 - [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/mysql-test/t/pool_of_threads.test b/mysql-test/t/pool_of_threads.test index 24e0218db62..f13a096985c 100644 --- a/mysql-test/t/pool_of_threads.test +++ b/mysql-test/t/pool_of_threads.test @@ -15,23 +15,26 @@ SET optimizer_switch=@save_optimizer_switch; # connections on the extra port. # First set two connections running, and check that extra connection -# on normal port fails due to--thread-pool-max_threads=2 +# on normal port fails due to --thread-pool-max-threads=2. +# We can afford using a really long sleep, because we won't wait +# till it ends, we'll interrupt it as soon as we don't need it anymore + connection default; +--let $con1_id= `SELECT CONNECTION_ID()` -# Sleep for slightly longer than 5 sec to trigger MDEV-4566 -# (abort in interruptible wait connection check) -send SELECT sleep(5.5); +send SELECT sleep(50); --sleep 1 connect(con2,localhost,root,,); -connection con2; -send SELECT sleep(5); +--let $con2_id= `SELECT CONNECTION_ID()` + +send SELECT sleep(50); --sleep 0.5 --disable_abort_on_error --disable_result_log --disable_query_log -connect(con3,localhost,root,,); +connect(con3,localhost,root,,,,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -45,24 +48,15 @@ if ($error) --echo # -- Success: more than --thread_pool_max_threads normal connections not possible } -connection default; ---reap -connection con2; ---reap - -# Now try again, but this time use the extra port to successfully connect. - -connection default; -send SELECT sleep(5); - -connection con2; -send SELECT sleep(5); ---sleep 1 - connect(extracon,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon; SELECT 'Connection on extra port ok'; +# Here, sleep just for slightly longer than 5 sec to trigger MDEV-4566 +# (abort in interruptible wait connection check). +send SELECT sleep(5.5); + + connect(extracon2,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon2; SELECT 'Connection on extra port 2 ok'; @@ -70,7 +64,7 @@ SELECT 'Connection on extra port 2 ok'; --disable_abort_on_error --disable_result_log --disable_query_log -connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); +connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -84,7 +78,16 @@ if ($error) --echo # -- Success: more than --extra-max-connections + 1 normal connections not possible } +connection extracon2; +--replace_result $con1_id <default_connection_ID> +eval KILL QUERY $con1_id; +--replace_result $con2_id <con2_connection_ID> +eval KILL QUERY $con2_id; + connection default; --reap connection con2; --reap + +connection extracon; +--reap diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index a6b238b84c6..d7ca29083b5 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -3680,5 +3680,38 @@ EXECUTE stmt; deallocate prepare stmt; drop table t1,t2,t3,t4; +--echo # +--echo # MDEV-11859: the plans for the first and the second executions +--echo # of PS are not the same +--echo # + +create table t1 (id int, c varchar(3), key idx(c))engine=myisam; +insert into t1 values (3,'bar'), (1,'xxx'), (2,'foo'), (5,'yyy'); + +prepare stmt1 from +"explain extended + select * from t1 where (1, 2) in ( select 3, 4 ) or c = 'foo'"; +execute stmt1; +execute stmt1; +deallocate prepare stmt1; + +prepare stmt1 from +"select * from t1 where (1, 2) in ( select 3, 4 ) or c = 'foo'"; +flush status; +execute stmt1; +show status like '%Handler_read%'; +flush status; +execute stmt1; +show status like '%Handler_read%'; +deallocate prepare stmt1; + +prepare stmt2 from +"explain extended + select * from t1 where (1, 2) in ( select 3, 4 )"; +execute stmt2; +execute stmt2; +deallocate prepare stmt2; + +drop table t1; --echo # End of 5.5 tests diff --git a/mysql-test/t/range_vs_index_merge.test b/mysql-test/t/range_vs_index_merge.test index 7ecca454f4c..5d12d46c9e9 100644 --- a/mysql-test/t/range_vs_index_merge.test +++ b/mysql-test/t/range_vs_index_merge.test @@ -57,9 +57,9 @@ SELECT * FROM City EXPLAIN SELECT * FROM City WHERE Population > 100000 AND Name LIKE 'Aba%' OR - Country IN ('CAN', 'ARG') AND ID < 3800 OR - Country < 'U' AND Name LIKE 'Zhu%' OR - ID BETWEEN 3800 AND 3810; + Country IN ('CAN', 'ARG') AND ID BETWEEN 120 AND 130 OR + Country <= 'ALB' AND Name LIKE 'L%' OR + ID BETWEEN 3807 AND 3810; # The output of the next 3 commands tells us about selectivities # of the conditions utilized in 2 queries following after them @@ -1206,6 +1206,41 @@ SELECT * FROM t1 DROP TABLE t1; + +--echo # +--echo # MDEV-8603: Wrong result OR/AND condition over index fields +--echo # + +CREATE TABLE t1 ( + id INT NOT NULL, + state VARCHAR(64), + capital VARCHAR(64), + UNIQUE KEY (id), + KEY state (state,id), + KEY capital (capital, id) +); + +INSERT INTO t1 VALUES + (1,'Arizona','Phoenix'), + (2,'Hawaii','Honolulu'), + (3,'Georgia','Atlanta'), + (4,'Florida','Tallahassee'), + (5,'Alaska','Juneau'), + (6,'Michigan','Lansing'), + (7,'Pennsylvania','Harrisburg'), + (8,'Virginia','Richmond') +; + +EXPLAIN +SELECT * FROM t1 FORCE KEY (state,capital) +WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 + OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas'; +SELECT * FROM t1 FORCE KEY (state,capital) +WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 + OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas'; + +DROP TABLE t1; + #the following command must be the last one in the file set session optimizer_switch='index_merge_sort_intersection=default'; diff --git a/mysql-test/t/repair_symlink-5543.test b/mysql-test/t/repair_symlink-5543.test index bad65a4175a..6bdf72b4d40 100644 --- a/mysql-test/t/repair_symlink-5543.test +++ b/mysql-test/t/repair_symlink-5543.test @@ -9,7 +9,7 @@ eval create table t1 (a int) engine=myisam data directory='$MYSQL_TMP_DIR'; insert t1 values (1); --system ln -s $MYSQL_TMP_DIR/foobar5543 $MYSQL_TMP_DIR/t1.TMD ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--replace_regex / '.*\/t1/ 'MYSQL_TMP_DIR\/t1/ repair table t1; drop table t1; @@ -17,7 +17,7 @@ drop table t1; eval create table t2 (a int) engine=aria data directory='$MYSQL_TMP_DIR'; insert t2 values (1); --system ln -s $MYSQL_TMP_DIR/foobar5543 $MYSQL_TMP_DIR/t2.TMD ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--replace_regex / '.*\/t2/ 'MYSQL_TMP_DIR\/t2/ repair table t2; drop table t2; diff --git a/mysql-test/t/sp-prelocking.test b/mysql-test/t/sp-prelocking.test index c1378d59196..38cbd5aa110 100644 --- a/mysql-test/t/sp-prelocking.test +++ b/mysql-test/t/sp-prelocking.test @@ -414,3 +414,33 @@ SELECT f1(); DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1,t2; + +--echo # +--echo # Bug #16672723 "CAN'T FIND TEMPORARY TABLE". +--echo # +CREATE FUNCTION f1() RETURNS INT RETURN 1; +CREATE TEMPORARY TABLE tmp1(a INT); +PREPARE stmt1 FROM "CREATE TEMPORARY TABLE tmp2 AS SELECT b FROM (SELECT f1() AS b FROM tmp1) AS t"; +--echo # The below statement failed before the fix. +EXECUTE stmt1; +DROP TEMPORARY TABLES tmp1, tmp2; +DEALLOCATE PREPARE stmt1; +DROP FUNCTION f1; + +# +# MDEV-9084 Calling a stored function from a nested select from temporary table causes unpredictable behavior +# +delimiter $$; +create procedure sp1() +begin + drop table if exists t1, t2; + create temporary table t1 select 1 v; + create table t2 (col varchar(45)) select distinct col from (select sf1() as col from t1) t; +end$$ +delimiter ;$$ +create function sf1() returns text return 'blah'; +call test.sp1(); +call test.sp1(); +drop procedure sp1; +drop function sf1; +drop table t2; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index 10c95f9a919..c5d37d1017d 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9422,3 +9422,43 @@ drop procedure p1; drop table t1,t2,t3,t4,t5,t6; +--echo # +--echo # MDEV-11935: Queries in stored procedures with and +--echo # EXISTS(SELECT * FROM VIEW) crashes and closes hte conneciton. +--echo # + +CREATE TABLE ANY_TABLE ( + ENTITY_UID BIGINT NOT NULL +); +CREATE TABLE SECURITY_PATH( +origid BIGINT UNSIGNED NOT NULL, +destid BIGINT UNSIGNED NOT NULL, +KEY (destid) +); +CREATE VIEW ENTITY_ACCESS ( +ENTITY_UID, +OWNER_UID +) AS +SELECT SP1.origid, + SP2.destid +FROM SECURITY_PATH SP1 +JOIN SECURITY_PATH SP2 ON SP1.destid = SP2.origid +; +--delimiter // +CREATE PROCEDURE SP_EXAMPLE_SELECT () +BEGIN + SELECT * + FROM ANY_TABLE AT1 + WHERE EXISTS ( SELECT * + FROM ENTITY_ACCESS EA + WHERE AT1.ENTITY_UID = EA.ENTITY_UID + AND EA.OWNER_UID IS NULL ); +END +// +--delimiter ; +CALL SP_EXAMPLE_SELECT (); +CALL SP_EXAMPLE_SELECT (); + +drop procedure SP_EXAMPLE_SELECT; +drop view ENTITY_ACCESS; +drop table ANY_TABLE, SECURITY_PATH; diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index a9e09bb4e27..ab864803864 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -6011,3 +6011,13 @@ SELECT ( SELECT MIN(v2.f2) FROM t1 ) AS sq FROM v2 GROUP BY sq; drop view v2; drop table t1,t2; + +--echo # +--echo # MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +--echo # + +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +DROP TABLE t1; diff --git a/mysql-test/t/subselect2.test b/mysql-test/t/subselect2.test index b3c1322184d..ae210b865a2 100644 --- a/mysql-test/t/subselect2.test +++ b/mysql-test/t/subselect2.test @@ -359,5 +359,55 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); drop table t1,t2,t3; +--echo # +--echo # MDEV-10148: Database crashes in the query to the View +--echo # +CREATE TABLE t1 ( + key_code INT(11) NOT NULL, + value_string VARCHAR(50) NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t2 ( + key_code INT(11) NOT NULL, + target_date DATE NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t3 ( + now_date DATE NOT NULL, + PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE VIEW v1 +AS +SELECT + B.key_code, + B.target_date +FROM + t2 B INNER JOIN t3 C ON + B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; #1st time -> success +EXECUTE stmt; #2nd time -> crash +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; + set optimizer_switch=@subselect2_test_tmp; +# +# Bug #23303485 : HANDLE_FATAL_SIGNAL (SIG=11) IN SUBSELECT_UNION_ENGINE::NO_ROWS +# +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +--error ER_SUBQUERY_NO_1_ROW +select null in (select a from t1 where a < out3.a union select a from t2 where + (select a from t3) +1 < out3.a+1) from t3 out3; +drop table t1, t2, t3; diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 4eb9701ee71..e54308b5e46 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -1956,5 +1956,81 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; drop table t1; +--echo # +--echo # MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +--echo # +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); + +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); + +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; + +EXECUTE stmt; +EXECUTE stmt; + +--echo # Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); + +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; + +EXECUTE stmt; +EXECUTE stmt; + +drop table t1, t2, t3; + +--echo # +--echo # MDEV-11078: NULL NOT IN (non-empty subquery) should never return results +--echo # + +create table t1(a int,b int); +create table t2(a int,b int); +insert into t1 value (1,2); +select (NULL) in (select 1 from t1); +select (null) in (select 1 from t2); +select 1 in (select 1 from t1); +select 1 in (select 1 from t2); +select 1 from dual where null in (select 1 from t1); +select 1 from dual where null in (select 1 from t2); +select (null,null) in (select * from t1); +select (null,null) in (select * from t2); +select 1 from dual where null not in (select 1 from t1); +select 1 from dual where null not in (select 1 from t2); +drop table t1,t2; + + +--echo # +--echo # MDEV-6486: Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' +--echo # failed with SELECT SQ, TEXT field +--echo # + +CREATE TABLE t1 (a VARCHAR(8), KEY(a)) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('foo'),( 'bar'); + +CREATE TABLE t2 (b VARCHAR(8), c TINYTEXT, KEY(b)) ENGINE=MyISAM; +INSERT INTO t2 VALUES ('baz','baz'),('qux', 'qux'); + +SELECT ( SELECT COUNT(*) FROM t1 WHERE a = c ) AS field, COUNT(DISTINCT c) +FROM t2 WHERE b <= 'quux' GROUP BY field; +drop table t1,t2; + SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; diff --git a/mysql-test/t/subselect_exists2in.test b/mysql-test/t/subselect_exists2in.test index 9450ef71494..a4fdbe5c50b 100644 --- a/mysql-test/t/subselect_exists2in.test +++ b/mysql-test/t/subselect_exists2in.test @@ -758,6 +758,34 @@ deallocate prepare stmt; drop view v1; drop table t1,t2; +--echo # +--echo #MDEV-10053: EXIST to IN transformation turned down +--echo # + +CREATE TABLE t1 ( + pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) +ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7); + +let $q= +SELECT STRAIGHT_JOIN sq1.f2 + FROM ( SELECT * FROM t1 ) AS sq1 + WHERE EXISTS ( SELECT * FROM t1 AS sq2 + WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); + +set @optimizer_switch_save=@@optimizer_switch; + +set optimizer_switch='exists_to_in=off'; +eval explain extended $q; +eval $q; +set optimizer_switch='exists_to_in=on'; +eval explain extended $q; +eval $q; + +set optimizer_switch= @optimizer_switch_save; + +DROP TABLE t1; + --echo # End of 10.0 tests #restore defaults diff --git a/mysql-test/t/subselect_innodb.test b/mysql-test/t/subselect_innodb.test index af6ec90ba74..2451bc60fee 100644 --- a/mysql-test/t/subselect_innodb.test +++ b/mysql-test/t/subselect_innodb.test @@ -483,6 +483,47 @@ drop table t1; set optimizer_switch=@subselect_innodb_tmp; --echo # +--echo # MDEV-9635:Server crashes in part_of_refkey or assertion +--echo # `!created && key_to_save < (int)s->keys' failed in +--echo # TABLE::use_index(int) or with join_cache_level>2 +--echo # + +SET join_cache_level=3; + +CREATE TABLE t1 (f1 VARCHAR(1024)) ENGINE=InnoDB; +CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1; + +CREATE TABLE t2 (f2 VARCHAR(4)) ENGINE=InnoDB; +INSERT INTO t2 VALUES ('foo'),('bar'); + +SELECT * FROM v1, t2 WHERE ( f1, f2 ) IN ( SELECT f1, f1 FROM t1 ); + +set join_cache_level = default; +drop view v1; +drop table t1,t2; + +--echo # +--echo # MDEV-10693: cost-based choice between materialization and in-to-exists +--echo # for a subquery from the expression used in ref access +--echo # + +--source include/have_innodb.inc + +CREATE TABLE t1 (i1 INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (i2 INT) ENGINE=InnoDB; + +CREATE TABLE t3 (i3 INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t3 VALUES (3); + +SELECT * FROM t1 + WHERE NULL IN ( SELECT i2 FROM t2 + WHERE i1 IN ( i2 IN ( SELECT i3 FROM t3 ) ) AND i2 = 2 ); + +DROP TABLE t1,t2,t3; + +--echo # --echo # MDEV-6041: ORDER BY+subqueries: subquery_table.key=outer_table.col is not recongized as binding --echo # create table t1(a int) engine=innodb; @@ -535,4 +576,3 @@ from t1; drop table t1,t2; - diff --git a/mysql-test/t/subselect_mat_cost_bugs.test b/mysql-test/t/subselect_mat_cost_bugs.test index 8205e94b203..316ac707bef 100644 --- a/mysql-test/t/subselect_mat_cost_bugs.test +++ b/mysql-test/t/subselect_mat_cost_bugs.test @@ -424,3 +424,42 @@ explain select * from t1 where c1 = 'a2' and (c1, c2) not in (select * from t2); select * from t1 where c1 = 'a2' and (c1, c2) not in (select * from t2); drop table t1, t2; + +--echo # +--echo # MDEV-12673: cost-based choice between materialization and in-to-exists +--echo # + +CREATE TABLE t1 ( + pk1 int, a1 varchar(3), b1 varchar(3), PRIMARY KEY (pk1), KEY(a1), KEY(b1) +) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,'foo','bar'),(2,'bar','foo'); + +CREATE TABLE t2 (pk2 INT PRIMARY KEY, a2 VARCHAR(3), KEY(a2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1,'abc'),(2,'xyz'),(3,'foo'); + +SELECT 'qux' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ); +SELECT 'bar' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ); +EXPLAIN +SELECT 'bar' IN ( SELECT a1 FROM t1 INNER JOIN t2 WHERE a2 = b1 AND pk2 = 3 ); + +DROP TABLE t1,t2; + +CREATE TABLE t1 (i1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (i2 int, c2 varchar(3), KEY(i2,c2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1,'abc'),(2,'foo'); + +CREATE TABLE t3 (pk3 int PRIMARY KEY, c3 varchar(3)) ENGINE=MyISAM; +INSERT INTO t3 VALUES (1,'foo'),(2,'bar'); + +SELECT * FROM t1 WHERE i1 NOT IN ( + SELECT i2 FROM t2 RIGHT JOIN t3 ON (c3 = c2) WHERE pk3 = i1 +); + +EXPLAIN +SELECT * FROM t1 WHERE i1 NOT IN ( + SELECT i2 FROM t2 RIGHT JOIN t3 ON (c3 = c2) WHERE pk3 = i1 +); + +DROP TABLE t1,t2,t3; diff --git a/mysql-test/t/subselect_sj_mat.test b/mysql-test/t/subselect_sj_mat.test index d2bafa86028..7b6c5818a04 100644 --- a/mysql-test/t/subselect_sj_mat.test +++ b/mysql-test/t/subselect_sj_mat.test @@ -1857,6 +1857,99 @@ execute stmt; drop table t1; +--echo # +--echo # MDEV-12429: IN subquery used in WHERE of EXISTS subquery +--echo # + +CREATE TABLE t1 ( + pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7); + +SELECT sq1.f2 FROM t1 AS sq1 + WHERE EXISTS ( SELECT * FROM t1 AS sq2 + WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); + +set @save_optimizer_switch= @@optimizer_switch; + +set optimizer_switch='exists_to_in=off'; +EXPLAIN +SELECT sq1.f2 FROM t1 AS sq1 + WHERE EXISTS ( SELECT * FROM t1 AS sq2 + WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); + +--echo # this checks the result set above +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT sq1.f2 FROM t1 AS sq1 + WHERE EXISTS ( SELECT * FROM t1 AS sq2 + WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 ); + +set optimizer_switch= @save_optimizer_switch; + +DROP TABLE t1; + +--echo # +--echo # MDEV-12145: IN subquery used in WHERE of EXISTS subquery +--echo # + +CREATE TABLE t1 (f1 INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); + +CREATE TABLE t2 (i2 INT, KEY(i2)) ENGINE=MyISAM; +INSERT INTO t2 VALUES (8),(7),(1); + +CREATE TABLE t3 (f3 INT, i3 INT, KEY(i3)) ENGINE=MyISAM; +INSERT INTO t3 VALUES (8,0),(6,3),(2,8),(3,8),(1,6),(0,0),(1,0),(1,5); + +set @save_optimizer_switch= @@optimizer_switch; + +set optimizer_switch='exists_to_in=off'; +SELECT * FROM t1 + WHERE EXISTS ( SELECT * FROM t2, t3 + WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); +EXPLAIN EXTENDED +SELECT * FROM t1 + WHERE EXISTS ( SELECT * FROM t2, t3 + WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); + +--echo # this checks the result set above +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT * FROM t1 + WHERE EXISTS ( SELECT * FROM t2, t3 + WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) ); + +set optimizer_switch= @save_optimizer_switch; + +DROP TABLE t1,t2,t3; + +--echo # +--echo # MDEV-9686: IN subquery used in WHERE of a subquery from select list +--echo # + +CREATE TABLE t1 (pk INT PRIMARY KEY, f1 INT); +INSERT INTO t1 VALUES (1, 4),(2, 3),(3, 3),(4, 6),(5, 3); + +CREATE TABLE t2 (f2 INT); +INSERT INTO t2 VALUES (1),(2),(3),(4),(5); + +--echo # t1.pk is always IN ( SELECT f2 FROM t2 ), +--echo # so the IN condition should be true for every row, +--echo # and thus COUNT(*) should always return 5 + +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 + WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +EXPLAIN EXTENDED +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 + WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; + +--echo # this checks the result set above +set @save_optimizer_switch= @@optimizer_switch; +set optimizer_switch= 'materialization=off,semijoin=off'; +SELECT pk, f1, ( SELECT COUNT(*) FROM t2 + WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1; +set optimizer_switch= @save_optimizer_switch; + +DROP TABLE t1,t2; + --echo # End of 5.5 tests --echo # --echo # MDEV-7220: Materialization strategy is not used for REPLACE ... SELECT diff --git a/mysql-test/t/symlink-aria-11902.test b/mysql-test/t/symlink-aria-11902.test new file mode 100644 index 00000000000..a2a266cbb25 --- /dev/null +++ b/mysql-test/t/symlink-aria-11902.test @@ -0,0 +1,6 @@ +# +# MDEV-11902 mi_open race condition +# +source include/have_maria.inc; +set default_storage_engine=Aria; +source symlink-myisam-11902.test; diff --git a/mysql-test/t/symlink-myisam-11902.test b/mysql-test/t/symlink-myisam-11902.test new file mode 100644 index 00000000000..8fd4961d1fb --- /dev/null +++ b/mysql-test/t/symlink-myisam-11902.test @@ -0,0 +1,60 @@ +# +# MDEV-11902 mi_open race condition +# +source include/have_debug_sync.inc; +source include/have_symlink.inc; +source include/not_windows.inc; +call mtr.add_suppression("File.*t1.* not found"); + +create table mysql.t1 (a int, b char(16), index(a)); +insert mysql.t1 values (100, 'test'),(101,'test'); +let $datadir=`select @@datadir`; + +exec mkdir $MYSQLTEST_VARDIR/tmp/foo; +replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR; +eval create table t1 (a int, b char(16), index(a)) + data directory="$MYSQLTEST_VARDIR/tmp/foo"; +insert t1 values (200, 'some'),(201,'some'); +select * from t1; +flush tables; +set debug_sync='mi_open_datafile SIGNAL ok WAIT_FOR go'; +send select * from t1; +connect con1, localhost, root; +set debug_sync='now WAIT_FOR ok'; +exec rm -r $MYSQLTEST_VARDIR/tmp/foo; +exec ln -s $datadir/mysql $MYSQLTEST_VARDIR/tmp/foo; +set debug_sync='now SIGNAL go'; +connection default; +replace_regex / '.*\/tmp\// 'MYSQLTEST_VARDIR\/tmp\// /31/20/ /40/20/ /20.*/20 <errmsg>)/; +error 29; +reap; +flush tables; +drop table if exists t1; +exec rm -r $MYSQLTEST_VARDIR/tmp/foo; + +# same with INDEX DIRECTORY +exec mkdir $MYSQLTEST_VARDIR/tmp/foo; +replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR; +eval create table t1 (a int, b char(16), index (a)) + index directory="$MYSQLTEST_VARDIR/tmp/foo"; +insert t1 values (200, 'some'),(201,'some'); +explain select a from t1; +select a from t1; +flush tables; +set debug_sync='mi_open_kfile SIGNAL waiting WAIT_FOR run'; +send select a from t1; +connection con1; +set debug_sync='now WAIT_FOR waiting'; +exec rm -r $MYSQLTEST_VARDIR/tmp/foo; +exec ln -s $datadir/mysql $MYSQLTEST_VARDIR/tmp/foo; +set debug_sync='now SIGNAL run'; +connection default; +replace_regex / '.*\/test\// '.\/test\// /31/20/ /40/20/ /20.*/20 <errmsg>)/; +error ER_FILE_NOT_FOUND; +reap; +flush tables; +drop table if exists t1; +exec rm -r $MYSQLTEST_VARDIR/tmp/foo; + +drop table mysql.t1; +set debug_sync='RESET'; diff --git a/mysql-test/t/symlink.test b/mysql-test/t/symlink.test index 20585dd9ab7..7638061a8aa 100644 --- a/mysql-test/t/symlink.test +++ b/mysql-test/t/symlink.test @@ -298,3 +298,19 @@ show create table t1; create table t2 like t1; show create table t2; drop tables t1, t2; + +--echo # +--echo # Test for bug #25514146 DB_NAME IS IGNORED WHEN CREATING TABLE +--echo # WITH DATA DIRECTORY +--echo # + +--echo # Make sure we have no current database +CREATE DATABASE x; +USE x; +DROP DATABASE x; + +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +eval CREATE TABLE test.t1(id INT(11)) ENGINE MYISAM +DATA DIRECTORY "$MYSQLTEST_VARDIR/tmp"; + +DROP TABLE test.t1; diff --git a/mysql-test/t/table_elim.test b/mysql-test/t/table_elim.test index 24f48206013..717aecb42e2 100644 --- a/mysql-test/t/table_elim.test +++ b/mysql-test/t/table_elim.test @@ -534,12 +534,12 @@ INSERT IGNORE INTO t1 VALUES (0,'g'); CREATE TABLE t3 ( a varchar(1)) ; INSERT IGNORE INTO t3 VALUES ('g'); -CREATE TABLE t2 ( a int(11) NOT NULL, PRIMARY KEY (a)) ; +CREATE TABLE t2 ( a int(11) NOT NULL, PRIMARY KEY (a)); +INSERT INTO t2 VALUES (9), (10); create view v1 as SELECT t1.* FROM t1 LEFT JOIN t2 ON ( t1.a = t2.a ) WHERE t2.a <> 0; SELECT alias1.* FROM t3 LEFT JOIN v1 as alias1 ON ( t3.a = alias1.b ); EXPLAIN SELECT alias1.* FROM t3 LEFT JOIN v1 as alias1 ON ( t3.a = alias1.b ); - drop view v1; DROP TABLE t1,t2,t3; diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test index 9204ddd22e5..f4dc6a5d449 100644 --- a/mysql-test/t/union.test +++ b/mysql-test/t/union.test @@ -1022,7 +1022,6 @@ ORDER BY a; DROP TABLE t1; ---echo End of 5.0 tests -- echo # -- echo # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take -- echo # subselects into account @@ -1126,6 +1125,8 @@ create table t1 (a int); insert into t1 values (10),(10),(10),(2),(3),(4),(5),(6),(7),(8),(9),(1),(10); --sorted_result select a from t1 where false UNION select a from t1 limit 8; +--sorted_result +(select a from t1 where false) UNION (select a from t1) limit 8; drop table t1; --echo # @@ -1350,3 +1351,22 @@ UNION ; drop table t1; + + +--echo # +--echo # MDEV-10172: UNION query returns incorrect rows outside +--echo # conditional evaluation +--echo # + +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( + select * from t1 where d between '2016-06-02' and '2016-06-05' + union + (select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +drop table t1; + +--echo End of 5.0 tests diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 241b3b414f6..89a8e0c9ffc 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -5505,6 +5505,66 @@ SHOW CREATE VIEW v1; drop view v1; drop table t1,t2; + +--echo # +--echo # MDEV-12099: usage of mergeable view with LEFT JOIN +--echo # that can be converted to INNER JOIN +--echo # + +create table t1 (a int, b int, key(a)) engine=myisam; +insert into t1 values + (3,20), (7,10), (2,10), (4,30), (8,70), + (7,70), (9,100), (9,60), (8,80), (7,60); + +create table t2 (c int, d int, key (c)) engine=myisam; +insert into t2 values + (50,100), (20, 200), (10,300), + (150,100), (120, 200), (110,300), + (250,100), (220, 200), (210,300); + +create table t3(e int, f int not null, key(e), unique (f)) engine=myisam; +insert into t3 values + (100, 3), (300, 5), (400, 4), (300,7), + (300,2), (600, 13), (800, 15), (700, 14), + (600, 23), (800, 25), (700, 24); + +create view v1 as + select * from t2 left join t3 on t3.e=t2.d where t3.f is not null; + +select * + from t1 left join v1 on v1.c=t1.b + where t1.a < 5; + +select * + from t1 left join ( t2 left join t3 on t3.e=t2.d ) + on t2.c=t1.b and t3.f is not null + where t1.a < 5; + +explain extended +select * + from t1 left join v1 on v1.c=t1.b + where t1.a < 5; + +explain extended +select * + from t1 left join ( t2 left join t3 on t3.e=t2.d ) + on t2.c=t1.b and t3.f is not null + where t1.a < 5; + +explain extended +select * + from t1 left join v1 on v1.c=t1.b and v1.f=t1.a + where t1.a < 5; + +explain extended +select * + from t1 left join ( t2 left join t3 on t3.e=t2.d ) + on t2.c=t1.b and t3.f=t1.a and t3.f is not null + where t1.a < 5; + +drop view v1; +drop table t1,t2,t3; + --echo # ----------------------------------------------------------------- --echo # -- End of 5.5 tests. --echo # ----------------------------------------------------------------- @@ -5694,6 +5754,66 @@ drop table t1, t2; --error ER_BAD_FIELD_ERROR SELECT 1 FROM (SELECT 1 as a) AS b HAVING (SELECT `SOME_GARBAGE`.b.a)=1; + +--echo # +--echo # MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 +--echo # FOR UPDATE +--echo # + +CREATE TABLE t1 (a INT); +insert into t1 values (1),(2); + +CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE; +SHOW CREATE VIEW v1; +select * from v1; +DROP VIEW v1; + +CREATE VIEW v1 AS SELECT * FROM t1 LOCK IN SHARE MODE; +SHOW CREATE VIEW v1; +select * from v1; +DROP VIEW v1; + +DROP TABLE t1; + +--echo # +--echo # MDEV-8642: WHERE Clause not applied on View - Empty result set returned +--echo # + +CREATE TABLE `t1` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `use_case` int(11) DEFAULT NULL, + `current_deadline` date DEFAULT NULL, + `ts_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `id_UNIQUE` (`id`) +) ENGINE=MyISAM AUTO_INCREMENT=13976 DEFAULT CHARSET=latin1; +INSERT INTO `t1` VALUES (1,10,'2015-12-18','2015-08-18 08:38:16'); +INSERT INTO `t1` VALUES (2,20,'2015-10-18','2015-08-18 08:43:30'); +CREATE VIEW v1 AS SELECT + use_case as use_case_id, + ( + SELECT + deadline_sub.current_deadline + FROM + t1 deadline_sub + WHERE + deadline_sub.use_case = use_case_id + AND ts_create = (SELECT + MIN(ts_create) + FROM + t1 startdate_sub + WHERE + startdate_sub.use_case = use_case_id + ) + ) AS InitialDeadline +FROM + t1; + +SELECT * FROM v1 where use_case_id = 10; + +drop view v1; +drop table t1; + --echo # ----------------------------------------------------------------- --echo # -- End of 10.0 tests. --echo # ----------------------------------------------------------------- diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 3e25115599f..6c84d8d3ba6 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,58 +23,62 @@ # ############################################################################## +main.alter_table : Modified in 10.0.30 +main.count_distinct2 : MDEV-11768 - timeout main.create_delayed : MDEV-10605 - failed with timeout -main.ctype_utf32 : Modified on 2016-09-27 (merge) -main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_math : Modified on 2016-08-10 (merge) -main.func_misc : Modified on 2016-08-10 (merge) -main.group_min_max_innodb : Modified on 2016-08-25 (MDEV-10595) +main.debug_sync : MDEV-10607 - internal error +main.derived : Modified in 10.0.30 +main.derived_opt : MDEV-11768 - timeout +main.events_restart : MDEV-11221 - assertion failure +main.grant : Modified in 10.0.30 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout -main.index_merge_myisam : Modified on 2016-09-05 (include file changed) -main.index_merge_innodb : Modified on 2016-09-05 (MDEV-7142) -main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) +main.index_merge_innodb : MDEV-7142 - wrong result main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.loaddata : Modified on 2016-08-10 (merge) +main.join_nested : Modified in 10.0.30 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) -main.mysqlcheck : Modified on 2016-08-10 (merge) -main.mysqldump : MDEV-10512 - sporadic assertion failure +main.mysqldump : Modified in 10.0.30 main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqltest : MDEV-9269 - fails on Alpha -main.named_pipe : Modified on 2016-08-02 (MDEV-10383) -main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections +main.partition_innodb : Modified in 10.0.30 +main.partition_myisam : Modified in 10.0.30 main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.range : Modified on 2016-08-10 (merge) -main.range_mrr_icp : Modified on 2016-08-10 (merge) -main.query_cache : MDEV-10611 - sporadic mutex problem -main.shutdown : MDEV-10563 - sporadic crashes -main.sp-prelocking : Modified on 2016-08-10 (merge) +main.range_vs_index_merge : Modified in 10.0.30 +main.repair_symlink-5543 : Modified in 10.0.30 +main.show_explain : MDEV-10674 - wrong result +main.signal_demo3 : MDEV-11720 - Thread stack overrun on Solaris +main.sp : Modified in 10.0.30 +main.sp_notembedded : MDEV-10607 - internal error main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl_compress : MDEV-11110 - valgrind failures -main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.type_date : Modified on 2016-08-10 (merge) -main.type_uint : Modified on 2016-09-27 (merge) -main.view : Modified on 2016-08-10 (merge) -main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) +main.symlink-aria-11902 : Added in 10.0.30 +main.symlink-myisam-11902 : Added in 10.0.30 +main.table_elim : Modified in 10.0.30 +main.view : Modified in 10.0.30 +main.xa : MDEV-11769 - lock wait timeout #---------------------------------------------------------------- archive.archive-big : MDEV-10615 - table is marked as crashed +archive.archive_bitfield : MDEV-11771 - table is marked as crashed +archive.archive_symlink : MDEV-12170 - unexpected error on rmdir archive.discover : MDEV-10510 - table is marked as crashed archive.mysqlhotcopy_archive : MDEV-10995 - test hangs on debug build #---------------------------------------------------------------- binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed +binlog.binlog_max_binlog_stmt_cache_size : Added in 10.0.30 binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results +connect.xml : Uses xsample2.xml modified in 10.0.30 +connect.xml_zip : Added in 10.0.30 +connect.zip : Added in 10.0.30 #---------------------------------------------------------------- @@ -82,10 +86,7 @@ engines/rr_trx.* : MDEV-10998 - tests not maintained #---------------------------------------------------------------- -extra/binlog_tests.database : Modified on 2016-10-21 (Upstream MIPS test fixes) - -#---------------------------------------------------------------- - +federated_bug_35333 : Modified in 10.0.30 federated.federatedx : MDEV-10617 - Wrong checksum, timeouts federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips federated.federated_partition : MDEV-10417 - Fails on Mips @@ -93,19 +94,29 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- -funcs_2/charset.* : MDEV-10999 - test not maintained +funcs_1.memory_views : MDEV-11773 - timeout +funcs_1.processlist_val_ps : MDEV-12175 - Wrong result + +funcs_2/charset.* : MDEV-10999 - test not maintained #---------------------------------------------------------------- -innodb.binlog_consistent : MDEV-10618 - Server fails to start -innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) -innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb_bug54044 : Modified on 2016-09-27 (merge) -innodb.innodb_monitor : MDEV-10939 - Testcase timeout -innodb.innodb-wl5522 : rdiff file modified on 2016-08-10 (merge) -innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption -innodb.system_tables : Added on 2016-09-23 (MDEV-10775) +innodb.alter_key_block_size-11757 : Added in 10.0.30 +innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch +innodb.innodb-alter-table : MDEV-10619 - Testcase timeout +innodb.innodb_blob_unrecoverable_crash : Modified in 10.0.30 +innodb.innodb_bug14676111 : Modified in 10.0.30 +innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan +innodb.innodb_bug59641 : Modified in 10.0.30 +innodb.innodb-get-fk : Modified in 10.0.30 +innodb.innodb_monitor : MDEV-10939 - Testcase timeout +innodb.log_file_size : Added in 10.0.30 +innodb.read_only_recovery : Added in 10.0.30 +innodb.xa_recovery : Modified in 10.0.30 + +innodb_fts.create : Added in 10.0.30 +innodb_fts.innodb_fts_stopword_charset : MDEV-12052 - Crash on shutdown #---------------------------------------------------------------- @@ -118,15 +129,34 @@ mroonga/storage.index_multiple_column_unique_date_order_32bit_desc : Wrong resul #---------------------------------------------------------------- -multi_source.gtid : MDEV-10620, MDEV-10417 - Timeout in wait condition, fails on Mips +multi_source.gtid : MDEV-10417 - Fails on Mips multi_source.multisource : MDEV-10417 - Fails on Mips multi_source.simple : MDEV-4633 - Wrong slave status output multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_heartbeats + #---------------------------------------------------------------- -parts.partition_float_myisam : MDEV-10621 - Testcase timeout -parts.partition_int_myisam : MDEV-10621 - Testcase timeout +oqgraph.regression_mdev6282 : Modified in 10.0.30 +oqgraph.regression_mdev6345 : Modified in 10.0.30 + +#---------------------------------------------------------------- + +parts.partition_bigint_innodb : Added in 10.0.30 +parts.partition_bigint_myisam : Added in 10.0.30 +parts.partition_double_innodb : Added in 10.0.30 +parts.partition_double_myisam : Added in 10.0.30 +parts.partition_exch_qa_10 : MDEV-11765 - wrong result +parts.partition_float_innodb : Modified in 10.0.30 +parts.partition_float_myisam : Modified in 10.0.30 +parts.partition_int_innodb : Modified in 10.0.30 +parts.partition_int_myisam : Modified in 10.0.30 +parts.partition_mediumint_innodb : Added in 10.0.30 +parts.partition_mediumint_myisam : Added in 10.0.30 +parts.partition_smallint_innodb : Added in 10.0.30 +parts.partition_smallint_myisam : Added in 10.0.30 +parts.partition_tinyint_innodb : Added in 10.0.30 +parts.partition_tinyint_myisam : Added in 10.0.30 #---------------------------------------------------------------- @@ -134,49 +164,51 @@ perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match +perfschema.stage_mdl_procedure : MDEV-11545 - Wrong result +perfschema.threads_mysql : MDEV-12177 - Wrong result perfschema_stress.* : MDEV-10996 - tests not maintained #---------------------------------------------------------------- -plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11112 - valgrind warnings -plugins.pam : Modified on 2016-08-03 (MDEV-7329) -plugins.pam_cleartext : Modified on 2016-08-03 +plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11118 - wrong result plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc #---------------------------------------------------------------- +roles.create_and_grant_role : MDEV-11772 - wrong result + +#---------------------------------------------------------------- + rpl.last_insert_id : MDEV-10625 - warnings in error log rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering on master -rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout -rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout +rpl.rpl_checksum_cache : MDEV-12173 - InnoDB error rpl.rpl_ddl : MDEV-10417 - Fails on Mips -rpl.rpl_drop_db : Modified on 2016-10-21 (Upstream MIPS test fixes) rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master -rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log +rpl.rpl_heartbeat_basic : Modified in 10.0.30 rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips -rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave -rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts +rpl.rpl_mdev6386 : Modified in 10.0.30 +rpl.rpl_parallel : MDEV-10653 - Timeouts rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips -rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout +rpl.rpl_row_basic_11bugs : MDEV-12171 - Server failed to start rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x +rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status +rpl.rpl_show_slave_hosts : MDEV-12171 - Server failed to start rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock -rpl.rpl_switch_stm_row_mixed : MDEV-10611 - Wrong usage of mutex rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries -rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl/extra/rpl_tests.* : MDEV-10994 - tests not maintained @@ -187,6 +219,8 @@ spider.* : MDEV-9329 - tests are too memory-consuming spider/bg.direct_aggregate : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked +spider/bg.spider3_fixes : MDEV-7098 - Trying to unlock mutex that wasn't locked +spider/bg.spider_fixes_part : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.ha : MDEV-7914, MDEV-9329 - Crash, failures on s390x spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x spider/bg.spider_fixes : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x @@ -194,7 +228,7 @@ spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x #---------------------------------------------------------------- -sphinx.* : MDEV-10747 - tests are not run in buildbot, they can't be stable +sphinx.* : MDEV-10986 - sphinx tests fail in buildbot and outside #---------------------------------------------------------------- @@ -203,38 +237,45 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x -sys_vars.general_log_file_basic : Modified on 2016-08-09 (MDEV-10465) -sys_vars.slow_query_log_file_basic : Modified on 2016-08-09 (MDEV-10465) sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists +sys_vars.innodb_force_recovery_crash_basic : Modified in 10.0.30 +sys_vars.innodb_stats_include_delete_marked_basic : Added in 10.0.30 +sys_vars.innodb_status_output_basic : MDEV-12174 - Timeout +sys_vars.secure_file_priv : Modified in 10.0.30 +sys_vars.thread_cache_size_func : MDEV-11775 - wrong result #---------------------------------------------------------------- -tokudb.background_job_manager : MDEV-10327 - Assertion failure on server shutdown tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan -tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown +tokudb.dir_per_db : MDEV-11537 - wrong result +tokudb.dir_per_db_rename_to_nonexisting_schema : Added in 10.0.30 +tokudb.gap_lock_error : Added in 10.0.30 +tokudb.locks-select-update-3 : Modified in 10.0.30 +tokudb.percona_kill_idle_trx_tokudb : Added in 10.0.30 + tokudb_backup.* : MDEV-11001 - tests don't work tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output -tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown -tokudb_parts.* : MDEV-9891 - massive crashes on shutdown -tokudb_rpl_suites.* : MDEV-11001 - tests don't work +tokudb_bugs.xa : MDEV-11804 - Lock wait timeout + +tokudb_rpl.* : MDEV-11001 - tests don't work tokudb_sys_vars.* : MDEV-11001 - tests don't work -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_backup.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown +rpl-tokudb.rpl_extra_col_slave_tokudb : Result file modified in 10.0.30 #---------------------------------------------------------------- unit.ma_test_loghandler : MDEV-10638 - record read not ok +unit.pfs : MySQL:84457 - unittest pft-t failing #---------------------------------------------------------------- vcol.not_supported : MDEV-10639 - Testcase timeout vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout +vcol.vcol_misc : Modified in 10.0.30 +vcol.vcol_select_myisam : Modified in 10.0.30 +vcol.vcol_trigger_sp_innodb : Include file modified in 10.0.30 +vcol.vcol_trigger_sp_myisam : Include file modified in 10.0.30 +vcol.wrong_arena : Added in 10.0.30 #---------------------------------------------------------------- diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 77f17cf07ec..620bc958196 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1,5 +1,5 @@ # Copyright (c) 2005, 2015, Oracle and/or its affiliates. -# Copyright (c) 2008, 2016, MariaDB +# Copyright (c) 2008, 2017, MariaDB # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Library General Public @@ -353,10 +353,23 @@ Memcheck:Leak fun:memalign ... - fun:call_init + fun:call_init* + fun:_dl_init +} + +# This one is on OpenSuse 10.3 with gcc 5.4 +{ + memory "loss" from _dl_init 2 + Memcheck:Leak + fun:malloc + fun:pool + ... + fun:call_init* fun:_dl_init } + + # # dlclose can allocate memory for error message, the memory will be # freed by dlerror or other dl* function. @@ -954,6 +967,14 @@ fun:backtrace } +{ + memory leak in mysqld_exit + Memcheck:Leak + fun:malloc + fun:_dl_close_worker + fun:_dl_close +} + # # Bug in Glibc 2.9: http://sourceware.org/bugzilla/show_bug.cgi?id=10391 # Fixed in latest Glibc, but suppressed here for running tests on hosts @@ -1068,6 +1089,15 @@ fun:SSL_library_init } +{ + OpenSSL still reachable. + Memcheck:Leak + fun:*alloc + fun:CRYPTO_malloc + fun:sk_new + fun:SSL_COMP_get_compression_methods + fun:SSL_library_init +} { OpenSSL still reachable. @@ -1116,6 +1146,16 @@ } { + Memory Leak in loader and valgrind malloc + Memcheck:Leak + match-leak-kinds:reachable + obj:*/vgpreload_memcheck*.so + ... + obj:*/ld-*.so + ... +} + +{ ConnectSE: unixODBC SQLAllocEnv leaves some "still reachable" pointers Memcheck:Leak fun:malloc @@ -1346,4 +1386,3 @@ obj:*/libssl.so.0.9.8 ... } - diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 4756bbccf2f..a2429e01eaa 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -46,7 +46,7 @@ IF (WIN32) ENDIF() IF(UNIX) - SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c) + SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c my_setuser.c) ENDIF() IF(HAVE_ALARM) diff --git a/mysys/hash.c b/mysys/hash.c index 344b698a433..c2f3555396e 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -115,14 +115,19 @@ my_hash_init2(HASH *hash, uint growth_size, CHARSET_INFO *charset, static inline void my_hash_free_elements(HASH *hash) { + uint records= hash->records; + /* + Set records to 0 early to guard against anyone looking at the structure + during the free process + */ + hash->records= 0; if (hash->free) { HASH_LINK *data=dynamic_element(&hash->array,0,HASH_LINK*); - HASH_LINK *end= data + hash->records; + HASH_LINK *end= data + records; while (data < end) (*hash->free)((data++)->data); } - hash->records=0; } @@ -519,6 +524,9 @@ my_bool my_hash_insert(HASH *info, const uchar *record) The record with the same record ptr is removed. If there is a free-function it's called if record was found. + hash->free() is guarantee to be called only after the row has been + deleted from the hash and the hash can be reused by other threads. + @return @retval 0 ok @retval 1 Record not found diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 282433ea48d..a60a2ae657c 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -328,12 +328,6 @@ static int match_pins(LF_PINS *el, void *addr) return 0; } -#if STACK_DIRECTION < 0 -#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END)) -#else -#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR)) -#endif - #define next_node(P, X) (*((uchar * volatile *)(((uchar *)(X)) + (P)->free_ptr_offset))) #define anext_node(X) next_node(&allocator->pinbox, (X)) diff --git a/mysys/mf_format.c b/mysys/mf_format.c index 91354db0b64..6672a4386e4 100644 --- a/mysys/mf_format.c +++ b/mysys/mf_format.c @@ -97,13 +97,8 @@ char * fn_format(char * to, const char *name, const char *dir, pos=strmake(strmov(to,dev),name,length); (void) strmov(pos,ext); /* Don't convert extension */ } - /* - If MY_RETURN_REAL_PATH and MY_RESOLVE_SYMLINK is given, only do - realpath if the file is a symbolic link - */ if (flag & MY_RETURN_REAL_PATH) - (void) my_realpath(to, to, MYF(flag & MY_RESOLVE_SYMLINKS ? - MY_RESOLVE_LINK: 0)); + (void) my_realpath(to, to, MYF(0)); else if (flag & MY_RESOLVE_SYMLINKS) { strmov(buff,to); diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index f891a22b17d..8687c2e0c48 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -1815,6 +1815,7 @@ int my_b_flush_io_cache(IO_CACHE *info, It's currently safe to call this if one has called init_io_cache() on the 'info' object, even if init_io_cache() failed. This function is also safe to call twice with the same handle. + Note that info->file is not reset as the caller may still use ut for my_close() RETURN 0 ok @@ -1850,10 +1851,12 @@ int end_io_cache(IO_CACHE *info) if (info->type == SEQ_READ_APPEND) { /* Destroy allocated mutex */ - info->type= TYPE_NOT_SET; mysql_mutex_destroy(&info->append_buffer_lock); } info->share= 0; + info->type= TYPE_NOT_SET; /* Ensure that flush_io_cache() does nothing */ + info->write_end= 0; /* Ensure that my_b_write() fails */ + info->write_function= 0; /* my_b_write will crash if used */ DBUG_RETURN(error); } /* end_io_cache */ diff --git a/mysys/my_context.c b/mysys/my_context.c index 017cebdc110..29ba9f79c72 100644 --- a/mysys/my_context.c +++ b/mysys/my_context.c @@ -206,7 +206,8 @@ my_context_spawn(struct my_context *c, void (*f)(void *), void *d) ( "movq %%rsp, (%[save])\n\t" "movq %[stack], %%rsp\n\t" -#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) && !defined(__INTEL_COMPILER) +#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4) || __clang__) && \ + !defined(__INTEL_COMPILER) /* This emits a DWARF DW_CFA_undefined directive to make the return address undefined. This indicates that this is the top of the stack frame, and diff --git a/mysys/my_create.c b/mysys/my_create.c index 51de343d4a1..32ad3d44a7a 100644 --- a/mysys/my_create.c +++ b/mysys/my_create.c @@ -36,7 +36,7 @@ File my_create(const char *FileName, int CreateFlags, int access_flags, myf MyFlags) { - int fd, rc; + int fd; DBUG_ENTER("my_create"); DBUG_PRINT("my",("Name: '%s' CreateFlags: %d AccessFlags: %d MyFlags: %lu", FileName, CreateFlags, access_flags, MyFlags)); @@ -54,21 +54,7 @@ File my_create(const char *FileName, int CreateFlags, int access_flags, fd= -1; } - rc= my_register_filename(fd, FileName, FILE_BY_CREATE, + fd= my_register_filename(fd, FileName, FILE_BY_CREATE, EE_CANTCREATEFILE, MyFlags); - /* - my_register_filename() may fail on some platforms even if the call to - *open() above succeeds. In this case, don't leave the stale file because - callers assume the file to not exist if my_create() fails, so they don't - do any cleanups. - */ - if (unlikely(fd >= 0 && rc < 0)) - { - int tmp= my_errno; - my_close(fd, MyFlags); - my_delete(FileName, MyFlags); - my_errno= tmp; - } - - DBUG_RETURN(rc); + DBUG_RETURN(fd); } /* my_create */ diff --git a/mysys/my_default.c b/mysys/my_default.c index 0f9b70ca326..03cfa973b2c 100644 --- a/mysys/my_default.c +++ b/mysys/my_default.c @@ -90,7 +90,7 @@ static my_bool defaults_already_read= FALSE; /* Which directories are searched for options (and in which order) */ -#define MAX_DEFAULT_DIRS 6 +#define MAX_DEFAULT_DIRS 7 #define DEFAULT_DIRS_SIZE (MAX_DEFAULT_DIRS + 1) /* Terminate with NULL */ static const char **default_directories = NULL; @@ -1219,7 +1219,12 @@ static const char **init_default_directories(MEM_ROOT *alloc) errors += add_directory(alloc, "C:/", dirs); if (my_get_module_parent(fname_buffer, sizeof(fname_buffer)) != NULL) + { + errors += add_directory(alloc, fname_buffer, dirs); + + strncat(fname_buffer, "/data", sizeof(fname_buffer)); errors += add_directory(alloc, fname_buffer, dirs); + } } #else diff --git a/mysys/my_delete.c b/mysys/my_delete.c index 3dfe290dabe..0faf6079d98 100644 --- a/mysys/my_delete.c +++ b/mysys/my_delete.c @@ -21,6 +21,12 @@ static int my_win_unlink(const char *name); #endif +CREATE_NOSYMLINK_FUNCTION( + unlink_nosymlinks(const char *pathname), + unlinkat(dfd, filename, 0), + unlink(pathname) +); + int my_delete(const char *name, myf MyFlags) { int err; @@ -30,7 +36,10 @@ int my_delete(const char *name, myf MyFlags) #ifdef _WIN32 err = my_win_unlink(name); #else - err = unlink(name); + if (MyFlags & MY_NOSYMLINKS) + err= unlink_nosymlinks(name); + else + err= unlink(name); #endif if(err) diff --git a/mysys/my_div.c b/mysys/my_div.c index 660b87e5ab4..44eb5392421 100644 --- a/mysys/my_div.c +++ b/mysys/my_div.c @@ -27,7 +27,7 @@ char * my_filename(File fd) { DBUG_ENTER("my_filename"); - if ((uint) fd >= (uint) my_file_limit) + if ((uint) fd >= (uint) my_file_limit || !my_file_info[fd].name) DBUG_RETURN((char*) "UNKNOWN"); if (fd >= 0 && my_file_info[fd].type != UNOPEN) { diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c index b6027a99c90..4f14bc5aaef 100644 --- a/mysys/my_fopen.c +++ b/mysys/my_fopen.c @@ -69,19 +69,13 @@ FILE *my_fopen(const char *filename, int flags, myf MyFlags) DBUG_RETURN(fd); /* safeguard */ } mysql_mutex_lock(&THR_LOCK_open); - if ((my_file_info[filedesc].name= (char*) - my_strdup(filename,MyFlags))) - { - my_stream_opened++; - my_file_total_opened++; - my_file_info[filedesc].type= STREAM_BY_FOPEN; - mysql_mutex_unlock(&THR_LOCK_open); - DBUG_PRINT("exit",("stream: 0x%lx", (long) fd)); - DBUG_RETURN(fd); - } + my_file_info[filedesc].name= (char*) my_strdup(filename,MyFlags); + my_stream_opened++; + my_file_total_opened++; + my_file_info[filedesc].type= STREAM_BY_FOPEN; mysql_mutex_unlock(&THR_LOCK_open); - (void) my_fclose(fd,MyFlags); - my_errno=ENOMEM; + DBUG_PRINT("exit",("stream: 0x%lx", (long) fd)); + DBUG_RETURN(fd); } else my_errno=errno; diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 2a4557118b0..e68a08ee735 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -895,15 +895,39 @@ my_bool getopt_compare_strings(register const char *s, register const char *t, /* function: eval_num_suffix + Transforms suffix like k/m/g to their real value. +*/ + +static inline long eval_num_suffix(char *suffix, int *error) +{ + long num= 1; + if (*suffix == 'k' || *suffix == 'K') + num*= 1024L; + else if (*suffix == 'm' || *suffix == 'M') + num*= 1024L * 1024L; + else if (*suffix == 'g' || *suffix == 'G') + num*= 1024L * 1024L * 1024L; + else if (*suffix) + { + *error= 1; + return 0; + } + return num; +} + +/* + function: eval_num_suffix_ll + Transforms a number with a suffix to real number. Suffix can be k|K for kilo, m|M for mega or g|G for giga. */ -static longlong eval_num_suffix(char *argument, int *error, char *option_name) +static longlong eval_num_suffix_ll(char *argument, + int *error, char *option_name) { char *endchar; longlong num; - DBUG_ENTER("eval_num_suffix"); + DBUG_ENTER("eval_num_suffix_ll"); *error= 0; @@ -916,23 +940,47 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) *error= 1; DBUG_RETURN(0); } - if (*endchar == 'k' || *endchar == 'K') - num*= 1024L; - else if (*endchar == 'm' || *endchar == 'M') - num*= 1024L * 1024L; - else if (*endchar == 'g' || *endchar == 'G') - num*= 1024L * 1024L * 1024L; - else if (*endchar) - { + num*= eval_num_suffix(endchar, error); + if (*error) fprintf(stderr, "Unknown suffix '%c' used for variable '%s' (value '%s')\n", *endchar, option_name, argument); + DBUG_RETURN(num); +} + +/* + function: eval_num_suffix_ull + + Transforms a number with a suffix to positive Integer. Suffix can + be k|K for kilo, m|M for mega or g|G for giga. +*/ + +static ulonglong eval_num_suffix_ull(char *argument, + int *error, char *option_name) +{ + char *endchar; + ulonglong num; + DBUG_ENTER("eval_num_suffix_ull"); + + *error= 0; + errno= 0; + num= strtoull(argument, &endchar, 10); + if (errno == ERANGE) + { + my_getopt_error_reporter(ERROR_LEVEL, + "Incorrect integer value: '%s'", argument); *error= 1; DBUG_RETURN(0); } + num*= eval_num_suffix(endchar, error); + if (*error) + fprintf(stderr, + "Unknown suffix '%c' used for variable '%s' (value '%s')\n", + *endchar, option_name, argument); DBUG_RETURN(num); } + /* function: getopt_ll @@ -946,7 +994,7 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) static longlong getopt_ll(char *arg, const struct my_option *optp, int *err) { - longlong num=eval_num_suffix(arg, err, (char*) optp->name); + longlong num=eval_num_suffix_ll(arg, err, (char*) optp->name); return getopt_ll_limit_value(num, optp, NULL); } @@ -1023,7 +1071,7 @@ longlong getopt_ll_limit_value(longlong num, const struct my_option *optp, static ulonglong getopt_ull(char *arg, const struct my_option *optp, int *err) { - ulonglong num= eval_num_suffix(arg, err, (char*) optp->name); + ulonglong num= eval_num_suffix_ull(arg, err, (char*) optp->name); return getopt_ull_limit_value(num, optp, NULL); } diff --git a/mysys/my_init.c b/mysys/my_init.c index 2c06425f6fb..dee41e1202a 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -227,7 +227,7 @@ Voluntary context switches %ld, Involuntary context switches %ld\n", /* At very last, delete mysys key, it is used everywhere including DBUG */ pthread_key_delete(THR_KEY_mysys); - my_init_done=0; + my_init_done= my_thr_key_mysys_exists= 0; } /* my_end */ #ifndef DBUG_OFF diff --git a/mysys/my_open.c b/mysys/my_open.c index 5263ba4b5c8..b1327a316e9 100644 --- a/mysys/my_open.c +++ b/mysys/my_open.c @@ -15,9 +15,14 @@ #include "mysys_priv.h" #include "mysys_err.h" -#include <my_dir.h> +#include <m_string.h> #include <errno.h> +CREATE_NOSYMLINK_FUNCTION( + open_nosymlinks(const char *pathname, int flags, int mode), + openat(dfd, filename, O_NOFOLLOW | flags, mode), + open(pathname, O_NOFOLLOW | flags, mode) +); /* Open a file @@ -45,10 +50,11 @@ File my_open(const char *FileName, int Flags, myf MyFlags) MyFlags|= my_global_flags; #if defined(_WIN32) fd= my_win_open(FileName, Flags); -#elif !defined(NO_OPEN_3) - fd = open(FileName, Flags, my_umask); /* Normal unix */ #else - fd = open((char *) FileName, Flags); + if (MyFlags & MY_NOSYMLINKS) + fd = open_nosymlinks(FileName, Flags, my_umask); + else + fd = open(FileName, Flags, my_umask); #endif fd= my_register_filename(fd, FileName, FILE_BY_OPEN, @@ -131,25 +137,16 @@ File my_register_filename(File fd, const char *FileName, enum file_type thread_safe_increment(my_file_opened,&THR_LOCK_open); DBUG_RETURN(fd); /* safeguard */ } - else - { - mysql_mutex_lock(&THR_LOCK_open); - if ((my_file_info[fd].name = (char*) my_strdup(FileName,MyFlags))) - { - my_file_opened++; - my_file_total_opened++; - my_file_info[fd].type = type_of_file; - mysql_mutex_unlock(&THR_LOCK_open); - DBUG_PRINT("exit",("fd: %d",fd)); - DBUG_RETURN(fd); - } - mysql_mutex_unlock(&THR_LOCK_open); - my_errno= ENOMEM; - } - (void) my_close(fd, MyFlags); + mysql_mutex_lock(&THR_LOCK_open); + my_file_info[fd].name = (char*) my_strdup(FileName, MyFlags); + my_file_opened++; + my_file_total_opened++; + my_file_info[fd].type = type_of_file; + mysql_mutex_unlock(&THR_LOCK_open); + DBUG_PRINT("exit",("fd: %d",fd)); + DBUG_RETURN(fd); } - else - my_errno= errno; + my_errno= errno; DBUG_PRINT("error",("Got error %d on open", my_errno)); if (MyFlags & (MY_FFNF | MY_FAE | MY_WME)) diff --git a/mysys/my_setuser.c b/mysys/my_setuser.c new file mode 100644 index 00000000000..14ab04dd10f --- /dev/null +++ b/mysys/my_setuser.c @@ -0,0 +1,82 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#ifdef HAVE_GRP_H +#include <grp.h> +#endif + +struct passwd *my_check_user(const char *user, myf MyFlags) +{ + struct passwd *user_info; + uid_t user_id= geteuid(); + DBUG_ENTER("my_check_user"); + + // Don't bother if we aren't superuser + if (user_id) + { + if (user) + { + /* Don't give a warning, if real user is same as given with --user */ + user_info= getpwnam(user); + if (!user_info || user_id != user_info->pw_uid) + { + my_errno= EPERM; + if (MyFlags & MY_WME) + my_printf_error(my_errno, "One can only use the --user switch if " + "running as root", MYF(ME_JUST_WARNING|ME_NOREFRESH)); + } + } + DBUG_RETURN(NULL); + } + if (!user) + { + if (MyFlags & MY_FAE) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Please consult the Knowledge Base to find " + "out how to run mysqld as root!", MYF(ME_NOREFRESH)); + } + DBUG_RETURN(NULL); + } + if (!strcmp(user,"root")) + DBUG_RETURN(NULL); + + if (!(user_info= getpwnam(user))) + { + // Allow a numeric uid to be used + int err= 0; + user_id= my_strtoll10(user, NULL, &err); + if (err || !(user_info= getpwuid(user_id))) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Can't change to run as user '%s'. Please " + "check that the user exists!", MYF(ME_NOREFRESH), user); + DBUG_RETURN(NULL); + } + } + DBUG_ASSERT(user_info); + DBUG_RETURN(user_info); +} + +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags) +{ + DBUG_ENTER("my_set_user"); + + DBUG_ASSERT(user_info != 0); +#ifdef HAVE_INITGROUPS + initgroups(user, user_info->pw_gid); +#endif + if (setgid(user_info->pw_gid) == -1 || setuid(user_info->pw_uid) == -1) + { + my_errno= errno; + if (MyFlags & MY_WME) + my_printf_error(errno, "Cannot change uid/gid (errno: %d)", MYF(ME_NOREFRESH), + errno); + DBUG_RETURN(my_errno); + } + DBUG_RETURN(0); +} diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c index b0e910f7ba0..72648d4c9a8 100644 --- a/mysys/my_symlink.c +++ b/mysys/my_symlink.c @@ -1,5 +1,6 @@ /* Copyright (c) 2001, 2011, Oracle and/or its affiliates + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,6 +24,14 @@ #include <sys/stat.h> #endif +static int always_valid(const char *filename __attribute__((unused))) +{ + return 0; +} + +int (*mysys_test_invalid_symlink)(const char *filename)= always_valid; + + /* Reads the content of a symbolic link If the file is not a symbolic link, return the original file name in to. @@ -120,6 +129,11 @@ int my_is_symlink(const char *filename __attribute__((unused))) to is guaranteed to never set to a string longer than FN_REFLEN (including the end \0) + + On error returns -1, unless error is file not found, in which case it + is 1. + + Sets my_errno to specific error number. */ int my_realpath(char *to, const char *filename, myf MyFlags) @@ -145,7 +159,10 @@ int my_realpath(char *to, const char *filename, myf MyFlags) if (MyFlags & MY_WME) my_error(EE_REALPATH, MYF(0), filename, my_errno); my_load_path(to, filename, NullS); - result= -1; + if (my_errno == ENOENT) + result= 1; + else + result= -1; } DBUG_RETURN(result); #elif defined(_WIN32) @@ -168,3 +185,78 @@ int my_realpath(char *to, const char *filename, myf MyFlags) #endif return 0; } + +#ifdef HAVE_OPEN_PARENT_DIR_NOSYMLINKS +/** opens the parent dir. walks the path, and does not resolve symlinks + + returns the pointer to the file name (basename) within the pathname + or NULL in case of an error + + stores the parent dir (dirname) file descriptor in pdfd. + It can be -1 even if there was no error! + + This is used for symlinked tables for DATA/INDEX DIRECTORY. + The paths there have been realpath()-ed. So, we can assume here that + + * `pathname` is an absolute path + * no '.', '..', and '//' in the path + * file exists +*/ + +const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd) +{ + char buf[PATH_MAX+1]; + char *s= buf, *e= buf+1, *end= strnmov(buf, pathname, sizeof(buf)); + int fd, dfd= -1; + + if (*end) + { + errno= ENAMETOOLONG; + return NULL; + } + + if (*s != '/') /* not an absolute path */ + { + errno= ENOENT; + return NULL; + } + + for (;;) + { + if (*e == '/') /* '//' in the path */ + { + errno= ENOENT; + goto err; + } + while (*e && *e != '/') + e++; + *e= 0; + + if (!memcmp(s, ".", 2) || !memcmp(s, "..", 3)) + { + errno= ENOENT; + goto err; + } + + if (++e >= end) + { + *pdfd= dfd; + return pathname + (s - buf); + } + + fd = openat(dfd, s, O_NOFOLLOW | O_PATH); + if (fd < 0) + goto err; + + if (dfd >= 0) + close(dfd); + + dfd= fd; + s= e; + } +err: + if (dfd >= 0) + close(dfd); + return NULL; +} +#endif diff --git a/mysys/my_symlink2.c b/mysys/my_symlink2.c index fcaf78ccff6..c851468ce1b 100644 --- a/mysys/my_symlink2.c +++ b/mysys/my_symlink2.c @@ -92,27 +92,6 @@ File my_create_with_symlink(const char *linkname, const char *filename, } /* - If the file was a symlink, delete both symlink and the file which the - symlink pointed to. -*/ - -int my_delete_with_symlink(const char *name, myf MyFlags) -{ - char link_name[FN_REFLEN]; - int was_symlink= (!my_disable_symlinks && - !my_readlink(link_name, name, MYF(0))); - int result; - DBUG_ENTER("my_delete_with_symlink"); - - if (!(result=my_delete(name, MyFlags))) - { - if (was_symlink) - result=my_delete(link_name, MyFlags); - } - DBUG_RETURN(result); -} - -/* If the file is a normal file, just rename it. If the file is a symlink: - Create a new file with the name 'to' that points at @@ -182,3 +161,29 @@ int my_rename_with_symlink(const char *from, const char *to, myf MyFlags) DBUG_RETURN(result); #endif /* HAVE_READLINK */ } + +/** delete a - possibly symlinked - table file + + This is used to delete a file that is part of a table (e.g. MYI or MYD + file of MyISAM) when dropping a table. A file might be a symlink - + if the table was created with DATA DIRECTORY or INDEX DIRECTORY - + in this case both the symlink and the symlinked file are deleted, + but only if the symlinked file is not in the datadir. +*/ +int my_handler_delete_with_symlink(const char *filename, myf sync_dir) +{ + char real[FN_REFLEN]; + int res= 0; + DBUG_ENTER("my_handler_delete_with_symlink"); + + if (my_is_symlink(filename)) + { + /* + Delete the symlinked file only if the symlink is not + pointing into datadir. + */ + if (!(my_realpath(real, filename, MYF(0)) || mysys_test_invalid_symlink(real))) + res= my_delete(real, MYF(MY_NOSYMLINKS | sync_dir)); + } + DBUG_RETURN(my_delete(filename, MYF(sync_dir)) || res); +} diff --git a/mysys/my_sync.c b/mysys/my_sync.c index c0afd587ada..d1e239692f1 100644 --- a/mysys/my_sync.c +++ b/mysys/my_sync.c @@ -196,7 +196,7 @@ int my_sync_dir_by_file(const char *file_name __attribute__((unused)), char dir_name[FN_REFLEN]; size_t dir_name_length; dirname_part(dir_name, file_name, &dir_name_length); - return my_sync_dir(dir_name, my_flags); + return my_sync_dir(dir_name, my_flags & ~MY_NOSYMLINKS); #else return 0; #endif diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 1e4b85583b1..be8af98e2b9 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -44,6 +44,8 @@ static uint get_thread_lib(void); /** True if @c my_thread_global_init() has been called. */ static my_bool my_thread_global_init_done= 0; +/* True if THR_KEY_mysys is created */ +my_bool my_thr_key_mysys_exists= 0; /* @@ -167,11 +169,20 @@ my_bool my_thread_global_init(void) return 0; my_thread_global_init_done= 1; - if ((pth_ret= pthread_key_create(&THR_KEY_mysys, NULL)) != 0) + /* + THR_KEY_mysys is deleted in my_end() as DBUG libraries are using it even + after my_thread_global_end() is called. + my_thr_key_mysys_exist is used to protect against application like QT + that calls my_thread_global_init() + my_thread_global_end() multiple times + without calling my_init() + my_end(). + */ + if (!my_thr_key_mysys_exists && + (pth_ret= pthread_key_create(&THR_KEY_mysys, NULL)) != 0) { fprintf(stderr, "Can't initialize threads: error %d\n", pth_ret); return 1; } + my_thr_key_mysys_exists= 1; /* Mutex used by my_thread_init() and after my_thread_destroy_mutex() */ my_thread_init_internal_mutex(); diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h index 9c6855bb92f..51650567609 100644 --- a/mysys/mysys_priv.h +++ b/mysys/mysys_priv.h @@ -92,6 +92,38 @@ size_t sf_malloc_usable_size(void *ptr, my_bool *is_thread_specific); void my_error_unregister_all(void); +#ifndef O_PATH /* not Linux */ +#if defined(O_SEARCH) /* Illumos */ +#define O_PATH O_SEARCH +#elif defined(O_EXEC) /* FreeBSD */ +#define O_PATH O_EXEC +#endif +#endif + +#ifdef O_PATH +#define HAVE_OPEN_PARENT_DIR_NOSYMLINKS +const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd); +#define NOSYMLINK_FUNCTION_BODY(AT,NOAT) \ + int dfd, res; \ + const char *filename= my_open_parent_dir_nosymlinks(pathname, &dfd); \ + if (filename == NULL) return -1; \ + res= AT; \ + if (dfd >= 0) close(dfd); \ + return res; +#elif defined(HAVE_REALPATH) +#define NOSYMLINK_FUNCTION_BODY(AT,NOAT) \ + char buf[PATH_MAX+1]; \ + if (realpath(pathname, buf) == NULL) return -1; \ + if (strcmp(pathname, buf)) { errno= ENOTDIR; return -1; } \ + return NOAT; +#else +#define NOSYMLINK_FUNCTION_BODY(AT,NOAT) \ + return NOAT; +#endif + +#define CREATE_NOSYMLINK_FUNCTION(PROTO,AT,NOAT) \ +static int PROTO { NOSYMLINK_FUNCTION_BODY(AT,NOAT) } + #ifdef _WIN32 #include <sys/stat.h> /* my_winfile.c exports, should not be used outside mysys */ diff --git a/pcre/AUTHORS b/pcre/AUTHORS index 342417a8a19..291657caef1 100644 --- a/pcre/AUTHORS +++ b/pcre/AUTHORS @@ -8,7 +8,7 @@ Email domain: cam.ac.uk University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2016 University of Cambridge +Copyright (c) 1997-2017 University of Cambridge All rights reserved @@ -19,7 +19,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2010-2016 Zoltan Herczeg +Copyright(c) 2010-2017 Zoltan Herczeg All rights reserved. @@ -30,7 +30,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2009-2016 Zoltan Herczeg +Copyright(c) 2009-2017 Zoltan Herczeg All rights reserved. diff --git a/pcre/CMakeLists.txt b/pcre/CMakeLists.txt index 91e2e781fc2..30b06a46fef 100644 --- a/pcre/CMakeLists.txt +++ b/pcre/CMakeLists.txt @@ -66,6 +66,7 @@ # 2013-10-08 PH got rid of the "source" command, which is a bash-ism (use ".") # 2013-11-05 PH added support for PARENS_NEST_LIMIT # 2016-03-01 PH applied Chris Wilson's patch for MSVC static build +# 2016-06-24 PH applied Chris Wilson's revised patch (adds a separate option) PROJECT(PCRE C CXX) diff --git a/pcre/ChangeLog b/pcre/ChangeLog index a34f845f8a1..01511b1327c 100644 --- a/pcre/ChangeLog +++ b/pcre/ChangeLog @@ -4,6 +4,53 @@ ChangeLog for PCRE Note that the PCRE 8.xx series (PCRE1) is now in a bugfix-only state. All development is happening in the PCRE2 10.xx series. +Version 8.40 11-January-2017 +---------------------------- + +1. Using -o with -M in pcregrep could cause unnecessary repeated output when + the match extended over a line boundary. + +2. Applied Chris Wilson's second patch (Bugzilla #1681) to CMakeLists.txt for + MSVC static compilation, putting the first patch under a new option. + +3. Fix register overwite in JIT when SSE2 acceleration is enabled. + +4. Ignore "show all captures" (/=) for DFA matching. + +5. Fix JIT unaligned accesses on x86. Patch by Marc Mutz. + +6. In any wide-character mode (8-bit UTF or any 16-bit or 32-bit mode), + without PCRE_UCP set, a negative character type such as \D in a positive + class should cause all characters greater than 255 to match, whatever else + is in the class. There was a bug that caused this not to happen if a + Unicode property item was added to such a class, for example [\D\P{Nd}] or + [\W\pL]. + +7. When pcretest was outputing information from a callout, the caret indicator + for the current position in the subject line was incorrect if it was after + an escape sequence for a character whose code point was greater than + \x{ff}. + +8. A pattern such as (?<RA>abc)(?(R)xyz) was incorrectly compiled such that + the conditional was interpreted as a reference to capturing group 1 instead + of a test for recursion. Any group whose name began with R was + misinterpreted in this way. (The reference interpretation should only + happen if the group's name is precisely "R".) + +9. A number of bugs have been mended relating to match start-up optimizations + when the first thing in a pattern is a positive lookahead. These all + applied only when PCRE_NO_START_OPTIMIZE was *not* set: + + (a) A pattern such as (?=.*X)X$ was incorrectly optimized as if it needed + both an initial 'X' and a following 'X'. + (b) Some patterns starting with an assertion that started with .* were + incorrectly optimized as having to match at the start of the subject or + after a newline. There are cases where this is not true, for example, + (?=.*[A-Z])(?=.{8,16})(?!.*[\s]) matches after the start in lines that + start with spaces. Starting .* in an assertion is no longer taken as an + indication of matching at the start (or after a newline). + + Version 8.39 14-June-2016 ------------------------- diff --git a/pcre/LICENCE b/pcre/LICENCE index dd977af971b..dd9071a8dd8 100644 --- a/pcre/LICENCE +++ b/pcre/LICENCE @@ -25,7 +25,7 @@ Email domain: cam.ac.uk University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2016 University of Cambridge +Copyright (c) 1997-2017 University of Cambridge All rights reserved. @@ -36,7 +36,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2010-2016 Zoltan Herczeg +Copyright(c) 2010-2017 Zoltan Herczeg All rights reserved. @@ -47,7 +47,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2009-2016 Zoltan Herczeg +Copyright(c) 2009-2017 Zoltan Herczeg All rights reserved. diff --git a/pcre/NEWS b/pcre/NEWS index 0ca1bab2a4c..b92c4f9605e 100644 --- a/pcre/NEWS +++ b/pcre/NEWS @@ -1,6 +1,12 @@ News about PCRE releases ------------------------ +Release 8.40 11-January-2017 +---------------------------- + +This is a bug-fix release. + + Release 8.39 14-June-2016 ------------------------- diff --git a/pcre/configure.ac b/pcre/configure.ac index 3cefaf100f9..24ef7271e05 100644 --- a/pcre/configure.ac +++ b/pcre/configure.ac @@ -9,17 +9,17 @@ dnl The PCRE_PRERELEASE feature is for identifying release candidates. It might dnl be defined as -RC2, for example. For real releases, it should be empty. m4_define(pcre_major, [8]) -m4_define(pcre_minor, [39]) +m4_define(pcre_minor, [40]) m4_define(pcre_prerelease, []) -m4_define(pcre_date, [2016-06-14]) +m4_define(pcre_date, [2017-01-11]) # NOTE: The CMakeLists.txt file searches for the above variables in the first # 50 lines of this file. Please update that if the variables above are moved. # Libtool shared library interface versions (current:revision:age) -m4_define(libpcre_version, [3:7:2]) -m4_define(libpcre16_version, [2:7:2]) -m4_define(libpcre32_version, [0:7:0]) +m4_define(libpcre_version, [3:8:2]) +m4_define(libpcre16_version, [2:8:2]) +m4_define(libpcre32_version, [0:8:0]) m4_define(libpcreposix_version, [0:4:0]) m4_define(libpcrecpp_version, [0:1:0]) diff --git a/pcre/doc/html/pcrecompat.html b/pcre/doc/html/pcrecompat.html index 3e6226692ee..d95570ef179 100644 --- a/pcre/doc/html/pcrecompat.html +++ b/pcre/doc/html/pcrecompat.html @@ -128,7 +128,7 @@ the pattern /^(a(b)?)+$/ in Perl leaves $2 unset, but in PCRE it is set to "b". 14. PCRE's handling of duplicate subpattern numbers and duplicate subpattern names is not as general as Perl's. This is a consequence of the fact the PCRE works internally just with numbers, using an external table to translate -between numbers and names. In particular, a pattern such as (?|(?<a>A)|(?<b)B), +between numbers and names. In particular, a pattern such as (?|(?<a>A)|(?<b>B), where the two capturing parentheses have the same number but different names, is not supported, and causes an error at compile time. If it were allowed, it would not be possible to distinguish which parentheses matched, because both diff --git a/pcre/doc/html/pcrepattern.html b/pcre/doc/html/pcrepattern.html index 55034a7edf6..96fc72986f6 100644 --- a/pcre/doc/html/pcrepattern.html +++ b/pcre/doc/html/pcrepattern.html @@ -358,24 +358,24 @@ When PCRE is compiled in EBCDIC mode, \a, \e, \f, \n, \r, and \t generate the appropriate EBCDIC code values. The \c escape is processed as specified for Perl in the <b>perlebcdic</b> document. The only characters that are allowed after \c are A-Z, a-z, or one of @, [, \, ], ^, _, or ?. Any -other character provokes a compile-time error. The sequence \@ encodes -character code 0; the letters (in either case) encode characters 1-26 (hex 01 -to hex 1A); [, \, ], ^, and _ encode characters 27-31 (hex 1B to hex 1F), and -\? becomes either 255 (hex FF) or 95 (hex 5F). +other character provokes a compile-time error. The sequence \c@ encodes +character code 0; after \c the letters (in either case) encode characters 1-26 +(hex 01 to hex 1A); [, \, ], ^, and _ encode characters 27-31 (hex 1B to hex +1F), and \c? becomes either 255 (hex FF) or 95 (hex 5F). </P> <P> -Thus, apart from \?, these escapes generate the same character code values as +Thus, apart from \c?, these escapes generate the same character code values as they do in an ASCII environment, though the meanings of the values mostly -differ. For example, \G always generates code value 7, which is BEL in ASCII +differ. For example, \cG always generates code value 7, which is BEL in ASCII but DEL in EBCDIC. </P> <P> -The sequence \? generates DEL (127, hex 7F) in an ASCII environment, but +The sequence \c? generates DEL (127, hex 7F) in an ASCII environment, but because 127 is not a control character in EBCDIC, Perl makes it generate the APC character. Unfortunately, there are several variants of EBCDIC. In most of them the APC character has the value 255 (hex FF), but in the one Perl calls POSIX-BC its value is 95 (hex 5F). If certain other characters have POSIX-BC -values, PCRE makes \? generate 95; otherwise it generates 255. +values, PCRE makes \c? generate 95; otherwise it generates 255. </P> <P> After \0 up to two further octal digits are read. If there are fewer than two @@ -1512,13 +1512,8 @@ J, U and X respectively. <P> When one of these option changes occurs at top level (that is, not inside subpattern parentheses), the change applies to the remainder of the pattern -that follows. If the change is placed right at the start of a pattern, PCRE -extracts it into the global options (and it will therefore show up in data -extracted by the <b>pcre_fullinfo()</b> function). -</P> -<P> -An option change within a subpattern (see below for a description of -subpatterns) affects only that part of the subpattern that follows it, so +that follows. An option change within a subpattern (see below for a description +of subpatterns) affects only that part of the subpattern that follows it, so <pre> (a(?i)b)c </pre> @@ -2160,6 +2155,14 @@ capturing is carried out only for positive assertions. (Perl sometimes, but not always, does do capturing in negative assertions.) </P> <P> +WARNING: If a positive assertion containing one or more capturing subpatterns +succeeds, but failure to match later in the pattern causes backtracking over +this assertion, the captures within the assertion are reset only if no higher +numbered captures are already set. This is, unfortunately, a fundamental +limitation of the current implementation, and as PCRE1 is now in +maintenance-only status, it is unlikely ever to change. +</P> +<P> For compatibility with Perl, assertion subpatterns may be repeated; though it makes no sense to assert the same thing several times, the side effect of capturing parentheses may occasionally be useful. In practice, there only three @@ -3264,9 +3267,9 @@ Cambridge CB2 3QH, England. </P> <br><a name="SEC30" href="#TOC1">REVISION</a><br> <P> -Last updated: 14 June 2015 +Last updated: 23 October 2016 <br> -Copyright © 1997-2015 University of Cambridge. +Copyright © 1997-2016 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE index page</a>. diff --git a/pcre/doc/pcre.txt b/pcre/doc/pcre.txt index f5eb347e18a..d68d5033ceb 100644 --- a/pcre/doc/pcre.txt +++ b/pcre/doc/pcre.txt @@ -4640,7 +4640,7 @@ DIFFERENCES BETWEEN PCRE AND PERL pattern names is not as general as Perl's. This is a consequence of the fact the PCRE works internally just with numbers, using an external ta- ble to translate between numbers and names. In particular, a pattern - such as (?|(?<a>A)|(?<b)B), where the two capturing parentheses have + such as (?|(?<a>A)|(?<b>B), where the two capturing parentheses have the same number but different names, is not supported, and causes an error at compile time. If it were allowed, it would not be possible to distinguish which parentheses matched, because both names map to cap- @@ -5028,55 +5028,56 @@ BACKSLASH ate the appropriate EBCDIC code values. The \c escape is processed as specified for Perl in the perlebcdic document. The only characters that are allowed after \c are A-Z, a-z, or one of @, [, \, ], ^, _, or ?. - Any other character provokes a compile-time error. The sequence \@ - encodes character code 0; the letters (in either case) encode charac- - ters 1-26 (hex 01 to hex 1A); [, \, ], ^, and _ encode characters 27-31 - (hex 1B to hex 1F), and \? becomes either 255 (hex FF) or 95 (hex 5F). - - Thus, apart from \?, these escapes generate the same character code - values as they do in an ASCII environment, though the meanings of the - values mostly differ. For example, \G always generates code value 7, + Any other character provokes a compile-time error. The sequence \c@ + encodes character code 0; after \c the letters (in either case) encode + characters 1-26 (hex 01 to hex 1A); [, \, ], ^, and _ encode characters + 27-31 (hex 1B to hex 1F), and \c? becomes either 255 (hex FF) or 95 + (hex 5F). + + Thus, apart from \c?, these escapes generate the same character code + values as they do in an ASCII environment, though the meanings of the + values mostly differ. For example, \cG always generates code value 7, which is BEL in ASCII but DEL in EBCDIC. - The sequence \? generates DEL (127, hex 7F) in an ASCII environment, - but because 127 is not a control character in EBCDIC, Perl makes it - generate the APC character. Unfortunately, there are several variants - of EBCDIC. In most of them the APC character has the value 255 (hex - FF), but in the one Perl calls POSIX-BC its value is 95 (hex 5F). If - certain other characters have POSIX-BC values, PCRE makes \? generate + The sequence \c? generates DEL (127, hex 7F) in an ASCII environment, + but because 127 is not a control character in EBCDIC, Perl makes it + generate the APC character. Unfortunately, there are several variants + of EBCDIC. In most of them the APC character has the value 255 (hex + FF), but in the one Perl calls POSIX-BC its value is 95 (hex 5F). If + certain other characters have POSIX-BC values, PCRE makes \c? generate 95; otherwise it generates 255. - After \0 up to two further octal digits are read. If there are fewer - than two digits, just those that are present are used. Thus the + After \0 up to two further octal digits are read. If there are fewer + than two digits, just those that are present are used. Thus the sequence \0\x\015 specifies two binary zeros followed by a CR character (code value 13). Make sure you supply two digits after the initial zero if the pattern character that follows is itself an octal digit. - The escape \o must be followed by a sequence of octal digits, enclosed - in braces. An error occurs if this is not the case. This escape is a - recent addition to Perl; it provides way of specifying character code - points as octal numbers greater than 0777, and it also allows octal + The escape \o must be followed by a sequence of octal digits, enclosed + in braces. An error occurs if this is not the case. This escape is a + recent addition to Perl; it provides way of specifying character code + points as octal numbers greater than 0777, and it also allows octal numbers and back references to be unambiguously specified. For greater clarity and unambiguity, it is best to avoid following \ by a digit greater than zero. Instead, use \o{} or \x{} to specify charac- - ter numbers, and \g{} to specify back references. The following para- + ter numbers, and \g{} to specify back references. The following para- graphs describe the old, ambiguous syntax. The handling of a backslash followed by a digit other than 0 is compli- - cated, and Perl has changed in recent releases, causing PCRE also to + cated, and Perl has changed in recent releases, causing PCRE also to change. Outside a character class, PCRE reads the digit and any follow- - ing digits as a decimal number. If the number is less than 8, or if - there have been at least that many previous capturing left parentheses - in the expression, the entire sequence is taken as a back reference. A - description of how this works is given later, following the discussion + ing digits as a decimal number. If the number is less than 8, or if + there have been at least that many previous capturing left parentheses + in the expression, the entire sequence is taken as a back reference. A + description of how this works is given later, following the discussion of parenthesized subpatterns. - Inside a character class, or if the decimal number following \ is + Inside a character class, or if the decimal number following \ is greater than 7 and there have not been that many capturing subpatterns, - PCRE handles \8 and \9 as the literal characters "8" and "9", and oth- + PCRE handles \8 and \9 as the literal characters "8" and "9", and oth- erwise re-reads up to three octal digits following the backslash, using - them to generate a data character. Any subsequent digits stand for + them to generate a data character. Any subsequent digits stand for themselves. For example: \040 is another way of writing an ASCII space @@ -5094,31 +5095,31 @@ BACKSLASH \81 is either a back reference, or the two characters "8" and "1" - Note that octal values of 100 or greater that are specified using this - syntax must not be introduced by a leading zero, because no more than + Note that octal values of 100 or greater that are specified using this + syntax must not be introduced by a leading zero, because no more than three octal digits are ever read. - By default, after \x that is not followed by {, from zero to two hexa- - decimal digits are read (letters can be in upper or lower case). Any + By default, after \x that is not followed by {, from zero to two hexa- + decimal digits are read (letters can be in upper or lower case). Any number of hexadecimal digits may appear between \x{ and }. If a charac- - ter other than a hexadecimal digit appears between \x{ and }, or if + ter other than a hexadecimal digit appears between \x{ and }, or if there is no terminating }, an error occurs. - If the PCRE_JAVASCRIPT_COMPAT option is set, the interpretation of \x - is as just described only when it is followed by two hexadecimal dig- - its. Otherwise, it matches a literal "x" character. In JavaScript + If the PCRE_JAVASCRIPT_COMPAT option is set, the interpretation of \x + is as just described only when it is followed by two hexadecimal dig- + its. Otherwise, it matches a literal "x" character. In JavaScript mode, support for code points greater than 256 is provided by \u, which - must be followed by four hexadecimal digits; otherwise it matches a + must be followed by four hexadecimal digits; otherwise it matches a literal "u" character. Characters whose value is less than 256 can be defined by either of the - two syntaxes for \x (or by \u in JavaScript mode). There is no differ- + two syntaxes for \x (or by \u in JavaScript mode). There is no differ- ence in the way they are handled. For example, \xdc is exactly the same as \x{dc} (or \u00dc in JavaScript mode). Constraints on character values - Characters that are specified using octal or hexadecimal numbers are + Characters that are specified using octal or hexadecimal numbers are limited to certain values, as follows: 8-bit non-UTF mode less than 0x100 @@ -5128,44 +5129,44 @@ BACKSLASH 32-bit non-UTF mode less than 0x100000000 32-bit UTF-32 mode less than 0x10ffff and a valid codepoint - Invalid Unicode codepoints are the range 0xd800 to 0xdfff (the so- + Invalid Unicode codepoints are the range 0xd800 to 0xdfff (the so- called "surrogate" codepoints), and 0xffef. Escape sequences in character classes All the sequences that define a single character value can be used both - inside and outside character classes. In addition, inside a character + inside and outside character classes. In addition, inside a character class, \b is interpreted as the backspace character (hex 08). - \N is not allowed in a character class. \B, \R, and \X are not special - inside a character class. Like other unrecognized escape sequences, - they are treated as the literal characters "B", "R", and "X" by - default, but cause an error if the PCRE_EXTRA option is set. Outside a + \N is not allowed in a character class. \B, \R, and \X are not special + inside a character class. Like other unrecognized escape sequences, + they are treated as the literal characters "B", "R", and "X" by + default, but cause an error if the PCRE_EXTRA option is set. Outside a character class, these sequences have different meanings. Unsupported escape sequences - In Perl, the sequences \l, \L, \u, and \U are recognized by its string - handler and used to modify the case of following characters. By - default, PCRE does not support these escape sequences. However, if the - PCRE_JAVASCRIPT_COMPAT option is set, \U matches a "U" character, and + In Perl, the sequences \l, \L, \u, and \U are recognized by its string + handler and used to modify the case of following characters. By + default, PCRE does not support these escape sequences. However, if the + PCRE_JAVASCRIPT_COMPAT option is set, \U matches a "U" character, and \u can be used to define a character by code point, as described in the previous section. Absolute and relative back references - The sequence \g followed by an unsigned or a negative number, option- - ally enclosed in braces, is an absolute or relative back reference. A + The sequence \g followed by an unsigned or a negative number, option- + ally enclosed in braces, is an absolute or relative back reference. A named back reference can be coded as \g{name}. Back references are dis- cussed later, following the discussion of parenthesized subpatterns. Absolute and relative subroutine calls - For compatibility with Oniguruma, the non-Perl syntax \g followed by a + For compatibility with Oniguruma, the non-Perl syntax \g followed by a name or a number enclosed either in angle brackets or single quotes, is - an alternative syntax for referencing a subpattern as a "subroutine". - Details are discussed later. Note that \g{...} (Perl syntax) and - \g<...> (Oniguruma syntax) are not synonymous. The former is a back + an alternative syntax for referencing a subpattern as a "subroutine". + Details are discussed later. Note that \g{...} (Perl syntax) and + \g<...> (Oniguruma syntax) are not synonymous. The former is a back reference; the latter is a subroutine call. Generic character types @@ -5184,59 +5185,59 @@ BACKSLASH \W any "non-word" character There is also the single sequence \N, which matches a non-newline char- - acter. This is the same as the "." metacharacter when PCRE_DOTALL is - not set. Perl also uses \N to match characters by name; PCRE does not + acter. This is the same as the "." metacharacter when PCRE_DOTALL is + not set. Perl also uses \N to match characters by name; PCRE does not support this. - Each pair of lower and upper case escape sequences partitions the com- - plete set of characters into two disjoint sets. Any given character - matches one, and only one, of each pair. The sequences can appear both - inside and outside character classes. They each match one character of - the appropriate type. If the current matching point is at the end of - the subject string, all of them fail, because there is no character to + Each pair of lower and upper case escape sequences partitions the com- + plete set of characters into two disjoint sets. Any given character + matches one, and only one, of each pair. The sequences can appear both + inside and outside character classes. They each match one character of + the appropriate type. If the current matching point is at the end of + the subject string, all of them fail, because there is no character to match. - For compatibility with Perl, \s did not used to match the VT character - (code 11), which made it different from the the POSIX "space" class. - However, Perl added VT at release 5.18, and PCRE followed suit at - release 8.34. The default \s characters are now HT (9), LF (10), VT - (11), FF (12), CR (13), and space (32), which are defined as white + For compatibility with Perl, \s did not used to match the VT character + (code 11), which made it different from the the POSIX "space" class. + However, Perl added VT at release 5.18, and PCRE followed suit at + release 8.34. The default \s characters are now HT (9), LF (10), VT + (11), FF (12), CR (13), and space (32), which are defined as white space in the "C" locale. This list may vary if locale-specific matching - is taking place. For example, in some locales the "non-breaking space" - character (\xA0) is recognized as white space, and in others the VT + is taking place. For example, in some locales the "non-breaking space" + character (\xA0) is recognized as white space, and in others the VT character is not. - A "word" character is an underscore or any character that is a letter - or digit. By default, the definition of letters and digits is con- - trolled by PCRE's low-valued character tables, and may vary if locale- - specific matching is taking place (see "Locale support" in the pcreapi - page). For example, in a French locale such as "fr_FR" in Unix-like - systems, or "french" in Windows, some character codes greater than 127 - are used for accented letters, and these are then matched by \w. The + A "word" character is an underscore or any character that is a letter + or digit. By default, the definition of letters and digits is con- + trolled by PCRE's low-valued character tables, and may vary if locale- + specific matching is taking place (see "Locale support" in the pcreapi + page). For example, in a French locale such as "fr_FR" in Unix-like + systems, or "french" in Windows, some character codes greater than 127 + are used for accented letters, and these are then matched by \w. The use of locales with Unicode is discouraged. - By default, characters whose code points are greater than 127 never + By default, characters whose code points are greater than 127 never match \d, \s, or \w, and always match \D, \S, and \W, although this may - vary for characters in the range 128-255 when locale-specific matching - is happening. These escape sequences retain their original meanings - from before Unicode support was available, mainly for efficiency rea- - sons. If PCRE is compiled with Unicode property support, and the - PCRE_UCP option is set, the behaviour is changed so that Unicode prop- + vary for characters in the range 128-255 when locale-specific matching + is happening. These escape sequences retain their original meanings + from before Unicode support was available, mainly for efficiency rea- + sons. If PCRE is compiled with Unicode property support, and the + PCRE_UCP option is set, the behaviour is changed so that Unicode prop- erties are used to determine character types, as follows: \d any character that matches \p{Nd} (decimal digit) \s any character that matches \p{Z} or \h or \v \w any character that matches \p{L} or \p{N}, plus underscore - The upper case escapes match the inverse sets of characters. Note that - \d matches only decimal digits, whereas \w matches any Unicode digit, - as well as any Unicode letter, and underscore. Note also that PCRE_UCP - affects \b, and \B because they are defined in terms of \w and \W. + The upper case escapes match the inverse sets of characters. Note that + \d matches only decimal digits, whereas \w matches any Unicode digit, + as well as any Unicode letter, and underscore. Note also that PCRE_UCP + affects \b, and \B because they are defined in terms of \w and \W. Matching these sequences is noticeably slower when PCRE_UCP is set. - The sequences \h, \H, \v, and \V are features that were added to Perl - at release 5.10. In contrast to the other sequences, which match only - ASCII characters by default, these always match certain high-valued + The sequences \h, \H, \v, and \V are features that were added to Perl + at release 5.10. In contrast to the other sequences, which match only + ASCII characters by default, these always match certain high-valued code points, whether or not PCRE_UCP is set. The horizontal space char- acters are: @@ -5275,110 +5276,110 @@ BACKSLASH Newline sequences - Outside a character class, by default, the escape sequence \R matches - any Unicode newline sequence. In 8-bit non-UTF-8 mode \R is equivalent + Outside a character class, by default, the escape sequence \R matches + any Unicode newline sequence. In 8-bit non-UTF-8 mode \R is equivalent to the following: (?>\r\n|\n|\x0b|\f|\r|\x85) - This is an example of an "atomic group", details of which are given + This is an example of an "atomic group", details of which are given below. This particular group matches either the two-character sequence - CR followed by LF, or one of the single characters LF (linefeed, - U+000A), VT (vertical tab, U+000B), FF (form feed, U+000C), CR (car- - riage return, U+000D), or NEL (next line, U+0085). The two-character + CR followed by LF, or one of the single characters LF (linefeed, + U+000A), VT (vertical tab, U+000B), FF (form feed, U+000C), CR (car- + riage return, U+000D), or NEL (next line, U+0085). The two-character sequence is treated as a single unit that cannot be split. - In other modes, two additional characters whose codepoints are greater + In other modes, two additional characters whose codepoints are greater than 255 are added: LS (line separator, U+2028) and PS (paragraph sepa- - rator, U+2029). Unicode character property support is not needed for + rator, U+2029). Unicode character property support is not needed for these characters to be recognized. It is possible to restrict \R to match only CR, LF, or CRLF (instead of - the complete set of Unicode line endings) by setting the option + the complete set of Unicode line endings) by setting the option PCRE_BSR_ANYCRLF either at compile time or when the pattern is matched. (BSR is an abbrevation for "backslash R".) This can be made the default - when PCRE is built; if this is the case, the other behaviour can be - requested via the PCRE_BSR_UNICODE option. It is also possible to - specify these settings by starting a pattern string with one of the + when PCRE is built; if this is the case, the other behaviour can be + requested via the PCRE_BSR_UNICODE option. It is also possible to + specify these settings by starting a pattern string with one of the following sequences: (*BSR_ANYCRLF) CR, LF, or CRLF only (*BSR_UNICODE) any Unicode newline sequence These override the default and the options given to the compiling func- - tion, but they can themselves be overridden by options given to a - matching function. Note that these special settings, which are not - Perl-compatible, are recognized only at the very start of a pattern, - and that they must be in upper case. If more than one of them is - present, the last one is used. They can be combined with a change of + tion, but they can themselves be overridden by options given to a + matching function. Note that these special settings, which are not + Perl-compatible, are recognized only at the very start of a pattern, + and that they must be in upper case. If more than one of them is + present, the last one is used. They can be combined with a change of newline convention; for example, a pattern can start with: (*ANY)(*BSR_ANYCRLF) - They can also be combined with the (*UTF8), (*UTF16), (*UTF32), (*UTF) + They can also be combined with the (*UTF8), (*UTF16), (*UTF32), (*UTF) or (*UCP) special sequences. Inside a character class, \R is treated as - an unrecognized escape sequence, and so matches the letter "R" by + an unrecognized escape sequence, and so matches the letter "R" by default, but causes an error if PCRE_EXTRA is set. Unicode character properties When PCRE is built with Unicode character property support, three addi- - tional escape sequences that match characters with specific properties - are available. When in 8-bit non-UTF-8 mode, these sequences are of - course limited to testing characters whose codepoints are less than + tional escape sequences that match characters with specific properties + are available. When in 8-bit non-UTF-8 mode, these sequences are of + course limited to testing characters whose codepoints are less than 256, but they do work in this mode. The extra escape sequences are: \p{xx} a character with the xx property \P{xx} a character without the xx property \X a Unicode extended grapheme cluster - The property names represented by xx above are limited to the Unicode + The property names represented by xx above are limited to the Unicode script names, the general category properties, "Any", which matches any - character (including newline), and some special PCRE properties - (described in the next section). Other Perl properties such as "InMu- - sicalSymbols" are not currently supported by PCRE. Note that \P{Any} + character (including newline), and some special PCRE properties + (described in the next section). Other Perl properties such as "InMu- + sicalSymbols" are not currently supported by PCRE. Note that \P{Any} does not match any characters, so always causes a match failure. Sets of Unicode characters are defined as belonging to certain scripts. - A character from one of these sets can be matched using a script name. + A character from one of these sets can be matched using a script name. For example: \p{Greek} \P{Han} - Those that are not part of an identified script are lumped together as + Those that are not part of an identified script are lumped together as "Common". The current list of scripts is: - Arabic, Armenian, Avestan, Balinese, Bamum, Bassa_Vah, Batak, Bengali, - Bopomofo, Brahmi, Braille, Buginese, Buhid, Canadian_Aboriginal, Car- + Arabic, Armenian, Avestan, Balinese, Bamum, Bassa_Vah, Batak, Bengali, + Bopomofo, Brahmi, Braille, Buginese, Buhid, Canadian_Aboriginal, Car- ian, Caucasian_Albanian, Chakma, Cham, Cherokee, Common, Coptic, Cunei- form, Cypriot, Cyrillic, Deseret, Devanagari, Duployan, Egyptian_Hiero- glyphs, Elbasan, Ethiopic, Georgian, Glagolitic, Gothic, Grantha, - Greek, Gujarati, Gurmukhi, Han, Hangul, Hanunoo, Hebrew, Hiragana, - Imperial_Aramaic, Inherited, Inscriptional_Pahlavi, Inscrip- - tional_Parthian, Javanese, Kaithi, Kannada, Katakana, Kayah_Li, - Kharoshthi, Khmer, Khojki, Khudawadi, Lao, Latin, Lepcha, Limbu, Lin- - ear_A, Linear_B, Lisu, Lycian, Lydian, Mahajani, Malayalam, Mandaic, - Manichaean, Meetei_Mayek, Mende_Kikakui, Meroitic_Cursive, - Meroitic_Hieroglyphs, Miao, Modi, Mongolian, Mro, Myanmar, Nabataean, - New_Tai_Lue, Nko, Ogham, Ol_Chiki, Old_Italic, Old_North_Arabian, + Greek, Gujarati, Gurmukhi, Han, Hangul, Hanunoo, Hebrew, Hiragana, + Imperial_Aramaic, Inherited, Inscriptional_Pahlavi, Inscrip- + tional_Parthian, Javanese, Kaithi, Kannada, Katakana, Kayah_Li, + Kharoshthi, Khmer, Khojki, Khudawadi, Lao, Latin, Lepcha, Limbu, Lin- + ear_A, Linear_B, Lisu, Lycian, Lydian, Mahajani, Malayalam, Mandaic, + Manichaean, Meetei_Mayek, Mende_Kikakui, Meroitic_Cursive, + Meroitic_Hieroglyphs, Miao, Modi, Mongolian, Mro, Myanmar, Nabataean, + New_Tai_Lue, Nko, Ogham, Ol_Chiki, Old_Italic, Old_North_Arabian, Old_Permic, Old_Persian, Old_South_Arabian, Old_Turkic, Oriya, Osmanya, Pahawh_Hmong, Palmyrene, Pau_Cin_Hau, Phags_Pa, Phoenician, - Psalter_Pahlavi, Rejang, Runic, Samaritan, Saurashtra, Sharada, Sha- - vian, Siddham, Sinhala, Sora_Sompeng, Sundanese, Syloti_Nagri, Syriac, - Tagalog, Tagbanwa, Tai_Le, Tai_Tham, Tai_Viet, Takri, Tamil, Telugu, - Thaana, Thai, Tibetan, Tifinagh, Tirhuta, Ugaritic, Vai, Warang_Citi, + Psalter_Pahlavi, Rejang, Runic, Samaritan, Saurashtra, Sharada, Sha- + vian, Siddham, Sinhala, Sora_Sompeng, Sundanese, Syloti_Nagri, Syriac, + Tagalog, Tagbanwa, Tai_Le, Tai_Tham, Tai_Viet, Takri, Tamil, Telugu, + Thaana, Thai, Tibetan, Tifinagh, Tirhuta, Ugaritic, Vai, Warang_Citi, Yi. Each character has exactly one Unicode general category property, spec- - ified by a two-letter abbreviation. For compatibility with Perl, nega- - tion can be specified by including a circumflex between the opening - brace and the property name. For example, \p{^Lu} is the same as + ified by a two-letter abbreviation. For compatibility with Perl, nega- + tion can be specified by including a circumflex between the opening + brace and the property name. For example, \p{^Lu} is the same as \P{Lu}. If only one letter is specified with \p or \P, it includes all the gen- - eral category properties that start with that letter. In this case, in - the absence of negation, the curly brackets in the escape sequence are + eral category properties that start with that letter. In this case, in + the absence of negation, the curly brackets in the escape sequence are optional; these two examples have the same effect: \p{L} @@ -5430,73 +5431,73 @@ BACKSLASH Zp Paragraph separator Zs Space separator - The special property L& is also supported: it matches a character that - has the Lu, Ll, or Lt property, in other words, a letter that is not + The special property L& is also supported: it matches a character that + has the Lu, Ll, or Lt property, in other words, a letter that is not classified as a modifier or "other". - The Cs (Surrogate) property applies only to characters in the range - U+D800 to U+DFFF. Such characters are not valid in Unicode strings and - so cannot be tested by PCRE, unless UTF validity checking has been + The Cs (Surrogate) property applies only to characters in the range + U+D800 to U+DFFF. Such characters are not valid in Unicode strings and + so cannot be tested by PCRE, unless UTF validity checking has been turned off (see the discussion of PCRE_NO_UTF8_CHECK, - PCRE_NO_UTF16_CHECK and PCRE_NO_UTF32_CHECK in the pcreapi page). Perl + PCRE_NO_UTF16_CHECK and PCRE_NO_UTF32_CHECK in the pcreapi page). Perl does not support the Cs property. - The long synonyms for property names that Perl supports (such as - \p{Letter}) are not supported by PCRE, nor is it permitted to prefix + The long synonyms for property names that Perl supports (such as + \p{Letter}) are not supported by PCRE, nor is it permitted to prefix any of these properties with "Is". No character that is in the Unicode table has the Cn (unassigned) prop- erty. Instead, this property is assumed for any code point that is not in the Unicode table. - Specifying caseless matching does not affect these escape sequences. - For example, \p{Lu} always matches only upper case letters. This is + Specifying caseless matching does not affect these escape sequences. + For example, \p{Lu} always matches only upper case letters. This is different from the behaviour of current versions of Perl. - Matching characters by Unicode property is not fast, because PCRE has - to do a multistage table lookup in order to find a character's prop- + Matching characters by Unicode property is not fast, because PCRE has + to do a multistage table lookup in order to find a character's prop- erty. That is why the traditional escape sequences such as \d and \w do not use Unicode properties in PCRE by default, though you can make them - do so by setting the PCRE_UCP option or by starting the pattern with + do so by setting the PCRE_UCP option or by starting the pattern with (*UCP). Extended grapheme clusters - The \X escape matches any number of Unicode characters that form an + The \X escape matches any number of Unicode characters that form an "extended grapheme cluster", and treats the sequence as an atomic group - (see below). Up to and including release 8.31, PCRE matched an ear- + (see below). Up to and including release 8.31, PCRE matched an ear- lier, simpler definition that was equivalent to (?>\PM\pM*) - That is, it matched a character without the "mark" property, followed - by zero or more characters with the "mark" property. Characters with - the "mark" property are typically non-spacing accents that affect the + That is, it matched a character without the "mark" property, followed + by zero or more characters with the "mark" property. Characters with + the "mark" property are typically non-spacing accents that affect the preceding character. - This simple definition was extended in Unicode to include more compli- - cated kinds of composite character by giving each character a grapheme - breaking property, and creating rules that use these properties to - define the boundaries of extended grapheme clusters. In releases of + This simple definition was extended in Unicode to include more compli- + cated kinds of composite character by giving each character a grapheme + breaking property, and creating rules that use these properties to + define the boundaries of extended grapheme clusters. In releases of PCRE later than 8.31, \X matches one of these clusters. - \X always matches at least one character. Then it decides whether to + \X always matches at least one character. Then it decides whether to add additional characters according to the following rules for ending a cluster: 1. End at the end of the subject string. - 2. Do not end between CR and LF; otherwise end after any control char- + 2. Do not end between CR and LF; otherwise end after any control char- acter. - 3. Do not break Hangul (a Korean script) syllable sequences. Hangul - characters are of five types: L, V, T, LV, and LVT. An L character may - be followed by an L, V, LV, or LVT character; an LV or V character may + 3. Do not break Hangul (a Korean script) syllable sequences. Hangul + characters are of five types: L, V, T, LV, and LVT. An L character may + be followed by an L, V, LV, or LVT character; an LV or V character may be followed by a V or T character; an LVT or T character may be follwed only by a T character. - 4. Do not end before extending characters or spacing marks. Characters - with the "mark" property always have the "extend" grapheme breaking + 4. Do not end before extending characters or spacing marks. Characters + with the "mark" property always have the "extend" grapheme breaking property. 5. Do not end after prepend characters. @@ -5505,9 +5506,9 @@ BACKSLASH PCRE's additional properties - As well as the standard Unicode properties described above, PCRE sup- - ports four more that make it possible to convert traditional escape - sequences such as \w and \s to use Unicode properties. PCRE uses these + As well as the standard Unicode properties described above, PCRE sup- + ports four more that make it possible to convert traditional escape + sequences such as \w and \s to use Unicode properties. PCRE uses these non-standard, non-Perl properties internally when PCRE_UCP is set. How- ever, they may also be used explicitly. These properties are: @@ -5516,54 +5517,54 @@ BACKSLASH Xsp Any Perl space character Xwd Any Perl "word" character - Xan matches characters that have either the L (letter) or the N (num- - ber) property. Xps matches the characters tab, linefeed, vertical tab, - form feed, or carriage return, and any other character that has the Z - (separator) property. Xsp is the same as Xps; it used to exclude ver- - tical tab, for Perl compatibility, but Perl changed, and so PCRE fol- - lowed at release 8.34. Xwd matches the same characters as Xan, plus + Xan matches characters that have either the L (letter) or the N (num- + ber) property. Xps matches the characters tab, linefeed, vertical tab, + form feed, or carriage return, and any other character that has the Z + (separator) property. Xsp is the same as Xps; it used to exclude ver- + tical tab, for Perl compatibility, but Perl changed, and so PCRE fol- + lowed at release 8.34. Xwd matches the same characters as Xan, plus underscore. - There is another non-standard property, Xuc, which matches any charac- - ter that can be represented by a Universal Character Name in C++ and - other programming languages. These are the characters $, @, ` (grave - accent), and all characters with Unicode code points greater than or - equal to U+00A0, except for the surrogates U+D800 to U+DFFF. Note that - most base (ASCII) characters are excluded. (Universal Character Names - are of the form \uHHHH or \UHHHHHHHH where H is a hexadecimal digit. + There is another non-standard property, Xuc, which matches any charac- + ter that can be represented by a Universal Character Name in C++ and + other programming languages. These are the characters $, @, ` (grave + accent), and all characters with Unicode code points greater than or + equal to U+00A0, except for the surrogates U+D800 to U+DFFF. Note that + most base (ASCII) characters are excluded. (Universal Character Names + are of the form \uHHHH or \UHHHHHHHH where H is a hexadecimal digit. Note that the Xuc property does not match these sequences but the char- acters that they represent.) Resetting the match start - The escape sequence \K causes any previously matched characters not to + The escape sequence \K causes any previously matched characters not to be included in the final matched sequence. For example, the pattern: foo\Kbar - matches "foobar", but reports that it has matched "bar". This feature - is similar to a lookbehind assertion (described below). However, in - this case, the part of the subject before the real match does not have - to be of fixed length, as lookbehind assertions do. The use of \K does - not interfere with the setting of captured substrings. For example, + matches "foobar", but reports that it has matched "bar". This feature + is similar to a lookbehind assertion (described below). However, in + this case, the part of the subject before the real match does not have + to be of fixed length, as lookbehind assertions do. The use of \K does + not interfere with the setting of captured substrings. For example, when the pattern (foo)\Kbar matches "foobar", the first substring is still set to "foo". - Perl documents that the use of \K within assertions is "not well - defined". In PCRE, \K is acted upon when it occurs inside positive - assertions, but is ignored in negative assertions. Note that when a - pattern such as (?=ab\K) matches, the reported start of the match can + Perl documents that the use of \K within assertions is "not well + defined". In PCRE, \K is acted upon when it occurs inside positive + assertions, but is ignored in negative assertions. Note that when a + pattern such as (?=ab\K) matches, the reported start of the match can be greater than the end of the match. Simple assertions - The final use of backslash is for certain simple assertions. An asser- - tion specifies a condition that has to be met at a particular point in - a match, without consuming any characters from the subject string. The - use of subpatterns for more complicated assertions is described below. + The final use of backslash is for certain simple assertions. An asser- + tion specifies a condition that has to be met at a particular point in + a match, without consuming any characters from the subject string. The + use of subpatterns for more complicated assertions is described below. The backslashed assertions are: \b matches at a word boundary @@ -5574,161 +5575,161 @@ BACKSLASH \z matches only at the end of the subject \G matches at the first matching position in the subject - Inside a character class, \b has a different meaning; it matches the - backspace character. If any other of these assertions appears in a - character class, by default it matches the corresponding literal char- + Inside a character class, \b has a different meaning; it matches the + backspace character. If any other of these assertions appears in a + character class, by default it matches the corresponding literal char- acter (for example, \B matches the letter B). However, if the - PCRE_EXTRA option is set, an "invalid escape sequence" error is gener- + PCRE_EXTRA option is set, an "invalid escape sequence" error is gener- ated instead. - A word boundary is a position in the subject string where the current - character and the previous character do not both match \w or \W (i.e. - one matches \w and the other matches \W), or the start or end of the - string if the first or last character matches \w, respectively. In a - UTF mode, the meanings of \w and \W can be changed by setting the - PCRE_UCP option. When this is done, it also affects \b and \B. Neither - PCRE nor Perl has a separate "start of word" or "end of word" metase- - quence. However, whatever follows \b normally determines which it is. + A word boundary is a position in the subject string where the current + character and the previous character do not both match \w or \W (i.e. + one matches \w and the other matches \W), or the start or end of the + string if the first or last character matches \w, respectively. In a + UTF mode, the meanings of \w and \W can be changed by setting the + PCRE_UCP option. When this is done, it also affects \b and \B. Neither + PCRE nor Perl has a separate "start of word" or "end of word" metase- + quence. However, whatever follows \b normally determines which it is. For example, the fragment \ba matches "a" at the start of a word. - The \A, \Z, and \z assertions differ from the traditional circumflex + The \A, \Z, and \z assertions differ from the traditional circumflex and dollar (described in the next section) in that they only ever match - at the very start and end of the subject string, whatever options are - set. Thus, they are independent of multiline mode. These three asser- + at the very start and end of the subject string, whatever options are + set. Thus, they are independent of multiline mode. These three asser- tions are not affected by the PCRE_NOTBOL or PCRE_NOTEOL options, which - affect only the behaviour of the circumflex and dollar metacharacters. - However, if the startoffset argument of pcre_exec() is non-zero, indi- + affect only the behaviour of the circumflex and dollar metacharacters. + However, if the startoffset argument of pcre_exec() is non-zero, indi- cating that matching is to start at a point other than the beginning of - the subject, \A can never match. The difference between \Z and \z is + the subject, \A can never match. The difference between \Z and \z is that \Z matches before a newline at the end of the string as well as at the very end, whereas \z matches only at the end. - The \G assertion is true only when the current matching position is at - the start point of the match, as specified by the startoffset argument - of pcre_exec(). It differs from \A when the value of startoffset is - non-zero. By calling pcre_exec() multiple times with appropriate argu- + The \G assertion is true only when the current matching position is at + the start point of the match, as specified by the startoffset argument + of pcre_exec(). It differs from \A when the value of startoffset is + non-zero. By calling pcre_exec() multiple times with appropriate argu- ments, you can mimic Perl's /g option, and it is in this kind of imple- mentation where \G can be useful. - Note, however, that PCRE's interpretation of \G, as the start of the + Note, however, that PCRE's interpretation of \G, as the start of the current match, is subtly different from Perl's, which defines it as the - end of the previous match. In Perl, these can be different when the - previously matched string was empty. Because PCRE does just one match + end of the previous match. In Perl, these can be different when the + previously matched string was empty. Because PCRE does just one match at a time, it cannot reproduce this behaviour. - If all the alternatives of a pattern begin with \G, the expression is + If all the alternatives of a pattern begin with \G, the expression is anchored to the starting match position, and the "anchored" flag is set in the compiled regular expression. CIRCUMFLEX AND DOLLAR - The circumflex and dollar metacharacters are zero-width assertions. - That is, they test for a particular condition being true without con- + The circumflex and dollar metacharacters are zero-width assertions. + That is, they test for a particular condition being true without con- suming any characters from the subject string. Outside a character class, in the default matching mode, the circumflex - character is an assertion that is true only if the current matching - point is at the start of the subject string. If the startoffset argu- - ment of pcre_exec() is non-zero, circumflex can never match if the - PCRE_MULTILINE option is unset. Inside a character class, circumflex + character is an assertion that is true only if the current matching + point is at the start of the subject string. If the startoffset argu- + ment of pcre_exec() is non-zero, circumflex can never match if the + PCRE_MULTILINE option is unset. Inside a character class, circumflex has an entirely different meaning (see below). - Circumflex need not be the first character of the pattern if a number - of alternatives are involved, but it should be the first thing in each - alternative in which it appears if the pattern is ever to match that - branch. If all possible alternatives start with a circumflex, that is, - if the pattern is constrained to match only at the start of the sub- - ject, it is said to be an "anchored" pattern. (There are also other + Circumflex need not be the first character of the pattern if a number + of alternatives are involved, but it should be the first thing in each + alternative in which it appears if the pattern is ever to match that + branch. If all possible alternatives start with a circumflex, that is, + if the pattern is constrained to match only at the start of the sub- + ject, it is said to be an "anchored" pattern. (There are also other constructs that can cause a pattern to be anchored.) - The dollar character is an assertion that is true only if the current - matching point is at the end of the subject string, or immediately - before a newline at the end of the string (by default). Note, however, - that it does not actually match the newline. Dollar need not be the + The dollar character is an assertion that is true only if the current + matching point is at the end of the subject string, or immediately + before a newline at the end of the string (by default). Note, however, + that it does not actually match the newline. Dollar need not be the last character of the pattern if a number of alternatives are involved, - but it should be the last item in any branch in which it appears. Dol- + but it should be the last item in any branch in which it appears. Dol- lar has no special meaning in a character class. - The meaning of dollar can be changed so that it matches only at the - very end of the string, by setting the PCRE_DOLLAR_ENDONLY option at + The meaning of dollar can be changed so that it matches only at the + very end of the string, by setting the PCRE_DOLLAR_ENDONLY option at compile time. This does not affect the \Z assertion. The meanings of the circumflex and dollar characters are changed if the - PCRE_MULTILINE option is set. When this is the case, a circumflex - matches immediately after internal newlines as well as at the start of - the subject string. It does not match after a newline that ends the - string. A dollar matches before any newlines in the string, as well as - at the very end, when PCRE_MULTILINE is set. When newline is specified - as the two-character sequence CRLF, isolated CR and LF characters do + PCRE_MULTILINE option is set. When this is the case, a circumflex + matches immediately after internal newlines as well as at the start of + the subject string. It does not match after a newline that ends the + string. A dollar matches before any newlines in the string, as well as + at the very end, when PCRE_MULTILINE is set. When newline is specified + as the two-character sequence CRLF, isolated CR and LF characters do not indicate newlines. - For example, the pattern /^abc$/ matches the subject string "def\nabc" - (where \n represents a newline) in multiline mode, but not otherwise. - Consequently, patterns that are anchored in single line mode because - all branches start with ^ are not anchored in multiline mode, and a - match for circumflex is possible when the startoffset argument of - pcre_exec() is non-zero. The PCRE_DOLLAR_ENDONLY option is ignored if + For example, the pattern /^abc$/ matches the subject string "def\nabc" + (where \n represents a newline) in multiline mode, but not otherwise. + Consequently, patterns that are anchored in single line mode because + all branches start with ^ are not anchored in multiline mode, and a + match for circumflex is possible when the startoffset argument of + pcre_exec() is non-zero. The PCRE_DOLLAR_ENDONLY option is ignored if PCRE_MULTILINE is set. - Note that the sequences \A, \Z, and \z can be used to match the start - and end of the subject in both modes, and if all branches of a pattern - start with \A it is always anchored, whether or not PCRE_MULTILINE is + Note that the sequences \A, \Z, and \z can be used to match the start + and end of the subject in both modes, and if all branches of a pattern + start with \A it is always anchored, whether or not PCRE_MULTILINE is set. FULL STOP (PERIOD, DOT) AND \N Outside a character class, a dot in the pattern matches any one charac- - ter in the subject string except (by default) a character that signi- + ter in the subject string except (by default) a character that signi- fies the end of a line. - When a line ending is defined as a single character, dot never matches - that character; when the two-character sequence CRLF is used, dot does - not match CR if it is immediately followed by LF, but otherwise it - matches all characters (including isolated CRs and LFs). When any Uni- - code line endings are being recognized, dot does not match CR or LF or + When a line ending is defined as a single character, dot never matches + that character; when the two-character sequence CRLF is used, dot does + not match CR if it is immediately followed by LF, but otherwise it + matches all characters (including isolated CRs and LFs). When any Uni- + code line endings are being recognized, dot does not match CR or LF or any of the other line ending characters. - The behaviour of dot with regard to newlines can be changed. If the - PCRE_DOTALL option is set, a dot matches any one character, without + The behaviour of dot with regard to newlines can be changed. If the + PCRE_DOTALL option is set, a dot matches any one character, without exception. If the two-character sequence CRLF is present in the subject string, it takes two dots to match it. - The handling of dot is entirely independent of the handling of circum- - flex and dollar, the only relationship being that they both involve + The handling of dot is entirely independent of the handling of circum- + flex and dollar, the only relationship being that they both involve newlines. Dot has no special meaning in a character class. - The escape sequence \N behaves like a dot, except that it is not - affected by the PCRE_DOTALL option. In other words, it matches any - character except one that signifies the end of a line. Perl also uses + The escape sequence \N behaves like a dot, except that it is not + affected by the PCRE_DOTALL option. In other words, it matches any + character except one that signifies the end of a line. Perl also uses \N to match characters by name; PCRE does not support this. MATCHING A SINGLE DATA UNIT - Outside a character class, the escape sequence \C matches any one data - unit, whether or not a UTF mode is set. In the 8-bit library, one data - unit is one byte; in the 16-bit library it is a 16-bit unit; in the - 32-bit library it is a 32-bit unit. Unlike a dot, \C always matches - line-ending characters. The feature is provided in Perl in order to + Outside a character class, the escape sequence \C matches any one data + unit, whether or not a UTF mode is set. In the 8-bit library, one data + unit is one byte; in the 16-bit library it is a 16-bit unit; in the + 32-bit library it is a 32-bit unit. Unlike a dot, \C always matches + line-ending characters. The feature is provided in Perl in order to match individual bytes in UTF-8 mode, but it is unclear how it can use- - fully be used. Because \C breaks up characters into individual data - units, matching one unit with \C in a UTF mode means that the rest of + fully be used. Because \C breaks up characters into individual data + units, matching one unit with \C in a UTF mode means that the rest of the string may start with a malformed UTF character. This has undefined results, because PCRE assumes that it is dealing with valid UTF strings - (and by default it checks this at the start of processing unless the - PCRE_NO_UTF8_CHECK, PCRE_NO_UTF16_CHECK or PCRE_NO_UTF32_CHECK option + (and by default it checks this at the start of processing unless the + PCRE_NO_UTF8_CHECK, PCRE_NO_UTF16_CHECK or PCRE_NO_UTF32_CHECK option is used). - PCRE does not allow \C to appear in lookbehind assertions (described - below) in a UTF mode, because this would make it impossible to calcu- + PCRE does not allow \C to appear in lookbehind assertions (described + below) in a UTF mode, because this would make it impossible to calcu- late the length of the lookbehind. In general, the \C escape sequence is best avoided. However, one way of - using it that avoids the problem of malformed UTF characters is to use - a lookahead to check the length of the next character, as in this pat- - tern, which could be used with a UTF-8 string (ignore white space and + using it that avoids the problem of malformed UTF characters is to use + a lookahead to check the length of the next character, as in this pat- + tern, which could be used with a UTF-8 string (ignore white space and line breaks): (?| (?=[\x00-\x7f])(\C) | @@ -5736,11 +5737,11 @@ MATCHING A SINGLE DATA UNIT (?=[\x{800}-\x{ffff}])(\C)(\C)(\C) | (?=[\x{10000}-\x{1fffff}])(\C)(\C)(\C)(\C)) - A group that starts with (?| resets the capturing parentheses numbers - in each alternative (see "Duplicate Subpattern Numbers" below). The - assertions at the start of each branch check the next UTF-8 character - for values whose encoding uses 1, 2, 3, or 4 bytes, respectively. The - character's individual bytes are then captured by the appropriate num- + A group that starts with (?| resets the capturing parentheses numbers + in each alternative (see "Duplicate Subpattern Numbers" below). The + assertions at the start of each branch check the next UTF-8 character + for values whose encoding uses 1, 2, 3, or 4 bytes, respectively. The + character's individual bytes are then captured by the appropriate num- ber of groups. @@ -5750,109 +5751,109 @@ SQUARE BRACKETS AND CHARACTER CLASSES closing square bracket. A closing square bracket on its own is not spe- cial by default. However, if the PCRE_JAVASCRIPT_COMPAT option is set, a lone closing square bracket causes a compile-time error. If a closing - square bracket is required as a member of the class, it should be the - first data character in the class (after an initial circumflex, if + square bracket is required as a member of the class, it should be the + first data character in the class (after an initial circumflex, if present) or escaped with a backslash. - A character class matches a single character in the subject. In a UTF - mode, the character may be more than one data unit long. A matched + A character class matches a single character in the subject. In a UTF + mode, the character may be more than one data unit long. A matched character must be in the set of characters defined by the class, unless - the first character in the class definition is a circumflex, in which + the first character in the class definition is a circumflex, in which case the subject character must not be in the set defined by the class. - If a circumflex is actually required as a member of the class, ensure + If a circumflex is actually required as a member of the class, ensure it is not the first character, or escape it with a backslash. - For example, the character class [aeiou] matches any lower case vowel, - while [^aeiou] matches any character that is not a lower case vowel. + For example, the character class [aeiou] matches any lower case vowel, + while [^aeiou] matches any character that is not a lower case vowel. Note that a circumflex is just a convenient notation for specifying the - characters that are in the class by enumerating those that are not. A - class that starts with a circumflex is not an assertion; it still con- - sumes a character from the subject string, and therefore it fails if + characters that are in the class by enumerating those that are not. A + class that starts with a circumflex is not an assertion; it still con- + sumes a character from the subject string, and therefore it fails if the current pointer is at the end of the string. In UTF-8 (UTF-16, UTF-32) mode, characters with values greater than 255 - (0xffff) can be included in a class as a literal string of data units, + (0xffff) can be included in a class as a literal string of data units, or by using the \x{ escaping mechanism. - When caseless matching is set, any letters in a class represent both - their upper case and lower case versions, so for example, a caseless - [aeiou] matches "A" as well as "a", and a caseless [^aeiou] does not - match "A", whereas a caseful version would. In a UTF mode, PCRE always - understands the concept of case for characters whose values are less - than 128, so caseless matching is always possible. For characters with - higher values, the concept of case is supported if PCRE is compiled - with Unicode property support, but not otherwise. If you want to use - caseless matching in a UTF mode for characters 128 and above, you must - ensure that PCRE is compiled with Unicode property support as well as + When caseless matching is set, any letters in a class represent both + their upper case and lower case versions, so for example, a caseless + [aeiou] matches "A" as well as "a", and a caseless [^aeiou] does not + match "A", whereas a caseful version would. In a UTF mode, PCRE always + understands the concept of case for characters whose values are less + than 128, so caseless matching is always possible. For characters with + higher values, the concept of case is supported if PCRE is compiled + with Unicode property support, but not otherwise. If you want to use + caseless matching in a UTF mode for characters 128 and above, you must + ensure that PCRE is compiled with Unicode property support as well as with UTF support. - Characters that might indicate line breaks are never treated in any - special way when matching character classes, whatever line-ending - sequence is in use, and whatever setting of the PCRE_DOTALL and + Characters that might indicate line breaks are never treated in any + special way when matching character classes, whatever line-ending + sequence is in use, and whatever setting of the PCRE_DOTALL and PCRE_MULTILINE options is used. A class such as [^a] always matches one of these characters. - The minus (hyphen) character can be used to specify a range of charac- - ters in a character class. For example, [d-m] matches any letter - between d and m, inclusive. If a minus character is required in a - class, it must be escaped with a backslash or appear in a position - where it cannot be interpreted as indicating a range, typically as the + The minus (hyphen) character can be used to specify a range of charac- + ters in a character class. For example, [d-m] matches any letter + between d and m, inclusive. If a minus character is required in a + class, it must be escaped with a backslash or appear in a position + where it cannot be interpreted as indicating a range, typically as the first or last character in the class, or immediately after a range. For - example, [b-d-z] matches letters in the range b to d, a hyphen charac- + example, [b-d-z] matches letters in the range b to d, a hyphen charac- ter, or z. It is not possible to have the literal character "]" as the end charac- - ter of a range. A pattern such as [W-]46] is interpreted as a class of - two characters ("W" and "-") followed by a literal string "46]", so it - would match "W46]" or "-46]". However, if the "]" is escaped with a - backslash it is interpreted as the end of range, so [W-\]46] is inter- - preted as a class containing a range followed by two other characters. - The octal or hexadecimal representation of "]" can also be used to end + ter of a range. A pattern such as [W-]46] is interpreted as a class of + two characters ("W" and "-") followed by a literal string "46]", so it + would match "W46]" or "-46]". However, if the "]" is escaped with a + backslash it is interpreted as the end of range, so [W-\]46] is inter- + preted as a class containing a range followed by two other characters. + The octal or hexadecimal representation of "]" can also be used to end a range. - An error is generated if a POSIX character class (see below) or an - escape sequence other than one that defines a single character appears - at a point where a range ending character is expected. For example, + An error is generated if a POSIX character class (see below) or an + escape sequence other than one that defines a single character appears + at a point where a range ending character is expected. For example, [z-\xff] is valid, but [A-\d] and [A-[:digit:]] are not. - Ranges operate in the collating sequence of character values. They can - also be used for characters specified numerically, for example - [\000-\037]. Ranges can include any characters that are valid for the + Ranges operate in the collating sequence of character values. They can + also be used for characters specified numerically, for example + [\000-\037]. Ranges can include any characters that are valid for the current mode. If a range that includes letters is used when caseless matching is set, it matches the letters in either case. For example, [W-c] is equivalent - to [][\\^_`wxyzabc], matched caselessly, and in a non-UTF mode, if - character tables for a French locale are in use, [\xc8-\xcb] matches - accented E characters in both cases. In UTF modes, PCRE supports the - concept of case for characters with values greater than 128 only when + to [][\\^_`wxyzabc], matched caselessly, and in a non-UTF mode, if + character tables for a French locale are in use, [\xc8-\xcb] matches + accented E characters in both cases. In UTF modes, PCRE supports the + concept of case for characters with values greater than 128 only when it is compiled with Unicode property support. - The character escape sequences \d, \D, \h, \H, \p, \P, \s, \S, \v, \V, + The character escape sequences \d, \D, \h, \H, \p, \P, \s, \S, \v, \V, \w, and \W may appear in a character class, and add the characters that - they match to the class. For example, [\dABCDEF] matches any hexadeci- - mal digit. In UTF modes, the PCRE_UCP option affects the meanings of - \d, \s, \w and their upper case partners, just as it does when they - appear outside a character class, as described in the section entitled + they match to the class. For example, [\dABCDEF] matches any hexadeci- + mal digit. In UTF modes, the PCRE_UCP option affects the meanings of + \d, \s, \w and their upper case partners, just as it does when they + appear outside a character class, as described in the section entitled "Generic character types" above. The escape sequence \b has a different - meaning inside a character class; it matches the backspace character. - The sequences \B, \N, \R, and \X are not special inside a character - class. Like any other unrecognized escape sequences, they are treated - as the literal characters "B", "N", "R", and "X" by default, but cause + meaning inside a character class; it matches the backspace character. + The sequences \B, \N, \R, and \X are not special inside a character + class. Like any other unrecognized escape sequences, they are treated + as the literal characters "B", "N", "R", and "X" by default, but cause an error if the PCRE_EXTRA option is set. - A circumflex can conveniently be used with the upper case character - types to specify a more restricted set of characters than the matching - lower case type. For example, the class [^\W_] matches any letter or + A circumflex can conveniently be used with the upper case character + types to specify a more restricted set of characters than the matching + lower case type. For example, the class [^\W_] matches any letter or digit, but not underscore, whereas [\w] includes underscore. A positive character class should be read as "something OR something OR ..." and a negative class as "NOT something AND NOT something AND NOT ...". - The only metacharacters that are recognized in character classes are - backslash, hyphen (only where it can be interpreted as specifying a - range), circumflex (only at the start), opening square bracket (only - when it can be interpreted as introducing a POSIX class name, or for a - special compatibility feature - see the next two sections), and the + The only metacharacters that are recognized in character classes are + backslash, hyphen (only where it can be interpreted as specifying a + range), circumflex (only at the start), opening square bracket (only + when it can be interpreted as introducing a POSIX class name, or for a + special compatibility feature - see the next two sections), and the terminating closing square bracket. However, escaping other non- alphanumeric characters does no harm. @@ -5860,7 +5861,7 @@ SQUARE BRACKETS AND CHARACTER CLASSES POSIX CHARACTER CLASSES Perl supports the POSIX notation for character classes. This uses names - enclosed by [: and :] within the enclosing square brackets. PCRE also + enclosed by [: and :] within the enclosing square brackets. PCRE also supports this notation. For example, [01[:alpha:]%] @@ -5883,28 +5884,28 @@ POSIX CHARACTER CLASSES word "word" characters (same as \w) xdigit hexadecimal digits - The default "space" characters are HT (9), LF (10), VT (11), FF (12), - CR (13), and space (32). If locale-specific matching is taking place, - the list of space characters may be different; there may be fewer or + The default "space" characters are HT (9), LF (10), VT (11), FF (12), + CR (13), and space (32). If locale-specific matching is taking place, + the list of space characters may be different; there may be fewer or more of them. "Space" used to be different to \s, which did not include VT, for Perl compatibility. However, Perl changed at release 5.18, and - PCRE followed at release 8.34. "Space" and \s now match the same set + PCRE followed at release 8.34. "Space" and \s now match the same set of characters. - The name "word" is a Perl extension, and "blank" is a GNU extension - from Perl 5.8. Another Perl extension is negation, which is indicated + The name "word" is a Perl extension, and "blank" is a GNU extension + from Perl 5.8. Another Perl extension is negation, which is indicated by a ^ character after the colon. For example, [12[:^digit:]] - matches "1", "2", or any non-digit. PCRE (and Perl) also recognize the + matches "1", "2", or any non-digit. PCRE (and Perl) also recognize the POSIX syntax [.ch.] and [=ch=] where "ch" is a "collating element", but these are not supported, and an error is given if they are encountered. By default, characters with values greater than 128 do not match any of - the POSIX character classes. However, if the PCRE_UCP option is passed - to pcre_compile(), some of the classes are changed so that Unicode - character properties are used. This is achieved by replacing certain + the POSIX character classes. However, if the PCRE_UCP option is passed + to pcre_compile(), some of the classes are changed so that Unicode + character properties are used. This is achieved by replacing certain POSIX classes by other sequences, as follows: [:alnum:] becomes \p{Xan} @@ -5916,10 +5917,10 @@ POSIX CHARACTER CLASSES [:upper:] becomes \p{Lu} [:word:] becomes \p{Xwd} - Negated versions, such as [:^alpha:] use \P instead of \p. Three other + Negated versions, such as [:^alpha:] use \P instead of \p. Three other POSIX classes are handled specially in UCP mode: - [:graph:] This matches characters that have glyphs that mark the page + [:graph:] This matches characters that have glyphs that mark the page when printed. In Unicode property terms, it matches all char- acters with the L, M, N, P, S, or Cf properties, except for: @@ -5928,58 +5929,58 @@ POSIX CHARACTER CLASSES U+2066 - U+2069 Various "isolate"s - [:print:] This matches the same characters as [:graph:] plus space - characters that are not controls, that is, characters with + [:print:] This matches the same characters as [:graph:] plus space + characters that are not controls, that is, characters with the Zs property. [:punct:] This matches all characters that have the Unicode P (punctua- - tion) property, plus those characters whose code points are + tion) property, plus those characters whose code points are less than 128 that have the S (Symbol) property. - The other POSIX classes are unchanged, and match only characters with + The other POSIX classes are unchanged, and match only characters with code points less than 128. COMPATIBILITY FEATURE FOR WORD BOUNDARIES - In the POSIX.2 compliant library that was included in 4.4BSD Unix, the - ugly syntax [[:<:]] and [[:>:]] is used for matching "start of word" + In the POSIX.2 compliant library that was included in 4.4BSD Unix, the + ugly syntax [[:<:]] and [[:>:]] is used for matching "start of word" and "end of word". PCRE treats these items as follows: [[:<:]] is converted to \b(?=\w) [[:>:]] is converted to \b(?<=\w) Only these exact character sequences are recognized. A sequence such as - [a[:<:]b] provokes error for an unrecognized POSIX class name. This - support is not compatible with Perl. It is provided to help migrations + [a[:<:]b] provokes error for an unrecognized POSIX class name. This + support is not compatible with Perl. It is provided to help migrations from other environments, and is best not used in any new patterns. Note - that \b matches at the start and the end of a word (see "Simple asser- - tions" above), and in a Perl-style pattern the preceding or following - character normally shows which is wanted, without the need for the - assertions that are used above in order to give exactly the POSIX be- + that \b matches at the start and the end of a word (see "Simple asser- + tions" above), and in a Perl-style pattern the preceding or following + character normally shows which is wanted, without the need for the + assertions that are used above in order to give exactly the POSIX be- haviour. VERTICAL BAR - Vertical bar characters are used to separate alternative patterns. For + Vertical bar characters are used to separate alternative patterns. For example, the pattern gilbert|sullivan - matches either "gilbert" or "sullivan". Any number of alternatives may - appear, and an empty alternative is permitted (matching the empty + matches either "gilbert" or "sullivan". Any number of alternatives may + appear, and an empty alternative is permitted (matching the empty string). The matching process tries each alternative in turn, from left - to right, and the first one that succeeds is used. If the alternatives - are within a subpattern (defined below), "succeeds" means matching the + to right, and the first one that succeeds is used. If the alternatives + are within a subpattern (defined below), "succeeds" means matching the rest of the main pattern as well as the alternative in the subpattern. INTERNAL OPTION SETTING - The settings of the PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, and - PCRE_EXTENDED options (which are Perl-compatible) can be changed from - within the pattern by a sequence of Perl option letters enclosed + The settings of the PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, and + PCRE_EXTENDED options (which are Perl-compatible) can be changed from + within the pattern by a sequence of Perl option letters enclosed between "(?" and ")". The option letters are i for PCRE_CASELESS @@ -5989,51 +5990,47 @@ INTERNAL OPTION SETTING For example, (?im) sets caseless, multiline matching. It is also possi- ble to unset these options by preceding the letter with a hyphen, and a - combined setting and unsetting such as (?im-sx), which sets PCRE_CASE- - LESS and PCRE_MULTILINE while unsetting PCRE_DOTALL and PCRE_EXTENDED, - is also permitted. If a letter appears both before and after the + combined setting and unsetting such as (?im-sx), which sets PCRE_CASE- + LESS and PCRE_MULTILINE while unsetting PCRE_DOTALL and PCRE_EXTENDED, + is also permitted. If a letter appears both before and after the hyphen, the option is unset. - The PCRE-specific options PCRE_DUPNAMES, PCRE_UNGREEDY, and PCRE_EXTRA - can be changed in the same way as the Perl-compatible options by using + The PCRE-specific options PCRE_DUPNAMES, PCRE_UNGREEDY, and PCRE_EXTRA + can be changed in the same way as the Perl-compatible options by using the characters J, U and X respectively. - When one of these option changes occurs at top level (that is, not - inside subpattern parentheses), the change applies to the remainder of - the pattern that follows. If the change is placed right at the start of - a pattern, PCRE extracts it into the global options (and it will there- - fore show up in data extracted by the pcre_fullinfo() function). - - An option change within a subpattern (see below for a description of - subpatterns) affects only that part of the subpattern that follows it, - so + When one of these option changes occurs at top level (that is, not + inside subpattern parentheses), the change applies to the remainder of + the pattern that follows. An option change within a subpattern (see + below for a description of subpatterns) affects only that part of the + subpattern that follows it, so (a(?i)b)c matches abc and aBc and no other strings (assuming PCRE_CASELESS is not - used). By this means, options can be made to have different settings - in different parts of the pattern. Any changes made in one alternative - do carry on into subsequent branches within the same subpattern. For + used). By this means, options can be made to have different settings + in different parts of the pattern. Any changes made in one alternative + do carry on into subsequent branches within the same subpattern. For example, (a(?i)b|c) - matches "ab", "aB", "c", and "C", even though when matching "C" the - first branch is abandoned before the option setting. This is because - the effects of option settings happen at compile time. There would be + matches "ab", "aB", "c", and "C", even though when matching "C" the + first branch is abandoned before the option setting. This is because + the effects of option settings happen at compile time. There would be some very weird behaviour otherwise. - Note: There are other PCRE-specific options that can be set by the - application when the compiling or matching functions are called. In - some cases the pattern can contain special leading sequences such as - (*CRLF) to override what the application has set or what has been - defaulted. Details are given in the section entitled "Newline - sequences" above. There are also the (*UTF8), (*UTF16),(*UTF32), and - (*UCP) leading sequences that can be used to set UTF and Unicode prop- - erty modes; they are equivalent to setting the PCRE_UTF8, PCRE_UTF16, - PCRE_UTF32 and the PCRE_UCP options, respectively. The (*UTF) sequence - is a generic version that can be used with any of the libraries. How- - ever, the application can set the PCRE_NEVER_UTF option, which locks + Note: There are other PCRE-specific options that can be set by the + application when the compiling or matching functions are called. In + some cases the pattern can contain special leading sequences such as + (*CRLF) to override what the application has set or what has been + defaulted. Details are given in the section entitled "Newline + sequences" above. There are also the (*UTF8), (*UTF16),(*UTF32), and + (*UCP) leading sequences that can be used to set UTF and Unicode prop- + erty modes; they are equivalent to setting the PCRE_UTF8, PCRE_UTF16, + PCRE_UTF32 and the PCRE_UCP options, respectively. The (*UTF) sequence + is a generic version that can be used with any of the libraries. How- + ever, the application can set the PCRE_NEVER_UTF option, which locks out the use of the (*UTF) sequences. @@ -6046,18 +6043,18 @@ SUBPATTERNS cat(aract|erpillar|) - matches "cataract", "caterpillar", or "cat". Without the parentheses, + matches "cataract", "caterpillar", or "cat". Without the parentheses, it would match "cataract", "erpillar" or an empty string. - 2. It sets up the subpattern as a capturing subpattern. This means - that, when the whole pattern matches, that portion of the subject + 2. It sets up the subpattern as a capturing subpattern. This means + that, when the whole pattern matches, that portion of the subject string that matched the subpattern is passed back to the caller via the - ovector argument of the matching function. (This applies only to the - traditional matching functions; the DFA matching functions do not sup- + ovector argument of the matching function. (This applies only to the + traditional matching functions; the DFA matching functions do not sup- port capturing.) Opening parentheses are counted from left to right (starting from 1) to - obtain numbers for the capturing subpatterns. For example, if the + obtain numbers for the capturing subpatterns. For example, if the string "the red king" is matched against the pattern the ((red|white) (king|queen)) @@ -6065,12 +6062,12 @@ SUBPATTERNS the captured substrings are "red king", "red", and "king", and are num- bered 1, 2, and 3, respectively. - The fact that plain parentheses fulfil two functions is not always - helpful. There are often times when a grouping subpattern is required - without a capturing requirement. If an opening parenthesis is followed - by a question mark and a colon, the subpattern does not do any captur- - ing, and is not counted when computing the number of any subsequent - capturing subpatterns. For example, if the string "the white queen" is + The fact that plain parentheses fulfil two functions is not always + helpful. There are often times when a grouping subpattern is required + without a capturing requirement. If an opening parenthesis is followed + by a question mark and a colon, the subpattern does not do any captur- + ing, and is not counted when computing the number of any subsequent + capturing subpatterns. For example, if the string "the white queen" is matched against the pattern the ((?:red|white) (king|queen)) @@ -6078,37 +6075,37 @@ SUBPATTERNS the captured substrings are "white queen" and "queen", and are numbered 1 and 2. The maximum number of capturing subpatterns is 65535. - As a convenient shorthand, if any option settings are required at the - start of a non-capturing subpattern, the option letters may appear + As a convenient shorthand, if any option settings are required at the + start of a non-capturing subpattern, the option letters may appear between the "?" and the ":". Thus the two patterns (?i:saturday|sunday) (?:(?i)saturday|sunday) match exactly the same set of strings. Because alternative branches are - tried from left to right, and options are not reset until the end of - the subpattern is reached, an option setting in one branch does affect - subsequent branches, so the above patterns match "SUNDAY" as well as + tried from left to right, and options are not reset until the end of + the subpattern is reached, an option setting in one branch does affect + subsequent branches, so the above patterns match "SUNDAY" as well as "Saturday". DUPLICATE SUBPATTERN NUMBERS Perl 5.10 introduced a feature whereby each alternative in a subpattern - uses the same numbers for its capturing parentheses. Such a subpattern - starts with (?| and is itself a non-capturing subpattern. For example, + uses the same numbers for its capturing parentheses. Such a subpattern + starts with (?| and is itself a non-capturing subpattern. For example, consider this pattern: (?|(Sat)ur|(Sun))day - Because the two alternatives are inside a (?| group, both sets of cap- - turing parentheses are numbered one. Thus, when the pattern matches, - you can look at captured substring number one, whichever alternative - matched. This construct is useful when you want to capture part, but + Because the two alternatives are inside a (?| group, both sets of cap- + turing parentheses are numbered one. Thus, when the pattern matches, + you can look at captured substring number one, whichever alternative + matched. This construct is useful when you want to capture part, but not all, of one of a number of alternatives. Inside a (?| group, paren- - theses are numbered as usual, but the number is reset at the start of - each branch. The numbers of any capturing parentheses that follow the - subpattern start after the highest number used in any branch. The fol- + theses are numbered as usual, but the number is reset at the start of + each branch. The numbers of any capturing parentheses that follow the + subpattern start after the highest number used in any branch. The fol- lowing example is taken from the Perl documentation. The numbers under- neath show in which buffer the captured content will be stored. @@ -6116,58 +6113,58 @@ DUPLICATE SUBPATTERN NUMBERS / ( a ) (?| x ( y ) z | (p (q) r) | (t) u (v) ) ( z ) /x # 1 2 2 3 2 3 4 - A back reference to a numbered subpattern uses the most recent value - that is set for that number by any subpattern. The following pattern + A back reference to a numbered subpattern uses the most recent value + that is set for that number by any subpattern. The following pattern matches "abcabc" or "defdef": /(?|(abc)|(def))\1/ - In contrast, a subroutine call to a numbered subpattern always refers - to the first one in the pattern with the given number. The following + In contrast, a subroutine call to a numbered subpattern always refers + to the first one in the pattern with the given number. The following pattern matches "abcabc" or "defabc": /(?|(abc)|(def))(?1)/ - If a condition test for a subpattern's having matched refers to a non- - unique number, the test is true if any of the subpatterns of that num- + If a condition test for a subpattern's having matched refers to a non- + unique number, the test is true if any of the subpatterns of that num- ber have matched. - An alternative approach to using this "branch reset" feature is to use + An alternative approach to using this "branch reset" feature is to use duplicate named subpatterns, as described in the next section. NAMED SUBPATTERNS - Identifying capturing parentheses by number is simple, but it can be - very hard to keep track of the numbers in complicated regular expres- - sions. Furthermore, if an expression is modified, the numbers may - change. To help with this difficulty, PCRE supports the naming of sub- + Identifying capturing parentheses by number is simple, but it can be + very hard to keep track of the numbers in complicated regular expres- + sions. Furthermore, if an expression is modified, the numbers may + change. To help with this difficulty, PCRE supports the naming of sub- patterns. This feature was not added to Perl until release 5.10. Python - had the feature earlier, and PCRE introduced it at release 4.0, using - the Python syntax. PCRE now supports both the Perl and the Python syn- - tax. Perl allows identically numbered subpatterns to have different + had the feature earlier, and PCRE introduced it at release 4.0, using + the Python syntax. PCRE now supports both the Perl and the Python syn- + tax. Perl allows identically numbered subpatterns to have different names, but PCRE does not. - In PCRE, a subpattern can be named in one of three ways: (?<name>...) - or (?'name'...) as in Perl, or (?P<name>...) as in Python. References - to capturing parentheses from other parts of the pattern, such as back - references, recursion, and conditions, can be made by name as well as + In PCRE, a subpattern can be named in one of three ways: (?<name>...) + or (?'name'...) as in Perl, or (?P<name>...) as in Python. References + to capturing parentheses from other parts of the pattern, such as back + references, recursion, and conditions, can be made by name as well as by number. - Names consist of up to 32 alphanumeric characters and underscores, but - must start with a non-digit. Named capturing parentheses are still - allocated numbers as well as names, exactly as if the names were not - present. The PCRE API provides function calls for extracting the name- - to-number translation table from a compiled pattern. There is also a + Names consist of up to 32 alphanumeric characters and underscores, but + must start with a non-digit. Named capturing parentheses are still + allocated numbers as well as names, exactly as if the names were not + present. The PCRE API provides function calls for extracting the name- + to-number translation table from a compiled pattern. There is also a convenience function for extracting a captured substring by name. - By default, a name must be unique within a pattern, but it is possible + By default, a name must be unique within a pattern, but it is possible to relax this constraint by setting the PCRE_DUPNAMES option at compile - time. (Duplicate names are also always permitted for subpatterns with - the same number, set up as described in the previous section.) Dupli- - cate names can be useful for patterns where only one instance of the - named parentheses can match. Suppose you want to match the name of a - weekday, either as a 3-letter abbreviation or as the full name, and in + time. (Duplicate names are also always permitted for subpatterns with + the same number, set up as described in the previous section.) Dupli- + cate names can be useful for patterns where only one instance of the + named parentheses can match. Suppose you want to match the name of a + weekday, either as a 3-letter abbreviation or as the full name, and in both cases you want to extract the abbreviation. This pattern (ignoring the line breaks) does the job: @@ -6177,18 +6174,18 @@ NAMED SUBPATTERNS (?<DN>Thu)(?:rsday)?| (?<DN>Sat)(?:urday)? - There are five capturing substrings, but only one is ever set after a + There are five capturing substrings, but only one is ever set after a match. (An alternative way of solving this problem is to use a "branch reset" subpattern, as described in the previous section.) - The convenience function for extracting the data by name returns the - substring for the first (and in this example, the only) subpattern of - that name that matched. This saves searching to find which numbered + The convenience function for extracting the data by name returns the + substring for the first (and in this example, the only) subpattern of + that name that matched. This saves searching to find which numbered subpattern it was. - If you make a back reference to a non-unique named subpattern from - elsewhere in the pattern, the subpatterns to which the name refers are - checked in the order in which they appear in the overall pattern. The + If you make a back reference to a non-unique named subpattern from + elsewhere in the pattern, the subpatterns to which the name refers are + checked in the order in which they appear in the overall pattern. The first one that is set is used for the reference. For example, this pat- tern matches both "foofoo" and "barbar" but not "foobar" or "barfoo": @@ -6196,29 +6193,29 @@ NAMED SUBPATTERNS If you make a subroutine call to a non-unique named subpattern, the one - that corresponds to the first occurrence of the name is used. In the + that corresponds to the first occurrence of the name is used. In the absence of duplicate numbers (see the previous section) this is the one with the lowest number. If you use a named reference in a condition test (see the section about conditions below), either to check whether a subpattern has matched, or - to check for recursion, all subpatterns with the same name are tested. - If the condition is true for any one of them, the overall condition is - true. This is the same behaviour as testing by number. For further - details of the interfaces for handling named subpatterns, see the + to check for recursion, all subpatterns with the same name are tested. + If the condition is true for any one of them, the overall condition is + true. This is the same behaviour as testing by number. For further + details of the interfaces for handling named subpatterns, see the pcreapi documentation. Warning: You cannot use different names to distinguish between two sub- - patterns with the same number because PCRE uses only the numbers when + patterns with the same number because PCRE uses only the numbers when matching. For this reason, an error is given at compile time if differ- - ent names are given to subpatterns with the same number. However, you + ent names are given to subpatterns with the same number. However, you can always give the same name to subpatterns with the same number, even when PCRE_DUPNAMES is not set. REPETITION - Repetition is specified by quantifiers, which can follow any of the + Repetition is specified by quantifiers, which can follow any of the following items: a literal data character @@ -6232,17 +6229,17 @@ REPETITION a parenthesized subpattern (including assertions) a subroutine call to a subpattern (recursive or otherwise) - The general repetition quantifier specifies a minimum and maximum num- - ber of permitted matches, by giving the two numbers in curly brackets - (braces), separated by a comma. The numbers must be less than 65536, + The general repetition quantifier specifies a minimum and maximum num- + ber of permitted matches, by giving the two numbers in curly brackets + (braces), separated by a comma. The numbers must be less than 65536, and the first must be less than or equal to the second. For example: z{2,4} - matches "zz", "zzz", or "zzzz". A closing brace on its own is not a - special character. If the second number is omitted, but the comma is - present, there is no upper limit; if the second number and the comma - are both omitted, the quantifier specifies an exact number of required + matches "zz", "zzz", or "zzzz". A closing brace on its own is not a + special character. If the second number is omitted, but the comma is + present, there is no upper limit; if the second number and the comma + are both omitted, the quantifier specifies an exact number of required matches. Thus [aeiou]{3,} @@ -6251,50 +6248,50 @@ REPETITION \d{8} - matches exactly 8 digits. An opening curly bracket that appears in a - position where a quantifier is not allowed, or one that does not match - the syntax of a quantifier, is taken as a literal character. For exam- + matches exactly 8 digits. An opening curly bracket that appears in a + position where a quantifier is not allowed, or one that does not match + the syntax of a quantifier, is taken as a literal character. For exam- ple, {,6} is not a quantifier, but a literal string of four characters. In UTF modes, quantifiers apply to characters rather than to individual - data units. Thus, for example, \x{100}{2} matches two characters, each + data units. Thus, for example, \x{100}{2} matches two characters, each of which is represented by a two-byte sequence in a UTF-8 string. Simi- - larly, \X{3} matches three Unicode extended grapheme clusters, each of - which may be several data units long (and they may be of different + larly, \X{3} matches three Unicode extended grapheme clusters, each of + which may be several data units long (and they may be of different lengths). The quantifier {0} is permitted, causing the expression to behave as if the previous item and the quantifier were not present. This may be use- - ful for subpatterns that are referenced as subroutines from elsewhere + ful for subpatterns that are referenced as subroutines from elsewhere in the pattern (but see also the section entitled "Defining subpatterns - for use by reference only" below). Items other than subpatterns that + for use by reference only" below). Items other than subpatterns that have a {0} quantifier are omitted from the compiled pattern. - For convenience, the three most common quantifiers have single-charac- + For convenience, the three most common quantifiers have single-charac- ter abbreviations: * is equivalent to {0,} + is equivalent to {1,} ? is equivalent to {0,1} - It is possible to construct infinite loops by following a subpattern + It is possible to construct infinite loops by following a subpattern that can match no characters with a quantifier that has no upper limit, for example: (a?)* Earlier versions of Perl and PCRE used to give an error at compile time - for such patterns. However, because there are cases where this can be - useful, such patterns are now accepted, but if any repetition of the - subpattern does in fact match no characters, the loop is forcibly bro- + for such patterns. However, because there are cases where this can be + useful, such patterns are now accepted, but if any repetition of the + subpattern does in fact match no characters, the loop is forcibly bro- ken. - By default, the quantifiers are "greedy", that is, they match as much - as possible (up to the maximum number of permitted times), without - causing the rest of the pattern to fail. The classic example of where + By default, the quantifiers are "greedy", that is, they match as much + as possible (up to the maximum number of permitted times), without + causing the rest of the pattern to fail. The classic example of where this gives problems is in trying to match comments in C programs. These - appear between /* and */ and within the comment, individual * and / - characters may appear. An attempt to match C comments by applying the + appear between /* and */ and within the comment, individual * and / + characters may appear. An attempt to match C comments by applying the pattern /\*.*\*/ @@ -6303,19 +6300,19 @@ REPETITION /* first comment */ not comment /* second comment */ - fails, because it matches the entire string owing to the greediness of + fails, because it matches the entire string owing to the greediness of the .* item. - However, if a quantifier is followed by a question mark, it ceases to + However, if a quantifier is followed by a question mark, it ceases to be greedy, and instead matches the minimum number of times possible, so the pattern /\*.*?\*/ - does the right thing with the C comments. The meaning of the various - quantifiers is not otherwise changed, just the preferred number of - matches. Do not confuse this use of question mark with its use as a - quantifier in its own right. Because it has two uses, it can sometimes + does the right thing with the C comments. The meaning of the various + quantifiers is not otherwise changed, just the preferred number of + matches. Do not confuse this use of question mark with its use as a + quantifier in its own right. Because it has two uses, it can sometimes appear doubled, as in \d??\d @@ -6323,45 +6320,45 @@ REPETITION which matches one digit by preference, but can match two if that is the only way the rest of the pattern matches. - If the PCRE_UNGREEDY option is set (an option that is not available in - Perl), the quantifiers are not greedy by default, but individual ones - can be made greedy by following them with a question mark. In other + If the PCRE_UNGREEDY option is set (an option that is not available in + Perl), the quantifiers are not greedy by default, but individual ones + can be made greedy by following them with a question mark. In other words, it inverts the default behaviour. - When a parenthesized subpattern is quantified with a minimum repeat - count that is greater than 1 or with a limited maximum, more memory is - required for the compiled pattern, in proportion to the size of the + When a parenthesized subpattern is quantified with a minimum repeat + count that is greater than 1 or with a limited maximum, more memory is + required for the compiled pattern, in proportion to the size of the minimum or maximum. If a pattern starts with .* or .{0,} and the PCRE_DOTALL option (equiv- - alent to Perl's /s) is set, thus allowing the dot to match newlines, - the pattern is implicitly anchored, because whatever follows will be - tried against every character position in the subject string, so there - is no point in retrying the overall match at any position after the - first. PCRE normally treats such a pattern as though it were preceded + alent to Perl's /s) is set, thus allowing the dot to match newlines, + the pattern is implicitly anchored, because whatever follows will be + tried against every character position in the subject string, so there + is no point in retrying the overall match at any position after the + first. PCRE normally treats such a pattern as though it were preceded by \A. - In cases where it is known that the subject string contains no new- - lines, it is worth setting PCRE_DOTALL in order to obtain this opti- + In cases where it is known that the subject string contains no new- + lines, it is worth setting PCRE_DOTALL in order to obtain this opti- mization, or alternatively using ^ to indicate anchoring explicitly. - However, there are some cases where the optimization cannot be used. + However, there are some cases where the optimization cannot be used. When .* is inside capturing parentheses that are the subject of a back reference elsewhere in the pattern, a match at the start may fail where a later one succeeds. Consider, for example: (.*)abc\1 - If the subject is "xyz123abc123" the match point is the fourth charac- + If the subject is "xyz123abc123" the match point is the fourth charac- ter. For this reason, such a pattern is not implicitly anchored. - Another case where implicit anchoring is not applied is when the lead- - ing .* is inside an atomic group. Once again, a match at the start may + Another case where implicit anchoring is not applied is when the lead- + ing .* is inside an atomic group. Once again, a match at the start may fail where a later one succeeds. Consider this pattern: (?>.*?a)b - It matches "ab" in the subject "aab". The use of the backtracking con- + It matches "ab" in the subject "aab". The use of the backtracking con- trol verbs (*PRUNE) and (*SKIP) also disable this optimization. When a capturing subpattern is repeated, the value captured is the sub- @@ -6370,8 +6367,8 @@ REPETITION (tweedle[dume]{3}\s*)+ has matched "tweedledum tweedledee" the value of the captured substring - is "tweedledee". However, if there are nested capturing subpatterns, - the corresponding captured values may have been set in previous itera- + is "tweedledee". However, if there are nested capturing subpatterns, + the corresponding captured values may have been set in previous itera- tions. For example, after /(a|(b))+/ @@ -6381,53 +6378,53 @@ REPETITION ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS - With both maximizing ("greedy") and minimizing ("ungreedy" or "lazy") - repetition, failure of what follows normally causes the repeated item - to be re-evaluated to see if a different number of repeats allows the - rest of the pattern to match. Sometimes it is useful to prevent this, - either to change the nature of the match, or to cause it fail earlier - than it otherwise might, when the author of the pattern knows there is + With both maximizing ("greedy") and minimizing ("ungreedy" or "lazy") + repetition, failure of what follows normally causes the repeated item + to be re-evaluated to see if a different number of repeats allows the + rest of the pattern to match. Sometimes it is useful to prevent this, + either to change the nature of the match, or to cause it fail earlier + than it otherwise might, when the author of the pattern knows there is no point in carrying on. - Consider, for example, the pattern \d+foo when applied to the subject + Consider, for example, the pattern \d+foo when applied to the subject line 123456bar After matching all 6 digits and then failing to match "foo", the normal - action of the matcher is to try again with only 5 digits matching the - \d+ item, and then with 4, and so on, before ultimately failing. - "Atomic grouping" (a term taken from Jeffrey Friedl's book) provides - the means for specifying that once a subpattern has matched, it is not + action of the matcher is to try again with only 5 digits matching the + \d+ item, and then with 4, and so on, before ultimately failing. + "Atomic grouping" (a term taken from Jeffrey Friedl's book) provides + the means for specifying that once a subpattern has matched, it is not to be re-evaluated in this way. - If we use atomic grouping for the previous example, the matcher gives - up immediately on failing to match "foo" the first time. The notation + If we use atomic grouping for the previous example, the matcher gives + up immediately on failing to match "foo" the first time. The notation is a kind of special parenthesis, starting with (?> as in this example: (?>\d+)foo - This kind of parenthesis "locks up" the part of the pattern it con- - tains once it has matched, and a failure further into the pattern is - prevented from backtracking into it. Backtracking past it to previous + This kind of parenthesis "locks up" the part of the pattern it con- + tains once it has matched, and a failure further into the pattern is + prevented from backtracking into it. Backtracking past it to previous items, however, works as normal. - An alternative description is that a subpattern of this type matches - the string of characters that an identical standalone pattern would + An alternative description is that a subpattern of this type matches + the string of characters that an identical standalone pattern would match, if anchored at the current point in the subject string. Atomic grouping subpatterns are not capturing subpatterns. Simple cases such as the above example can be thought of as a maximizing repeat that - must swallow everything it can. So, while both \d+ and \d+? are pre- - pared to adjust the number of digits they match in order to make the + must swallow everything it can. So, while both \d+ and \d+? are pre- + pared to adjust the number of digits they match in order to make the rest of the pattern match, (?>\d+) can only match an entire sequence of digits. - Atomic groups in general can of course contain arbitrarily complicated - subpatterns, and can be nested. However, when the subpattern for an + Atomic groups in general can of course contain arbitrarily complicated + subpatterns, and can be nested. However, when the subpattern for an atomic group is just a single repeated item, as in the example above, a - simpler notation, called a "possessive quantifier" can be used. This - consists of an additional + character following a quantifier. Using + simpler notation, called a "possessive quantifier" can be used. This + consists of an additional + character following a quantifier. Using this notation, the previous example can be rewritten as \d++foo @@ -6437,45 +6434,45 @@ ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS (abc|xyz){2,3}+ - Possessive quantifiers are always greedy; the setting of the + Possessive quantifiers are always greedy; the setting of the PCRE_UNGREEDY option is ignored. They are a convenient notation for the - simpler forms of atomic group. However, there is no difference in the - meaning of a possessive quantifier and the equivalent atomic group, - though there may be a performance difference; possessive quantifiers + simpler forms of atomic group. However, there is no difference in the + meaning of a possessive quantifier and the equivalent atomic group, + though there may be a performance difference; possessive quantifiers should be slightly faster. - The possessive quantifier syntax is an extension to the Perl 5.8 syn- - tax. Jeffrey Friedl originated the idea (and the name) in the first + The possessive quantifier syntax is an extension to the Perl 5.8 syn- + tax. Jeffrey Friedl originated the idea (and the name) in the first edition of his book. Mike McCloskey liked it, so implemented it when he - built Sun's Java package, and PCRE copied it from there. It ultimately + built Sun's Java package, and PCRE copied it from there. It ultimately found its way into Perl at release 5.10. PCRE has an optimization that automatically "possessifies" certain sim- - ple pattern constructs. For example, the sequence A+B is treated as - A++B because there is no point in backtracking into a sequence of A's + ple pattern constructs. For example, the sequence A+B is treated as + A++B because there is no point in backtracking into a sequence of A's when B must follow. - When a pattern contains an unlimited repeat inside a subpattern that - can itself be repeated an unlimited number of times, the use of an - atomic group is the only way to avoid some failing matches taking a + When a pattern contains an unlimited repeat inside a subpattern that + can itself be repeated an unlimited number of times, the use of an + atomic group is the only way to avoid some failing matches taking a very long time indeed. The pattern (\D+|<\d+>)*[!?] - matches an unlimited number of substrings that either consist of non- - digits, or digits enclosed in <>, followed by either ! or ?. When it + matches an unlimited number of substrings that either consist of non- + digits, or digits enclosed in <>, followed by either ! or ?. When it matches, it runs quickly. However, if it is applied to aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - it takes a long time before reporting failure. This is because the - string can be divided between the internal \D+ repeat and the external - * repeat in a large number of ways, and all have to be tried. (The - example uses [!?] rather than a single character at the end, because - both PCRE and Perl have an optimization that allows for fast failure - when a single character is used. They remember the last single charac- - ter that is required for a match, and fail early if it is not present - in the string.) If the pattern is changed so that it uses an atomic + it takes a long time before reporting failure. This is because the + string can be divided between the internal \D+ repeat and the external + * repeat in a large number of ways, and all have to be tried. (The + example uses [!?] rather than a single character at the end, because + both PCRE and Perl have an optimization that allows for fast failure + when a single character is used. They remember the last single charac- + ter that is required for a match, and fail early if it is not present + in the string.) If the pattern is changed so that it uses an atomic group, like this: ((?>\D+)|<\d+>)*[!?] @@ -6487,28 +6484,28 @@ BACK REFERENCES Outside a character class, a backslash followed by a digit greater than 0 (and possibly further digits) is a back reference to a capturing sub- - pattern earlier (that is, to its left) in the pattern, provided there + pattern earlier (that is, to its left) in the pattern, provided there have been that many previous capturing left parentheses. However, if the decimal number following the backslash is less than 10, - it is always taken as a back reference, and causes an error only if - there are not that many capturing left parentheses in the entire pat- - tern. In other words, the parentheses that are referenced need not be - to the left of the reference for numbers less than 10. A "forward back - reference" of this type can make sense when a repetition is involved - and the subpattern to the right has participated in an earlier itera- + it is always taken as a back reference, and causes an error only if + there are not that many capturing left parentheses in the entire pat- + tern. In other words, the parentheses that are referenced need not be + to the left of the reference for numbers less than 10. A "forward back + reference" of this type can make sense when a repetition is involved + and the subpattern to the right has participated in an earlier itera- tion. - It is not possible to have a numerical "forward back reference" to a - subpattern whose number is 10 or more using this syntax because a - sequence such as \50 is interpreted as a character defined in octal. + It is not possible to have a numerical "forward back reference" to a + subpattern whose number is 10 or more using this syntax because a + sequence such as \50 is interpreted as a character defined in octal. See the subsection entitled "Non-printing characters" above for further - details of the handling of digits following a backslash. There is no - such problem when named parentheses are used. A back reference to any + details of the handling of digits following a backslash. There is no + such problem when named parentheses are used. A back reference to any subpattern is possible using named parentheses (see below). - Another way of avoiding the ambiguity inherent in the use of digits - following a backslash is to use the \g escape sequence. This escape + Another way of avoiding the ambiguity inherent in the use of digits + following a backslash is to use the \g escape sequence. This escape must be followed by an unsigned number or a negative number, optionally enclosed in braces. These examples are all identical: @@ -6516,7 +6513,7 @@ BACK REFERENCES (ring), \g1 (ring), \g{1} - An unsigned number specifies an absolute reference without the ambigu- + An unsigned number specifies an absolute reference without the ambigu- ity that is present in the older syntax. It is also useful when literal digits follow the reference. A negative number is a relative reference. Consider this example: @@ -6525,33 +6522,33 @@ BACK REFERENCES The sequence \g{-1} is a reference to the most recently started captur- ing subpattern before \g, that is, is it equivalent to \2 in this exam- - ple. Similarly, \g{-2} would be equivalent to \1. The use of relative - references can be helpful in long patterns, and also in patterns that - are created by joining together fragments that contain references + ple. Similarly, \g{-2} would be equivalent to \1. The use of relative + references can be helpful in long patterns, and also in patterns that + are created by joining together fragments that contain references within themselves. - A back reference matches whatever actually matched the capturing sub- - pattern in the current subject string, rather than anything matching + A back reference matches whatever actually matched the capturing sub- + pattern in the current subject string, rather than anything matching the subpattern itself (see "Subpatterns as subroutines" below for a way of doing that). So the pattern (sens|respons)e and \1ibility - matches "sense and sensibility" and "response and responsibility", but - not "sense and responsibility". If caseful matching is in force at the - time of the back reference, the case of letters is relevant. For exam- + matches "sense and sensibility" and "response and responsibility", but + not "sense and responsibility". If caseful matching is in force at the + time of the back reference, the case of letters is relevant. For exam- ple, ((?i)rah)\s+\1 - matches "rah rah" and "RAH RAH", but not "RAH rah", even though the + matches "rah rah" and "RAH RAH", but not "RAH rah", even though the original capturing subpattern is matched caselessly. - There are several different ways of writing back references to named - subpatterns. The .NET syntax \k{name} and the Perl syntax \k<name> or - \k'name' are supported, as is the Python syntax (?P=name). Perl 5.10's + There are several different ways of writing back references to named + subpatterns. The .NET syntax \k{name} and the Perl syntax \k<name> or + \k'name' are supported, as is the Python syntax (?P=name). Perl 5.10's unified back reference syntax, in which \g can be used for both numeric - and named references, is also supported. We could rewrite the above + and named references, is also supported. We could rewrite the above example in any of the following ways: (?<p1>(?i)rah)\s+\k<p1> @@ -6559,84 +6556,92 @@ BACK REFERENCES (?P<p1>(?i)rah)\s+(?P=p1) (?<p1>(?i)rah)\s+\g{p1} - A subpattern that is referenced by name may appear in the pattern + A subpattern that is referenced by name may appear in the pattern before or after the reference. - There may be more than one back reference to the same subpattern. If a - subpattern has not actually been used in a particular match, any back + There may be more than one back reference to the same subpattern. If a + subpattern has not actually been used in a particular match, any back references to it always fail by default. For example, the pattern (a|(bc))\2 - always fails if it starts to match "a" rather than "bc". However, if + always fails if it starts to match "a" rather than "bc". However, if the PCRE_JAVASCRIPT_COMPAT option is set at compile time, a back refer- ence to an unset value matches an empty string. - Because there may be many capturing parentheses in a pattern, all dig- - its following a backslash are taken as part of a potential back refer- - ence number. If the pattern continues with a digit character, some - delimiter must be used to terminate the back reference. If the - PCRE_EXTENDED option is set, this can be white space. Otherwise, the + Because there may be many capturing parentheses in a pattern, all dig- + its following a backslash are taken as part of a potential back refer- + ence number. If the pattern continues with a digit character, some + delimiter must be used to terminate the back reference. If the + PCRE_EXTENDED option is set, this can be white space. Otherwise, the \g{ syntax or an empty comment (see "Comments" below) can be used. Recursive back references - A back reference that occurs inside the parentheses to which it refers - fails when the subpattern is first used, so, for example, (a\1) never - matches. However, such references can be useful inside repeated sub- + A back reference that occurs inside the parentheses to which it refers + fails when the subpattern is first used, so, for example, (a\1) never + matches. However, such references can be useful inside repeated sub- patterns. For example, the pattern (a|b\1)+ matches any number of "a"s and also "aba", "ababbaa" etc. At each iter- - ation of the subpattern, the back reference matches the character - string corresponding to the previous iteration. In order for this to - work, the pattern must be such that the first iteration does not need - to match the back reference. This can be done using alternation, as in + ation of the subpattern, the back reference matches the character + string corresponding to the previous iteration. In order for this to + work, the pattern must be such that the first iteration does not need + to match the back reference. This can be done using alternation, as in the example above, or by a quantifier with a minimum of zero. - Back references of this type cause the group that they reference to be - treated as an atomic group. Once the whole group has been matched, a - subsequent matching failure cannot cause backtracking into the middle + Back references of this type cause the group that they reference to be + treated as an atomic group. Once the whole group has been matched, a + subsequent matching failure cannot cause backtracking into the middle of the group. ASSERTIONS - An assertion is a test on the characters following or preceding the - current matching point that does not actually consume any characters. - The simple assertions coded as \b, \B, \A, \G, \Z, \z, ^ and $ are + An assertion is a test on the characters following or preceding the + current matching point that does not actually consume any characters. + The simple assertions coded as \b, \B, \A, \G, \Z, \z, ^ and $ are described above. - More complicated assertions are coded as subpatterns. There are two - kinds: those that look ahead of the current position in the subject - string, and those that look behind it. An assertion subpattern is - matched in the normal way, except that it does not cause the current + More complicated assertions are coded as subpatterns. There are two + kinds: those that look ahead of the current position in the subject + string, and those that look behind it. An assertion subpattern is + matched in the normal way, except that it does not cause the current matching position to be changed. - Assertion subpatterns are not capturing subpatterns. If such an asser- - tion contains capturing subpatterns within it, these are counted for - the purposes of numbering the capturing subpatterns in the whole pat- - tern. However, substring capturing is carried out only for positive + Assertion subpatterns are not capturing subpatterns. If such an asser- + tion contains capturing subpatterns within it, these are counted for + the purposes of numbering the capturing subpatterns in the whole pat- + tern. However, substring capturing is carried out only for positive assertions. (Perl sometimes, but not always, does do capturing in nega- tive assertions.) - For compatibility with Perl, assertion subpatterns may be repeated; - though it makes no sense to assert the same thing several times, the - side effect of capturing parentheses may occasionally be useful. In + WARNING: If a positive assertion containing one or more capturing sub- + patterns succeeds, but failure to match later in the pattern causes + backtracking over this assertion, the captures within the assertion are + reset only if no higher numbered captures are already set. This is, + unfortunately, a fundamental limitation of the current implementation, + and as PCRE1 is now in maintenance-only status, it is unlikely ever to + change. + + For compatibility with Perl, assertion subpatterns may be repeated; + though it makes no sense to assert the same thing several times, the + side effect of capturing parentheses may occasionally be useful. In practice, there only three cases: - (1) If the quantifier is {0}, the assertion is never obeyed during - matching. However, it may contain internal capturing parenthesized + (1) If the quantifier is {0}, the assertion is never obeyed during + matching. However, it may contain internal capturing parenthesized groups that are called from elsewhere via the subroutine mechanism. - (2) If quantifier is {0,n} where n is greater than zero, it is treated - as if it were {0,1}. At run time, the rest of the pattern match is + (2) If quantifier is {0,n} where n is greater than zero, it is treated + as if it were {0,1}. At run time, the rest of the pattern match is tried with and without the assertion, the order depending on the greed- iness of the quantifier. - (3) If the minimum repetition is greater than zero, the quantifier is - ignored. The assertion is obeyed just once when encountered during + (3) If the minimum repetition is greater than zero, the quantifier is + ignored. The assertion is obeyed just once when encountered during matching. Lookahead assertions @@ -6646,38 +6651,38 @@ ASSERTIONS \w+(?=;) - matches a word followed by a semicolon, but does not include the semi- + matches a word followed by a semicolon, but does not include the semi- colon in the match, and foo(?!bar) - matches any occurrence of "foo" that is not followed by "bar". Note + matches any occurrence of "foo" that is not followed by "bar". Note that the apparently similar pattern (?!foo)bar - does not find an occurrence of "bar" that is preceded by something - other than "foo"; it finds any occurrence of "bar" whatsoever, because + does not find an occurrence of "bar" that is preceded by something + other than "foo"; it finds any occurrence of "bar" whatsoever, because the assertion (?!foo) is always true when the next three characters are "bar". A lookbehind assertion is needed to achieve the other effect. If you want to force a matching failure at some point in a pattern, the - most convenient way to do it is with (?!) because an empty string - always matches, so an assertion that requires there not to be an empty + most convenient way to do it is with (?!) because an empty string + always matches, so an assertion that requires there not to be an empty string must always fail. The backtracking control verb (*FAIL) or (*F) is a synonym for (?!). Lookbehind assertions - Lookbehind assertions start with (?<= for positive assertions and (?<! + Lookbehind assertions start with (?<= for positive assertions and (?<! for negative assertions. For example, (?<!foo)bar - does find an occurrence of "bar" that is not preceded by "foo". The - contents of a lookbehind assertion are restricted such that all the + does find an occurrence of "bar" that is not preceded by "foo". The + contents of a lookbehind assertion are restricted such that all the strings it matches must have a fixed length. However, if there are sev- - eral top-level alternatives, they do not all have to have the same + eral top-level alternatives, they do not all have to have the same fixed length. Thus (?<=bullock|donkey) @@ -6686,62 +6691,62 @@ ASSERTIONS (?<!dogs?|cats?) - causes an error at compile time. Branches that match different length - strings are permitted only at the top level of a lookbehind assertion. + causes an error at compile time. Branches that match different length + strings are permitted only at the top level of a lookbehind assertion. This is an extension compared with Perl, which requires all branches to match the same length of string. An assertion such as (?<=ab(c|de)) - is not permitted, because its single top-level branch can match two + is not permitted, because its single top-level branch can match two different lengths, but it is acceptable to PCRE if rewritten to use two top-level branches: (?<=abc|abde) - In some cases, the escape sequence \K (see above) can be used instead + In some cases, the escape sequence \K (see above) can be used instead of a lookbehind assertion to get round the fixed-length restriction. - The implementation of lookbehind assertions is, for each alternative, - to temporarily move the current position back by the fixed length and + The implementation of lookbehind assertions is, for each alternative, + to temporarily move the current position back by the fixed length and then try to match. If there are insufficient characters before the cur- rent position, the assertion fails. - In a UTF mode, PCRE does not allow the \C escape (which matches a sin- - gle data unit even in a UTF mode) to appear in lookbehind assertions, - because it makes it impossible to calculate the length of the lookbe- - hind. The \X and \R escapes, which can match different numbers of data + In a UTF mode, PCRE does not allow the \C escape (which matches a sin- + gle data unit even in a UTF mode) to appear in lookbehind assertions, + because it makes it impossible to calculate the length of the lookbe- + hind. The \X and \R escapes, which can match different numbers of data units, are also not permitted. - "Subroutine" calls (see below) such as (?2) or (?&X) are permitted in - lookbehinds, as long as the subpattern matches a fixed-length string. + "Subroutine" calls (see below) such as (?2) or (?&X) are permitted in + lookbehinds, as long as the subpattern matches a fixed-length string. Recursion, however, is not supported. - Possessive quantifiers can be used in conjunction with lookbehind + Possessive quantifiers can be used in conjunction with lookbehind assertions to specify efficient matching of fixed-length strings at the end of subject strings. Consider a simple pattern such as abcd$ - when applied to a long string that does not match. Because matching + when applied to a long string that does not match. Because matching proceeds from left to right, PCRE will look for each "a" in the subject - and then see if what follows matches the rest of the pattern. If the + and then see if what follows matches the rest of the pattern. If the pattern is specified as ^.*abcd$ - the initial .* matches the entire string at first, but when this fails + the initial .* matches the entire string at first, but when this fails (because there is no following "a"), it backtracks to match all but the - last character, then all but the last two characters, and so on. Once - again the search for "a" covers the entire string, from right to left, + last character, then all but the last two characters, and so on. Once + again the search for "a" covers the entire string, from right to left, so we are no better off. However, if the pattern is written as ^.*+(?<=abcd) - there can be no backtracking for the .*+ item; it can match only the - entire string. The subsequent lookbehind assertion does a single test - on the last four characters. If it fails, the match fails immediately. - For long strings, this approach makes a significant difference to the + there can be no backtracking for the .*+ item; it can match only the + entire string. The subsequent lookbehind assertion does a single test + on the last four characters. If it fails, the match fails immediately. + For long strings, this approach makes a significant difference to the processing time. Using multiple assertions @@ -6750,18 +6755,18 @@ ASSERTIONS (?<=\d{3})(?<!999)foo - matches "foo" preceded by three digits that are not "999". Notice that - each of the assertions is applied independently at the same point in - the subject string. First there is a check that the previous three - characters are all digits, and then there is a check that the same + matches "foo" preceded by three digits that are not "999". Notice that + each of the assertions is applied independently at the same point in + the subject string. First there is a check that the previous three + characters are all digits, and then there is a check that the same three characters are not "999". This pattern does not match "foo" pre- - ceded by six characters, the first of which are digits and the last - three of which are not "999". For example, it doesn't match "123abc- + ceded by six characters, the first of which are digits and the last + three of which are not "999". For example, it doesn't match "123abc- foo". A pattern to do that is (?<=\d{3}...)(?<!999)foo - This time the first assertion looks at the preceding six characters, + This time the first assertion looks at the preceding six characters, checking that the first three are digits, and then the second assertion checks that the preceding three characters are not "999". @@ -6769,29 +6774,29 @@ ASSERTIONS (?<=(?<!foo)bar)baz - matches an occurrence of "baz" that is preceded by "bar" which in turn + matches an occurrence of "baz" that is preceded by "bar" which in turn is not preceded by "foo", while (?<=\d{3}(?!999)...)foo - is another pattern that matches "foo" preceded by three digits and any + is another pattern that matches "foo" preceded by three digits and any three characters that are not "999". CONDITIONAL SUBPATTERNS - It is possible to cause the matching process to obey a subpattern con- - ditionally or to choose between two alternative subpatterns, depending - on the result of an assertion, or whether a specific capturing subpat- - tern has already been matched. The two possible forms of conditional + It is possible to cause the matching process to obey a subpattern con- + ditionally or to choose between two alternative subpatterns, depending + on the result of an assertion, or whether a specific capturing subpat- + tern has already been matched. The two possible forms of conditional subpattern are: (?(condition)yes-pattern) (?(condition)yes-pattern|no-pattern) - If the condition is satisfied, the yes-pattern is used; otherwise the - no-pattern (if present) is used. If there are more than two alterna- - tives in the subpattern, a compile-time error occurs. Each of the two + If the condition is satisfied, the yes-pattern is used; otherwise the + no-pattern (if present) is used. If there are more than two alterna- + tives in the subpattern, a compile-time error occurs. Each of the two alternatives may itself contain nested subpatterns of any form, includ- ing conditional subpatterns; the restriction to two alternatives applies only at the level of the condition. This pattern fragment is an @@ -6800,68 +6805,68 @@ CONDITIONAL SUBPATTERNS (?(1) (A|B|C) | (D | (?(2)E|F) | E) ) - There are four kinds of condition: references to subpatterns, refer- + There are four kinds of condition: references to subpatterns, refer- ences to recursion, a pseudo-condition called DEFINE, and assertions. Checking for a used subpattern by number - If the text between the parentheses consists of a sequence of digits, + If the text between the parentheses consists of a sequence of digits, the condition is true if a capturing subpattern of that number has pre- - viously matched. If there is more than one capturing subpattern with - the same number (see the earlier section about duplicate subpattern - numbers), the condition is true if any of them have matched. An alter- - native notation is to precede the digits with a plus or minus sign. In - this case, the subpattern number is relative rather than absolute. The - most recently opened parentheses can be referenced by (?(-1), the next - most recent by (?(-2), and so on. Inside loops it can also make sense + viously matched. If there is more than one capturing subpattern with + the same number (see the earlier section about duplicate subpattern + numbers), the condition is true if any of them have matched. An alter- + native notation is to precede the digits with a plus or minus sign. In + this case, the subpattern number is relative rather than absolute. The + most recently opened parentheses can be referenced by (?(-1), the next + most recent by (?(-2), and so on. Inside loops it can also make sense to refer to subsequent groups. The next parentheses to be opened can be - referenced as (?(+1), and so on. (The value zero in any of these forms + referenced as (?(+1), and so on. (The value zero in any of these forms is not used; it provokes a compile-time error.) - Consider the following pattern, which contains non-significant white + Consider the following pattern, which contains non-significant white space to make it more readable (assume the PCRE_EXTENDED option) and to divide it into three parts for ease of discussion: ( \( )? [^()]+ (?(1) \) ) - The first part matches an optional opening parenthesis, and if that + The first part matches an optional opening parenthesis, and if that character is present, sets it as the first captured substring. The sec- - ond part matches one or more characters that are not parentheses. The - third part is a conditional subpattern that tests whether or not the - first set of parentheses matched. If they did, that is, if subject - started with an opening parenthesis, the condition is true, and so the - yes-pattern is executed and a closing parenthesis is required. Other- - wise, since no-pattern is not present, the subpattern matches nothing. - In other words, this pattern matches a sequence of non-parentheses, + ond part matches one or more characters that are not parentheses. The + third part is a conditional subpattern that tests whether or not the + first set of parentheses matched. If they did, that is, if subject + started with an opening parenthesis, the condition is true, and so the + yes-pattern is executed and a closing parenthesis is required. Other- + wise, since no-pattern is not present, the subpattern matches nothing. + In other words, this pattern matches a sequence of non-parentheses, optionally enclosed in parentheses. - If you were embedding this pattern in a larger one, you could use a + If you were embedding this pattern in a larger one, you could use a relative reference: ...other stuff... ( \( )? [^()]+ (?(-1) \) ) ... - This makes the fragment independent of the parentheses in the larger + This makes the fragment independent of the parentheses in the larger pattern. Checking for a used subpattern by name - Perl uses the syntax (?(<name>)...) or (?('name')...) to test for a - used subpattern by name. For compatibility with earlier versions of - PCRE, which had this facility before Perl, the syntax (?(name)...) is + Perl uses the syntax (?(<name>)...) or (?('name')...) to test for a + used subpattern by name. For compatibility with earlier versions of + PCRE, which had this facility before Perl, the syntax (?(name)...) is also recognized. Rewriting the above example to use a named subpattern gives this: (?<OPEN> \( )? [^()]+ (?(<OPEN>) \) ) - If the name used in a condition of this kind is a duplicate, the test - is applied to all subpatterns of the same name, and is true if any one + If the name used in a condition of this kind is a duplicate, the test + is applied to all subpatterns of the same name, and is true if any one of them has matched. Checking for pattern recursion If the condition is the string (R), and there is no subpattern with the - name R, the condition is true if a recursive call to the whole pattern + name R, the condition is true if a recursive call to the whole pattern or any subpattern has been made. If digits or a name preceded by amper- sand follow the letter R, for example: @@ -6869,51 +6874,51 @@ CONDITIONAL SUBPATTERNS the condition is true if the most recent recursion is into a subpattern whose number or name is given. This condition does not check the entire - recursion stack. If the name used in a condition of this kind is a + recursion stack. If the name used in a condition of this kind is a duplicate, the test is applied to all subpatterns of the same name, and is true if any one of them is the most recent recursion. - At "top level", all these recursion test conditions are false. The + At "top level", all these recursion test conditions are false. The syntax for recursive patterns is described below. Defining subpatterns for use by reference only - If the condition is the string (DEFINE), and there is no subpattern - with the name DEFINE, the condition is always false. In this case, - there may be only one alternative in the subpattern. It is always - skipped if control reaches this point in the pattern; the idea of - DEFINE is that it can be used to define subroutines that can be refer- - enced from elsewhere. (The use of subroutines is described below.) For - example, a pattern to match an IPv4 address such as "192.168.23.245" + If the condition is the string (DEFINE), and there is no subpattern + with the name DEFINE, the condition is always false. In this case, + there may be only one alternative in the subpattern. It is always + skipped if control reaches this point in the pattern; the idea of + DEFINE is that it can be used to define subroutines that can be refer- + enced from elsewhere. (The use of subroutines is described below.) For + example, a pattern to match an IPv4 address such as "192.168.23.245" could be written like this (ignore white space and line breaks): (?(DEFINE) (?<byte> 2[0-4]\d | 25[0-5] | 1\d\d | [1-9]?\d) ) \b (?&byte) (\.(?&byte)){3} \b - The first part of the pattern is a DEFINE group inside which a another - group named "byte" is defined. This matches an individual component of - an IPv4 address (a number less than 256). When matching takes place, - this part of the pattern is skipped because DEFINE acts like a false - condition. The rest of the pattern uses references to the named group - to match the four dot-separated components of an IPv4 address, insist- + The first part of the pattern is a DEFINE group inside which a another + group named "byte" is defined. This matches an individual component of + an IPv4 address (a number less than 256). When matching takes place, + this part of the pattern is skipped because DEFINE acts like a false + condition. The rest of the pattern uses references to the named group + to match the four dot-separated components of an IPv4 address, insist- ing on a word boundary at each end. Assertion conditions - If the condition is not in any of the above formats, it must be an - assertion. This may be a positive or negative lookahead or lookbehind - assertion. Consider this pattern, again containing non-significant + If the condition is not in any of the above formats, it must be an + assertion. This may be a positive or negative lookahead or lookbehind + assertion. Consider this pattern, again containing non-significant white space, and with the two alternatives on the second line: (?(?=[^a-z]*[a-z]) \d{2}-[a-z]{3}-\d{2} | \d{2}-\d{2}-\d{2} ) - The condition is a positive lookahead assertion that matches an - optional sequence of non-letters followed by a letter. In other words, - it tests for the presence of at least one letter in the subject. If a - letter is found, the subject is matched against the first alternative; - otherwise it is matched against the second. This pattern matches - strings in one of the two forms dd-aaa-dd or dd-dd-dd, where aaa are + The condition is a positive lookahead assertion that matches an + optional sequence of non-letters followed by a letter. In other words, + it tests for the presence of at least one letter in the subject. If a + letter is found, the subject is matched against the first alternative; + otherwise it is matched against the second. This pattern matches + strings in one of the two forms dd-aaa-dd or dd-dd-dd, where aaa are letters and dd are digits. @@ -6922,41 +6927,41 @@ COMMENTS There are two ways of including comments in patterns that are processed by PCRE. In both cases, the start of the comment must not be in a char- acter class, nor in the middle of any other sequence of related charac- - ters such as (?: or a subpattern name or number. The characters that + ters such as (?: or a subpattern name or number. The characters that make up a comment play no part in the pattern matching. - The sequence (?# marks the start of a comment that continues up to the - next closing parenthesis. Nested parentheses are not permitted. If the + The sequence (?# marks the start of a comment that continues up to the + next closing parenthesis. Nested parentheses are not permitted. If the PCRE_EXTENDED option is set, an unescaped # character also introduces a - comment, which in this case continues to immediately after the next - newline character or character sequence in the pattern. Which charac- + comment, which in this case continues to immediately after the next + newline character or character sequence in the pattern. Which charac- ters are interpreted as newlines is controlled by the options passed to - a compiling function or by a special sequence at the start of the pat- + a compiling function or by a special sequence at the start of the pat- tern, as described in the section entitled "Newline conventions" above. Note that the end of this type of comment is a literal newline sequence - in the pattern; escape sequences that happen to represent a newline do - not count. For example, consider this pattern when PCRE_EXTENDED is + in the pattern; escape sequences that happen to represent a newline do + not count. For example, consider this pattern when PCRE_EXTENDED is set, and the default newline convention is in force: abc #comment \n still comment - On encountering the # character, pcre_compile() skips along, looking - for a newline in the pattern. The sequence \n is still literal at this - stage, so it does not terminate the comment. Only an actual character + On encountering the # character, pcre_compile() skips along, looking + for a newline in the pattern. The sequence \n is still literal at this + stage, so it does not terminate the comment. Only an actual character with the code value 0x0a (the default newline) does so. RECURSIVE PATTERNS - Consider the problem of matching a string in parentheses, allowing for - unlimited nested parentheses. Without the use of recursion, the best - that can be done is to use a pattern that matches up to some fixed - depth of nesting. It is not possible to handle an arbitrary nesting + Consider the problem of matching a string in parentheses, allowing for + unlimited nested parentheses. Without the use of recursion, the best + that can be done is to use a pattern that matches up to some fixed + depth of nesting. It is not possible to handle an arbitrary nesting depth. For some time, Perl has provided a facility that allows regular expres- - sions to recurse (amongst other things). It does this by interpolating - Perl code in the expression at run time, and the code can refer to the + sions to recurse (amongst other things). It does this by interpolating + Perl code in the expression at run time, and the code can refer to the expression itself. A Perl pattern using code interpolation to solve the parentheses problem can be created like this: @@ -6966,201 +6971,201 @@ RECURSIVE PATTERNS refers recursively to the pattern in which it appears. Obviously, PCRE cannot support the interpolation of Perl code. Instead, - it supports special syntax for recursion of the entire pattern, and - also for individual subpattern recursion. After its introduction in - PCRE and Python, this kind of recursion was subsequently introduced + it supports special syntax for recursion of the entire pattern, and + also for individual subpattern recursion. After its introduction in + PCRE and Python, this kind of recursion was subsequently introduced into Perl at release 5.10. - A special item that consists of (? followed by a number greater than - zero and a closing parenthesis is a recursive subroutine call of the - subpattern of the given number, provided that it occurs inside that - subpattern. (If not, it is a non-recursive subroutine call, which is - described in the next section.) The special item (?R) or (?0) is a + A special item that consists of (? followed by a number greater than + zero and a closing parenthesis is a recursive subroutine call of the + subpattern of the given number, provided that it occurs inside that + subpattern. (If not, it is a non-recursive subroutine call, which is + described in the next section.) The special item (?R) or (?0) is a recursive call of the entire regular expression. - This PCRE pattern solves the nested parentheses problem (assume the + This PCRE pattern solves the nested parentheses problem (assume the PCRE_EXTENDED option is set so that white space is ignored): \( ( [^()]++ | (?R) )* \) - First it matches an opening parenthesis. Then it matches any number of - substrings which can either be a sequence of non-parentheses, or a - recursive match of the pattern itself (that is, a correctly parenthe- + First it matches an opening parenthesis. Then it matches any number of + substrings which can either be a sequence of non-parentheses, or a + recursive match of the pattern itself (that is, a correctly parenthe- sized substring). Finally there is a closing parenthesis. Note the use of a possessive quantifier to avoid backtracking into sequences of non- parentheses. - If this were part of a larger pattern, you would not want to recurse + If this were part of a larger pattern, you would not want to recurse the entire pattern, so instead you could use this: ( \( ( [^()]++ | (?1) )* \) ) - We have put the pattern into parentheses, and caused the recursion to + We have put the pattern into parentheses, and caused the recursion to refer to them instead of the whole pattern. - In a larger pattern, keeping track of parenthesis numbers can be - tricky. This is made easier by the use of relative references. Instead + In a larger pattern, keeping track of parenthesis numbers can be + tricky. This is made easier by the use of relative references. Instead of (?1) in the pattern above you can write (?-2) to refer to the second - most recently opened parentheses preceding the recursion. In other - words, a negative number counts capturing parentheses leftwards from + most recently opened parentheses preceding the recursion. In other + words, a negative number counts capturing parentheses leftwards from the point at which it is encountered. - It is also possible to refer to subsequently opened parentheses, by - writing references such as (?+2). However, these cannot be recursive - because the reference is not inside the parentheses that are refer- - enced. They are always non-recursive subroutine calls, as described in + It is also possible to refer to subsequently opened parentheses, by + writing references such as (?+2). However, these cannot be recursive + because the reference is not inside the parentheses that are refer- + enced. They are always non-recursive subroutine calls, as described in the next section. - An alternative approach is to use named parentheses instead. The Perl - syntax for this is (?&name); PCRE's earlier syntax (?P>name) is also + An alternative approach is to use named parentheses instead. The Perl + syntax for this is (?&name); PCRE's earlier syntax (?P>name) is also supported. We could rewrite the above example as follows: (?<pn> \( ( [^()]++ | (?&pn) )* \) ) - If there is more than one subpattern with the same name, the earliest + If there is more than one subpattern with the same name, the earliest one is used. - This particular example pattern that we have been looking at contains + This particular example pattern that we have been looking at contains nested unlimited repeats, and so the use of a possessive quantifier for matching strings of non-parentheses is important when applying the pat- - tern to strings that do not match. For example, when this pattern is + tern to strings that do not match. For example, when this pattern is applied to (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() - it yields "no match" quickly. However, if a possessive quantifier is - not used, the match runs for a very long time indeed because there are - so many different ways the + and * repeats can carve up the subject, + it yields "no match" quickly. However, if a possessive quantifier is + not used, the match runs for a very long time indeed because there are + so many different ways the + and * repeats can carve up the subject, and all have to be tested before failure can be reported. - At the end of a match, the values of capturing parentheses are those - from the outermost level. If you want to obtain intermediate values, a - callout function can be used (see below and the pcrecallout documenta- + At the end of a match, the values of capturing parentheses are those + from the outermost level. If you want to obtain intermediate values, a + callout function can be used (see below and the pcrecallout documenta- tion). If the pattern above is matched against (ab(cd)ef) - the value for the inner capturing parentheses (numbered 2) is "ef", - which is the last value taken on at the top level. If a capturing sub- - pattern is not matched at the top level, its final captured value is - unset, even if it was (temporarily) set at a deeper level during the + the value for the inner capturing parentheses (numbered 2) is "ef", + which is the last value taken on at the top level. If a capturing sub- + pattern is not matched at the top level, its final captured value is + unset, even if it was (temporarily) set at a deeper level during the matching process. - If there are more than 15 capturing parentheses in a pattern, PCRE has - to obtain extra memory to store data during a recursion, which it does + If there are more than 15 capturing parentheses in a pattern, PCRE has + to obtain extra memory to store data during a recursion, which it does by using pcre_malloc, freeing it via pcre_free afterwards. If no memory can be obtained, the match fails with the PCRE_ERROR_NOMEMORY error. - Do not confuse the (?R) item with the condition (R), which tests for - recursion. Consider this pattern, which matches text in angle brack- - ets, allowing for arbitrary nesting. Only digits are allowed in nested - brackets (that is, when recursing), whereas any characters are permit- + Do not confuse the (?R) item with the condition (R), which tests for + recursion. Consider this pattern, which matches text in angle brack- + ets, allowing for arbitrary nesting. Only digits are allowed in nested + brackets (that is, when recursing), whereas any characters are permit- ted at the outer level. < (?: (?(R) \d++ | [^<>]*+) | (?R)) * > - In this pattern, (?(R) is the start of a conditional subpattern, with - two different alternatives for the recursive and non-recursive cases. + In this pattern, (?(R) is the start of a conditional subpattern, with + two different alternatives for the recursive and non-recursive cases. The (?R) item is the actual recursive call. Differences in recursion processing between PCRE and Perl - Recursion processing in PCRE differs from Perl in two important ways. - In PCRE (like Python, but unlike Perl), a recursive subpattern call is + Recursion processing in PCRE differs from Perl in two important ways. + In PCRE (like Python, but unlike Perl), a recursive subpattern call is always treated as an atomic group. That is, once it has matched some of the subject string, it is never re-entered, even if it contains untried - alternatives and there is a subsequent matching failure. This can be - illustrated by the following pattern, which purports to match a palin- - dromic string that contains an odd number of characters (for example, + alternatives and there is a subsequent matching failure. This can be + illustrated by the following pattern, which purports to match a palin- + dromic string that contains an odd number of characters (for example, "a", "aba", "abcba", "abcdcba"): ^(.|(.)(?1)\2)$ The idea is that it either matches a single character, or two identical - characters surrounding a sub-palindrome. In Perl, this pattern works; - in PCRE it does not if the pattern is longer than three characters. + characters surrounding a sub-palindrome. In Perl, this pattern works; + in PCRE it does not if the pattern is longer than three characters. Consider the subject string "abcba": - At the top level, the first character is matched, but as it is not at + At the top level, the first character is matched, but as it is not at the end of the string, the first alternative fails; the second alterna- tive is taken and the recursion kicks in. The recursive call to subpat- - tern 1 successfully matches the next character ("b"). (Note that the + tern 1 successfully matches the next character ("b"). (Note that the beginning and end of line tests are not part of the recursion). - Back at the top level, the next character ("c") is compared with what - subpattern 2 matched, which was "a". This fails. Because the recursion - is treated as an atomic group, there are now no backtracking points, - and so the entire match fails. (Perl is able, at this point, to re- - enter the recursion and try the second alternative.) However, if the + Back at the top level, the next character ("c") is compared with what + subpattern 2 matched, which was "a". This fails. Because the recursion + is treated as an atomic group, there are now no backtracking points, + and so the entire match fails. (Perl is able, at this point, to re- + enter the recursion and try the second alternative.) However, if the pattern is written with the alternatives in the other order, things are different: ^((.)(?1)\2|.)$ - This time, the recursing alternative is tried first, and continues to - recurse until it runs out of characters, at which point the recursion - fails. But this time we do have another alternative to try at the - higher level. That is the big difference: in the previous case the + This time, the recursing alternative is tried first, and continues to + recurse until it runs out of characters, at which point the recursion + fails. But this time we do have another alternative to try at the + higher level. That is the big difference: in the previous case the remaining alternative is at a deeper recursion level, which PCRE cannot use. - To change the pattern so that it matches all palindromic strings, not - just those with an odd number of characters, it is tempting to change + To change the pattern so that it matches all palindromic strings, not + just those with an odd number of characters, it is tempting to change the pattern to this: ^((.)(?1)\2|.?)$ - Again, this works in Perl, but not in PCRE, and for the same reason. - When a deeper recursion has matched a single character, it cannot be - entered again in order to match an empty string. The solution is to - separate the two cases, and write out the odd and even cases as alter- + Again, this works in Perl, but not in PCRE, and for the same reason. + When a deeper recursion has matched a single character, it cannot be + entered again in order to match an empty string. The solution is to + separate the two cases, and write out the odd and even cases as alter- natives at the higher level: ^(?:((.)(?1)\2|)|((.)(?3)\4|.)) - If you want to match typical palindromic phrases, the pattern has to + If you want to match typical palindromic phrases, the pattern has to ignore all non-word characters, which can be done like this: ^\W*+(?:((.)\W*+(?1)\W*+\2|)|((.)\W*+(?3)\W*+\4|\W*+.\W*+))\W*+$ If run with the PCRE_CASELESS option, this pattern matches phrases such as "A man, a plan, a canal: Panama!" and it works well in both PCRE and - Perl. Note the use of the possessive quantifier *+ to avoid backtrack- - ing into sequences of non-word characters. Without this, PCRE takes a - great deal longer (ten times or more) to match typical phrases, and + Perl. Note the use of the possessive quantifier *+ to avoid backtrack- + ing into sequences of non-word characters. Without this, PCRE takes a + great deal longer (ten times or more) to match typical phrases, and Perl takes so long that you think it has gone into a loop. - WARNING: The palindrome-matching patterns above work only if the sub- - ject string does not start with a palindrome that is shorter than the - entire string. For example, although "abcba" is correctly matched, if - the subject is "ababa", PCRE finds the palindrome "aba" at the start, - then fails at top level because the end of the string does not follow. - Once again, it cannot jump back into the recursion to try other alter- + WARNING: The palindrome-matching patterns above work only if the sub- + ject string does not start with a palindrome that is shorter than the + entire string. For example, although "abcba" is correctly matched, if + the subject is "ababa", PCRE finds the palindrome "aba" at the start, + then fails at top level because the end of the string does not follow. + Once again, it cannot jump back into the recursion to try other alter- natives, so the entire match fails. - The second way in which PCRE and Perl differ in their recursion pro- - cessing is in the handling of captured values. In Perl, when a subpat- - tern is called recursively or as a subpattern (see the next section), - it has no access to any values that were captured outside the recur- - sion, whereas in PCRE these values can be referenced. Consider this + The second way in which PCRE and Perl differ in their recursion pro- + cessing is in the handling of captured values. In Perl, when a subpat- + tern is called recursively or as a subpattern (see the next section), + it has no access to any values that were captured outside the recur- + sion, whereas in PCRE these values can be referenced. Consider this pattern: ^(.)(\1|a(?2)) - In PCRE, this pattern matches "bab". The first capturing parentheses - match "b", then in the second group, when the back reference \1 fails - to match "b", the second alternative matches "a" and then recurses. In - the recursion, \1 does now match "b" and so the whole match succeeds. - In Perl, the pattern fails to match because inside the recursive call + In PCRE, this pattern matches "bab". The first capturing parentheses + match "b", then in the second group, when the back reference \1 fails + to match "b", the second alternative matches "a" and then recurses. In + the recursion, \1 does now match "b" and so the whole match succeeds. + In Perl, the pattern fails to match because inside the recursive call \1 cannot access the externally set value. SUBPATTERNS AS SUBROUTINES - If the syntax for a recursive subpattern call (either by number or by - name) is used outside the parentheses to which it refers, it operates - like a subroutine in a programming language. The called subpattern may - be defined before or after the reference. A numbered reference can be + If the syntax for a recursive subpattern call (either by number or by + name) is used outside the parentheses to which it refers, it operates + like a subroutine in a programming language. The called subpattern may + be defined before or after the reference. A numbered reference can be absolute or relative, as in these examples: (...(absolute)...)...(?2)... @@ -7171,79 +7176,79 @@ SUBPATTERNS AS SUBROUTINES (sens|respons)e and \1ibility - matches "sense and sensibility" and "response and responsibility", but + matches "sense and sensibility" and "response and responsibility", but not "sense and responsibility". If instead the pattern (sens|respons)e and (?1)ibility - is used, it does match "sense and responsibility" as well as the other - two strings. Another example is given in the discussion of DEFINE + is used, it does match "sense and responsibility" as well as the other + two strings. Another example is given in the discussion of DEFINE above. - All subroutine calls, whether recursive or not, are always treated as - atomic groups. That is, once a subroutine has matched some of the sub- + All subroutine calls, whether recursive or not, are always treated as + atomic groups. That is, once a subroutine has matched some of the sub- ject string, it is never re-entered, even if it contains untried alter- - natives and there is a subsequent matching failure. Any capturing - parentheses that are set during the subroutine call revert to their + natives and there is a subsequent matching failure. Any capturing + parentheses that are set during the subroutine call revert to their previous values afterwards. - Processing options such as case-independence are fixed when a subpat- - tern is defined, so if it is used as a subroutine, such options cannot + Processing options such as case-independence are fixed when a subpat- + tern is defined, so if it is used as a subroutine, such options cannot be changed for different calls. For example, consider this pattern: (abc)(?i:(?-1)) - It matches "abcabc". It does not match "abcABC" because the change of + It matches "abcabc". It does not match "abcABC" because the change of processing option does not affect the called subpattern. ONIGURUMA SUBROUTINE SYNTAX - For compatibility with Oniguruma, the non-Perl syntax \g followed by a + For compatibility with Oniguruma, the non-Perl syntax \g followed by a name or a number enclosed either in angle brackets or single quotes, is - an alternative syntax for referencing a subpattern as a subroutine, - possibly recursively. Here are two of the examples used above, rewrit- + an alternative syntax for referencing a subpattern as a subroutine, + possibly recursively. Here are two of the examples used above, rewrit- ten using this syntax: (?<pn> \( ( (?>[^()]+) | \g<pn> )* \) ) (sens|respons)e and \g'1'ibility - PCRE supports an extension to Oniguruma: if a number is preceded by a + PCRE supports an extension to Oniguruma: if a number is preceded by a plus or a minus sign it is taken as a relative reference. For example: (abc)(?i:\g<-1>) - Note that \g{...} (Perl syntax) and \g<...> (Oniguruma syntax) are not - synonymous. The former is a back reference; the latter is a subroutine + Note that \g{...} (Perl syntax) and \g<...> (Oniguruma syntax) are not + synonymous. The former is a back reference; the latter is a subroutine call. CALLOUTS Perl has a feature whereby using the sequence (?{...}) causes arbitrary - Perl code to be obeyed in the middle of matching a regular expression. + Perl code to be obeyed in the middle of matching a regular expression. This makes it possible, amongst other things, to extract different sub- strings that match the same pair of parentheses when there is a repeti- tion. PCRE provides a similar feature, but of course it cannot obey arbitrary Perl code. The feature is called "callout". The caller of PCRE provides - an external function by putting its entry point in the global variable - pcre_callout (8-bit library) or pcre[16|32]_callout (16-bit or 32-bit - library). By default, this variable contains NULL, which disables all + an external function by putting its entry point in the global variable + pcre_callout (8-bit library) or pcre[16|32]_callout (16-bit or 32-bit + library). By default, this variable contains NULL, which disables all calling out. - Within a regular expression, (?C) indicates the points at which the - external function is to be called. If you want to identify different - callout points, you can put a number less than 256 after the letter C. - The default value is zero. For example, this pattern has two callout + Within a regular expression, (?C) indicates the points at which the + external function is to be called. If you want to identify different + callout points, you can put a number less than 256 after the letter C. + The default value is zero. For example, this pattern has two callout points: (?C1)abc(?C2)def - If the PCRE_AUTO_CALLOUT flag is passed to a compiling function, call- - outs are automatically installed before each item in the pattern. They - are all numbered 255. If there is a conditional group in the pattern + If the PCRE_AUTO_CALLOUT flag is passed to a compiling function, call- + outs are automatically installed before each item in the pattern. They + are all numbered 255. If there is a conditional group in the pattern whose condition is an assertion, an additional callout is inserted just before the condition. An explicit callout may also be set at this posi- tion, as in this example: @@ -7253,120 +7258,120 @@ CALLOUTS Note that this applies only to assertion conditions, not to other types of condition. - During matching, when PCRE reaches a callout point, the external func- - tion is called. It is provided with the number of the callout, the - position in the pattern, and, optionally, one item of data originally - supplied by the caller of the matching function. The callout function + During matching, when PCRE reaches a callout point, the external func- + tion is called. It is provided with the number of the callout, the + position in the pattern, and, optionally, one item of data originally + supplied by the caller of the matching function. The callout function may cause matching to proceed, to backtrack, or to fail altogether. - By default, PCRE implements a number of optimizations at compile time - and matching time, and one side-effect is that sometimes callouts are - skipped. If you need all possible callouts to happen, you need to set - options that disable the relevant optimizations. More details, and a - complete description of the interface to the callout function, are + By default, PCRE implements a number of optimizations at compile time + and matching time, and one side-effect is that sometimes callouts are + skipped. If you need all possible callouts to happen, you need to set + options that disable the relevant optimizations. More details, and a + complete description of the interface to the callout function, are given in the pcrecallout documentation. BACKTRACKING CONTROL - Perl 5.10 introduced a number of "Special Backtracking Control Verbs", - which are still described in the Perl documentation as "experimental - and subject to change or removal in a future version of Perl". It goes - on to say: "Their usage in production code should be noted to avoid - problems during upgrades." The same remarks apply to the PCRE features + Perl 5.10 introduced a number of "Special Backtracking Control Verbs", + which are still described in the Perl documentation as "experimental + and subject to change or removal in a future version of Perl". It goes + on to say: "Their usage in production code should be noted to avoid + problems during upgrades." The same remarks apply to the PCRE features described in this section. - The new verbs make use of what was previously invalid syntax: an open- + The new verbs make use of what was previously invalid syntax: an open- ing parenthesis followed by an asterisk. They are generally of the form - (*VERB) or (*VERB:NAME). Some may take either form, possibly behaving - differently depending on whether or not a name is present. A name is + (*VERB) or (*VERB:NAME). Some may take either form, possibly behaving + differently depending on whether or not a name is present. A name is any sequence of characters that does not include a closing parenthesis. The maximum length of name is 255 in the 8-bit library and 65535 in the - 16-bit and 32-bit libraries. If the name is empty, that is, if the - closing parenthesis immediately follows the colon, the effect is as if - the colon were not there. Any number of these verbs may occur in a + 16-bit and 32-bit libraries. If the name is empty, that is, if the + closing parenthesis immediately follows the colon, the effect is as if + the colon were not there. Any number of these verbs may occur in a pattern. - Since these verbs are specifically related to backtracking, most of - them can be used only when the pattern is to be matched using one of - the traditional matching functions, because these use a backtracking - algorithm. With the exception of (*FAIL), which behaves like a failing - negative assertion, the backtracking control verbs cause an error if + Since these verbs are specifically related to backtracking, most of + them can be used only when the pattern is to be matched using one of + the traditional matching functions, because these use a backtracking + algorithm. With the exception of (*FAIL), which behaves like a failing + negative assertion, the backtracking control verbs cause an error if encountered by a DFA matching function. - The behaviour of these verbs in repeated groups, assertions, and in + The behaviour of these verbs in repeated groups, assertions, and in subpatterns called as subroutines (whether or not recursively) is docu- mented below. Optimizations that affect backtracking verbs - PCRE contains some optimizations that are used to speed up matching by + PCRE contains some optimizations that are used to speed up matching by running some checks at the start of each match attempt. For example, it - may know the minimum length of matching subject, or that a particular + may know the minimum length of matching subject, or that a particular character must be present. When one of these optimizations bypasses the - running of a match, any included backtracking verbs will not, of + running of a match, any included backtracking verbs will not, of course, be processed. You can suppress the start-of-match optimizations - by setting the PCRE_NO_START_OPTIMIZE option when calling pcre_com- + by setting the PCRE_NO_START_OPTIMIZE option when calling pcre_com- pile() or pcre_exec(), or by starting the pattern with (*NO_START_OPT). There is more discussion of this option in the section entitled "Option bits for pcre_exec()" in the pcreapi documentation. - Experiments with Perl suggest that it too has similar optimizations, + Experiments with Perl suggest that it too has similar optimizations, sometimes leading to anomalous results. Verbs that act immediately - The following verbs act as soon as they are encountered. They may not + The following verbs act as soon as they are encountered. They may not be followed by a name. (*ACCEPT) - This verb causes the match to end successfully, skipping the remainder - of the pattern. However, when it is inside a subpattern that is called - as a subroutine, only that subpattern is ended successfully. Matching + This verb causes the match to end successfully, skipping the remainder + of the pattern. However, when it is inside a subpattern that is called + as a subroutine, only that subpattern is ended successfully. Matching then continues at the outer level. If (*ACCEPT) in triggered in a posi- - tive assertion, the assertion succeeds; in a negative assertion, the + tive assertion, the assertion succeeds; in a negative assertion, the assertion fails. - If (*ACCEPT) is inside capturing parentheses, the data so far is cap- + If (*ACCEPT) is inside capturing parentheses, the data so far is cap- tured. For example: A((?:A|B(*ACCEPT)|C)D) - This matches "AB", "AAD", or "ACD"; when it matches "AB", "B" is cap- + This matches "AB", "AAD", or "ACD"; when it matches "AB", "B" is cap- tured by the outer parentheses. (*FAIL) or (*F) - This verb causes a matching failure, forcing backtracking to occur. It - is equivalent to (?!) but easier to read. The Perl documentation notes - that it is probably useful only when combined with (?{}) or (??{}). - Those are, of course, Perl features that are not present in PCRE. The - nearest equivalent is the callout feature, as for example in this pat- + This verb causes a matching failure, forcing backtracking to occur. It + is equivalent to (?!) but easier to read. The Perl documentation notes + that it is probably useful only when combined with (?{}) or (??{}). + Those are, of course, Perl features that are not present in PCRE. The + nearest equivalent is the callout feature, as for example in this pat- tern: a+(?C)(*FAIL) - A match with the string "aaaa" always fails, but the callout is taken + A match with the string "aaaa" always fails, but the callout is taken before each backtrack happens (in this example, 10 times). Recording which path was taken - There is one verb whose main purpose is to track how a match was - arrived at, though it also has a secondary use in conjunction with + There is one verb whose main purpose is to track how a match was + arrived at, though it also has a secondary use in conjunction with advancing the match starting point (see (*SKIP) below). (*MARK:NAME) or (*:NAME) - A name is always required with this verb. There may be as many - instances of (*MARK) as you like in a pattern, and their names do not + A name is always required with this verb. There may be as many + instances of (*MARK) as you like in a pattern, and their names do not have to be unique. - When a match succeeds, the name of the last-encountered (*MARK:NAME), - (*PRUNE:NAME), or (*THEN:NAME) on the matching path is passed back to - the caller as described in the section entitled "Extra data for - pcre_exec()" in the pcreapi documentation. Here is an example of - pcretest output, where the /K modifier requests the retrieval and out- + When a match succeeds, the name of the last-encountered (*MARK:NAME), + (*PRUNE:NAME), or (*THEN:NAME) on the matching path is passed back to + the caller as described in the section entitled "Extra data for + pcre_exec()" in the pcreapi documentation. Here is an example of + pcretest output, where the /K modifier requests the retrieval and out- putting of (*MARK) data: re> /X(*MARK:A)Y|X(*MARK:B)Z/K @@ -7378,73 +7383,73 @@ BACKTRACKING CONTROL MK: B The (*MARK) name is tagged with "MK:" in this output, and in this exam- - ple it indicates which of the two alternatives matched. This is a more - efficient way of obtaining this information than putting each alterna- + ple it indicates which of the two alternatives matched. This is a more + efficient way of obtaining this information than putting each alterna- tive in its own capturing parentheses. - If a verb with a name is encountered in a positive assertion that is - true, the name is recorded and passed back if it is the last-encoun- + If a verb with a name is encountered in a positive assertion that is + true, the name is recorded and passed back if it is the last-encoun- tered. This does not happen for negative assertions or failing positive assertions. - After a partial match or a failed match, the last encountered name in + After a partial match or a failed match, the last encountered name in the entire match process is returned. For example: re> /X(*MARK:A)Y|X(*MARK:B)Z/K data> XP No match, mark = B - Note that in this unanchored example the mark is retained from the + Note that in this unanchored example the mark is retained from the match attempt that started at the letter "X" in the subject. Subsequent match attempts starting at "P" and then with an empty string do not get as far as the (*MARK) item, but nevertheless do not reset it. - If you are interested in (*MARK) values after failed matches, you - should probably set the PCRE_NO_START_OPTIMIZE option (see above) to + If you are interested in (*MARK) values after failed matches, you + should probably set the PCRE_NO_START_OPTIMIZE option (see above) to ensure that the match is always attempted. Verbs that act after backtracking The following verbs do nothing when they are encountered. Matching con- - tinues with what follows, but if there is no subsequent match, causing - a backtrack to the verb, a failure is forced. That is, backtracking - cannot pass to the left of the verb. However, when one of these verbs + tinues with what follows, but if there is no subsequent match, causing + a backtrack to the verb, a failure is forced. That is, backtracking + cannot pass to the left of the verb. However, when one of these verbs appears inside an atomic group or an assertion that is true, its effect - is confined to that group, because once the group has been matched, - there is never any backtracking into it. In this situation, backtrack- - ing can "jump back" to the left of the entire atomic group or asser- - tion. (Remember also, as stated above, that this localization also + is confined to that group, because once the group has been matched, + there is never any backtracking into it. In this situation, backtrack- + ing can "jump back" to the left of the entire atomic group or asser- + tion. (Remember also, as stated above, that this localization also applies in subroutine calls.) - These verbs differ in exactly what kind of failure occurs when back- - tracking reaches them. The behaviour described below is what happens - when the verb is not in a subroutine or an assertion. Subsequent sec- + These verbs differ in exactly what kind of failure occurs when back- + tracking reaches them. The behaviour described below is what happens + when the verb is not in a subroutine or an assertion. Subsequent sec- tions cover these special cases. (*COMMIT) - This verb, which may not be followed by a name, causes the whole match + This verb, which may not be followed by a name, causes the whole match to fail outright if there is a later matching failure that causes back- - tracking to reach it. Even if the pattern is unanchored, no further + tracking to reach it. Even if the pattern is unanchored, no further attempts to find a match by advancing the starting point take place. If - (*COMMIT) is the only backtracking verb that is encountered, once it + (*COMMIT) is the only backtracking verb that is encountered, once it has been passed pcre_exec() is committed to finding a match at the cur- rent starting point, or not at all. For example: a+(*COMMIT)b - This matches "xxaab" but not "aacaab". It can be thought of as a kind + This matches "xxaab" but not "aacaab". It can be thought of as a kind of dynamic anchor, or "I've started, so I must finish." The name of the - most recently passed (*MARK) in the path is passed back when (*COMMIT) + most recently passed (*MARK) in the path is passed back when (*COMMIT) forces a match failure. - If there is more than one backtracking verb in a pattern, a different - one that follows (*COMMIT) may be triggered first, so merely passing + If there is more than one backtracking verb in a pattern, a different + one that follows (*COMMIT) may be triggered first, so merely passing (*COMMIT) during a match does not always guarantee that a match must be at this starting point. - Note that (*COMMIT) at the start of a pattern is not the same as an - anchor, unless PCRE's start-of-match optimizations are turned off, as + Note that (*COMMIT) at the start of a pattern is not the same as an + anchor, unless PCRE's start-of-match optimizations are turned off, as shown in this output from pcretest: re> /(*COMMIT)abc/ @@ -7455,207 +7460,207 @@ BACKTRACKING CONTROL For this pattern, PCRE knows that any match must start with "a", so the optimization skips along the subject to "a" before applying the pattern - to the first set of data. The match attempt then succeeds. In the sec- - ond set of data, the escape sequence \Y is interpreted by the pcretest - program. It causes the PCRE_NO_START_OPTIMIZE option to be set when + to the first set of data. The match attempt then succeeds. In the sec- + ond set of data, the escape sequence \Y is interpreted by the pcretest + program. It causes the PCRE_NO_START_OPTIMIZE option to be set when pcre_exec() is called. This disables the optimization that skips along to the first character. The pattern is now applied starting at "x", and - so the (*COMMIT) causes the match to fail without trying any other + so the (*COMMIT) causes the match to fail without trying any other starting points. (*PRUNE) or (*PRUNE:NAME) - This verb causes the match to fail at the current starting position in + This verb causes the match to fail at the current starting position in the subject if there is a later matching failure that causes backtrack- - ing to reach it. If the pattern is unanchored, the normal "bumpalong" - advance to the next starting character then happens. Backtracking can - occur as usual to the left of (*PRUNE), before it is reached, or when - matching to the right of (*PRUNE), but if there is no match to the - right, backtracking cannot cross (*PRUNE). In simple cases, the use of - (*PRUNE) is just an alternative to an atomic group or possessive quan- + ing to reach it. If the pattern is unanchored, the normal "bumpalong" + advance to the next starting character then happens. Backtracking can + occur as usual to the left of (*PRUNE), before it is reached, or when + matching to the right of (*PRUNE), but if there is no match to the + right, backtracking cannot cross (*PRUNE). In simple cases, the use of + (*PRUNE) is just an alternative to an atomic group or possessive quan- tifier, but there are some uses of (*PRUNE) that cannot be expressed in - any other way. In an anchored pattern (*PRUNE) has the same effect as + any other way. In an anchored pattern (*PRUNE) has the same effect as (*COMMIT). The behaviour of (*PRUNE:NAME) is the not the same as - (*MARK:NAME)(*PRUNE). It is like (*MARK:NAME) in that the name is - remembered for passing back to the caller. However, (*SKIP:NAME) + (*MARK:NAME)(*PRUNE). It is like (*MARK:NAME) in that the name is + remembered for passing back to the caller. However, (*SKIP:NAME) searches only for names set with (*MARK). (*SKIP) - This verb, when given without a name, is like (*PRUNE), except that if - the pattern is unanchored, the "bumpalong" advance is not to the next + This verb, when given without a name, is like (*PRUNE), except that if + the pattern is unanchored, the "bumpalong" advance is not to the next character, but to the position in the subject where (*SKIP) was encoun- - tered. (*SKIP) signifies that whatever text was matched leading up to + tered. (*SKIP) signifies that whatever text was matched leading up to it cannot be part of a successful match. Consider: a+(*SKIP)b - If the subject is "aaaac...", after the first match attempt fails - (starting at the first character in the string), the starting point + If the subject is "aaaac...", after the first match attempt fails + (starting at the first character in the string), the starting point skips on to start the next attempt at "c". Note that a possessive quan- - tifer does not have the same effect as this example; although it would - suppress backtracking during the first match attempt, the second - attempt would start at the second character instead of skipping on to + tifer does not have the same effect as this example; although it would + suppress backtracking during the first match attempt, the second + attempt would start at the second character instead of skipping on to "c". (*SKIP:NAME) When (*SKIP) has an associated name, its behaviour is modified. When it is triggered, the previous path through the pattern is searched for the - most recent (*MARK) that has the same name. If one is found, the + most recent (*MARK) that has the same name. If one is found, the "bumpalong" advance is to the subject position that corresponds to that (*MARK) instead of to where (*SKIP) was encountered. If no (*MARK) with a matching name is found, the (*SKIP) is ignored. - Note that (*SKIP:NAME) searches only for names set by (*MARK:NAME). It + Note that (*SKIP:NAME) searches only for names set by (*MARK:NAME). It ignores names that are set by (*PRUNE:NAME) or (*THEN:NAME). (*THEN) or (*THEN:NAME) - This verb causes a skip to the next innermost alternative when back- - tracking reaches it. That is, it cancels any further backtracking - within the current alternative. Its name comes from the observation + This verb causes a skip to the next innermost alternative when back- + tracking reaches it. That is, it cancels any further backtracking + within the current alternative. Its name comes from the observation that it can be used for a pattern-based if-then-else block: ( COND1 (*THEN) FOO | COND2 (*THEN) BAR | COND3 (*THEN) BAZ ) ... - If the COND1 pattern matches, FOO is tried (and possibly further items - after the end of the group if FOO succeeds); on failure, the matcher - skips to the second alternative and tries COND2, without backtracking - into COND1. If that succeeds and BAR fails, COND3 is tried. If subse- - quently BAZ fails, there are no more alternatives, so there is a back- - track to whatever came before the entire group. If (*THEN) is not + If the COND1 pattern matches, FOO is tried (and possibly further items + after the end of the group if FOO succeeds); on failure, the matcher + skips to the second alternative and tries COND2, without backtracking + into COND1. If that succeeds and BAR fails, COND3 is tried. If subse- + quently BAZ fails, there are no more alternatives, so there is a back- + track to whatever came before the entire group. If (*THEN) is not inside an alternation, it acts like (*PRUNE). - The behaviour of (*THEN:NAME) is the not the same as - (*MARK:NAME)(*THEN). It is like (*MARK:NAME) in that the name is - remembered for passing back to the caller. However, (*SKIP:NAME) + The behaviour of (*THEN:NAME) is the not the same as + (*MARK:NAME)(*THEN). It is like (*MARK:NAME) in that the name is + remembered for passing back to the caller. However, (*SKIP:NAME) searches only for names set with (*MARK). - A subpattern that does not contain a | character is just a part of the - enclosing alternative; it is not a nested alternation with only one - alternative. The effect of (*THEN) extends beyond such a subpattern to - the enclosing alternative. Consider this pattern, where A, B, etc. are - complex pattern fragments that do not contain any | characters at this + A subpattern that does not contain a | character is just a part of the + enclosing alternative; it is not a nested alternation with only one + alternative. The effect of (*THEN) extends beyond such a subpattern to + the enclosing alternative. Consider this pattern, where A, B, etc. are + complex pattern fragments that do not contain any | characters at this level: A (B(*THEN)C) | D - If A and B are matched, but there is a failure in C, matching does not + If A and B are matched, but there is a failure in C, matching does not backtrack into A; instead it moves to the next alternative, that is, D. - However, if the subpattern containing (*THEN) is given an alternative, + However, if the subpattern containing (*THEN) is given an alternative, it behaves differently: A (B(*THEN)C | (*FAIL)) | D - The effect of (*THEN) is now confined to the inner subpattern. After a + The effect of (*THEN) is now confined to the inner subpattern. After a failure in C, matching moves to (*FAIL), which causes the whole subpat- - tern to fail because there are no more alternatives to try. In this + tern to fail because there are no more alternatives to try. In this case, matching does now backtrack into A. - Note that a conditional subpattern is not considered as having two - alternatives, because only one is ever used. In other words, the | + Note that a conditional subpattern is not considered as having two + alternatives, because only one is ever used. In other words, the | character in a conditional subpattern has a different meaning. Ignoring white space, consider: ^.*? (?(?=a) a | b(*THEN)c ) - If the subject is "ba", this pattern does not match. Because .*? is - ungreedy, it initially matches zero characters. The condition (?=a) - then fails, the character "b" is matched, but "c" is not. At this - point, matching does not backtrack to .*? as might perhaps be expected - from the presence of the | character. The conditional subpattern is + If the subject is "ba", this pattern does not match. Because .*? is + ungreedy, it initially matches zero characters. The condition (?=a) + then fails, the character "b" is matched, but "c" is not. At this + point, matching does not backtrack to .*? as might perhaps be expected + from the presence of the | character. The conditional subpattern is part of the single alternative that comprises the whole pattern, and so - the match fails. (If there was a backtrack into .*?, allowing it to + the match fails. (If there was a backtrack into .*?, allowing it to match "b", the match would succeed.) - The verbs just described provide four different "strengths" of control + The verbs just described provide four different "strengths" of control when subsequent matching fails. (*THEN) is the weakest, carrying on the - match at the next alternative. (*PRUNE) comes next, failing the match - at the current starting position, but allowing an advance to the next - character (for an unanchored pattern). (*SKIP) is similar, except that + match at the next alternative. (*PRUNE) comes next, failing the match + at the current starting position, but allowing an advance to the next + character (for an unanchored pattern). (*SKIP) is similar, except that the advance may be more than one character. (*COMMIT) is the strongest, causing the entire match to fail. More than one backtracking verb - If more than one backtracking verb is present in a pattern, the one - that is backtracked onto first acts. For example, consider this pat- + If more than one backtracking verb is present in a pattern, the one + that is backtracked onto first acts. For example, consider this pat- tern, where A, B, etc. are complex pattern fragments: (A(*COMMIT)B(*THEN)C|ABD) - If A matches but B fails, the backtrack to (*COMMIT) causes the entire + If A matches but B fails, the backtrack to (*COMMIT) causes the entire match to fail. However, if A and B match, but C fails, the backtrack to - (*THEN) causes the next alternative (ABD) to be tried. This behaviour - is consistent, but is not always the same as Perl's. It means that if - two or more backtracking verbs appear in succession, all the the last + (*THEN) causes the next alternative (ABD) to be tried. This behaviour + is consistent, but is not always the same as Perl's. It means that if + two or more backtracking verbs appear in succession, all the the last of them has no effect. Consider this example: ...(*COMMIT)(*PRUNE)... If there is a matching failure to the right, backtracking onto (*PRUNE) - causes it to be triggered, and its action is taken. There can never be + causes it to be triggered, and its action is taken. There can never be a backtrack onto (*COMMIT). Backtracking verbs in repeated groups - PCRE differs from Perl in its handling of backtracking verbs in + PCRE differs from Perl in its handling of backtracking verbs in repeated groups. For example, consider: /(a(*COMMIT)b)+ac/ - If the subject is "abac", Perl matches, but PCRE fails because the + If the subject is "abac", Perl matches, but PCRE fails because the (*COMMIT) in the second repeat of the group acts. Backtracking verbs in assertions - (*FAIL) in an assertion has its normal effect: it forces an immediate + (*FAIL) in an assertion has its normal effect: it forces an immediate backtrack. (*ACCEPT) in a positive assertion causes the assertion to succeed with- - out any further processing. In a negative assertion, (*ACCEPT) causes + out any further processing. In a negative assertion, (*ACCEPT) causes the assertion to fail without any further processing. - The other backtracking verbs are not treated specially if they appear - in a positive assertion. In particular, (*THEN) skips to the next - alternative in the innermost enclosing group that has alternations, + The other backtracking verbs are not treated specially if they appear + in a positive assertion. In particular, (*THEN) skips to the next + alternative in the innermost enclosing group that has alternations, whether or not this is within the assertion. - Negative assertions are, however, different, in order to ensure that - changing a positive assertion into a negative assertion changes its + Negative assertions are, however, different, in order to ensure that + changing a positive assertion into a negative assertion changes its result. Backtracking into (*COMMIT), (*SKIP), or (*PRUNE) causes a neg- ative assertion to be true, without considering any further alternative branches in the assertion. Backtracking into (*THEN) causes it to skip - to the next enclosing alternative within the assertion (the normal be- - haviour), but if the assertion does not have such an alternative, + to the next enclosing alternative within the assertion (the normal be- + haviour), but if the assertion does not have such an alternative, (*THEN) behaves like (*PRUNE). Backtracking verbs in subroutines - These behaviours occur whether or not the subpattern is called recur- + These behaviours occur whether or not the subpattern is called recur- sively. Perl's treatment of subroutines is different in some cases. - (*FAIL) in a subpattern called as a subroutine has its normal effect: + (*FAIL) in a subpattern called as a subroutine has its normal effect: it forces an immediate backtrack. - (*ACCEPT) in a subpattern called as a subroutine causes the subroutine - match to succeed without any further processing. Matching then contin- + (*ACCEPT) in a subpattern called as a subroutine causes the subroutine + match to succeed without any further processing. Matching then contin- ues after the subroutine call. (*COMMIT), (*SKIP), and (*PRUNE) in a subpattern called as a subroutine cause the subroutine match to fail. - (*THEN) skips to the next alternative in the innermost enclosing group - within the subpattern that has alternatives. If there is no such group + (*THEN) skips to the next alternative in the innermost enclosing group + within the subpattern that has alternatives. If there is no such group within the subpattern, (*THEN) causes the subroutine match to fail. SEE ALSO - pcreapi(3), pcrecallout(3), pcrematching(3), pcresyntax(3), pcre(3), + pcreapi(3), pcrecallout(3), pcrematching(3), pcresyntax(3), pcre(3), pcre16(3), pcre32(3). @@ -7668,8 +7673,8 @@ AUTHOR REVISION - Last updated: 14 June 2015 - Copyright (c) 1997-2015 University of Cambridge. + Last updated: 23 October 2016 + Copyright (c) 1997-2016 University of Cambridge. ------------------------------------------------------------------------------ diff --git a/pcre/doc/pcrecompat.3 b/pcre/doc/pcrecompat.3 index 0cc40198235..6156e776f53 100644 --- a/pcre/doc/pcrecompat.3 +++ b/pcre/doc/pcrecompat.3 @@ -113,7 +113,7 @@ the pattern /^(a(b)?)+$/ in Perl leaves $2 unset, but in PCRE it is set to "b". 14. PCRE's handling of duplicate subpattern numbers and duplicate subpattern names is not as general as Perl's. This is a consequence of the fact the PCRE works internally just with numbers, using an external table to translate -between numbers and names. In particular, a pattern such as (?|(?<a>A)|(?<b)B), +between numbers and names. In particular, a pattern such as (?|(?<a>A)|(?<b>B), where the two capturing parentheses have the same number but different names, is not supported, and causes an error at compile time. If it were allowed, it would not be possible to distinguish which parentheses matched, because both diff --git a/pcre/doc/pcrepattern.3 b/pcre/doc/pcrepattern.3 index 3b8c6393d21..97df217fdb2 100644 --- a/pcre/doc/pcrepattern.3 +++ b/pcre/doc/pcrepattern.3 @@ -1,4 +1,4 @@ -.TH PCREPATTERN 3 "14 June 2015" "PCRE 8.38" +.TH PCREPATTERN 3 "23 October 2016" "PCRE 8.40" .SH NAME PCRE - Perl-compatible regular expressions .SH "PCRE REGULAR EXPRESSION DETAILS" @@ -336,22 +336,22 @@ When PCRE is compiled in EBCDIC mode, \ea, \ee, \ef, \en, \er, and \et generate the appropriate EBCDIC code values. The \ec escape is processed as specified for Perl in the \fBperlebcdic\fP document. The only characters that are allowed after \ec are A-Z, a-z, or one of @, [, \e, ], ^, _, or ?. Any -other character provokes a compile-time error. The sequence \e@ encodes -character code 0; the letters (in either case) encode characters 1-26 (hex 01 -to hex 1A); [, \e, ], ^, and _ encode characters 27-31 (hex 1B to hex 1F), and -\e? becomes either 255 (hex FF) or 95 (hex 5F). +other character provokes a compile-time error. The sequence \ec@ encodes +character code 0; after \ec the letters (in either case) encode characters 1-26 +(hex 01 to hex 1A); [, \e, ], ^, and _ encode characters 27-31 (hex 1B to hex +1F), and \ec? becomes either 255 (hex FF) or 95 (hex 5F). .P -Thus, apart from \e?, these escapes generate the same character code values as +Thus, apart from \ec?, these escapes generate the same character code values as they do in an ASCII environment, though the meanings of the values mostly -differ. For example, \eG always generates code value 7, which is BEL in ASCII +differ. For example, \ecG always generates code value 7, which is BEL in ASCII but DEL in EBCDIC. .P -The sequence \e? generates DEL (127, hex 7F) in an ASCII environment, but +The sequence \ec? generates DEL (127, hex 7F) in an ASCII environment, but because 127 is not a control character in EBCDIC, Perl makes it generate the APC character. Unfortunately, there are several variants of EBCDIC. In most of them the APC character has the value 255 (hex FF), but in the one Perl calls POSIX-BC its value is 95 (hex 5F). If certain other characters have POSIX-BC -values, PCRE makes \e? generate 95; otherwise it generates 255. +values, PCRE makes \ec? generate 95; otherwise it generates 255. .P After \e0 up to two further octal digits are read. If there are fewer than two digits, just those that are present are used. Thus the sequence \e0\ex\e015 @@ -1511,12 +1511,8 @@ J, U and X respectively. .P When one of these option changes occurs at top level (that is, not inside subpattern parentheses), the change applies to the remainder of the pattern -that follows. If the change is placed right at the start of a pattern, PCRE -extracts it into the global options (and it will therefore show up in data -extracted by the \fBpcre_fullinfo()\fP function). -.P -An option change within a subpattern (see below for a description of -subpatterns) affects only that part of the subpattern that follows it, so +that follows. An option change within a subpattern (see below for a description +of subpatterns) affects only that part of the subpattern that follows it, so .sp (a(?i)b)c .sp @@ -2171,6 +2167,13 @@ numbering the capturing subpatterns in the whole pattern. However, substring capturing is carried out only for positive assertions. (Perl sometimes, but not always, does do capturing in negative assertions.) .P +WARNING: If a positive assertion containing one or more capturing subpatterns +succeeds, but failure to match later in the pattern causes backtracking over +this assertion, the captures within the assertion are reset only if no higher +numbered captures are already set. This is, unfortunately, a fundamental +limitation of the current implementation, and as PCRE1 is now in +maintenance-only status, it is unlikely ever to change. +.P For compatibility with Perl, assertion subpatterns may be repeated; though it makes no sense to assert the same thing several times, the side effect of capturing parentheses may occasionally be useful. In practice, there only three @@ -3296,6 +3299,6 @@ Cambridge CB2 3QH, England. .rs .sp .nf -Last updated: 14 June 2015 -Copyright (c) 1997-2015 University of Cambridge. +Last updated: 23 October 2016 +Copyright (c) 1997-2016 University of Cambridge. .fi diff --git a/pcre/pcre_compile.c b/pcre/pcre_compile.c index 7cd39501230..de92313e2f8 100644 --- a/pcre/pcre_compile.c +++ b/pcre/pcre_compile.c @@ -5579,6 +5579,34 @@ for (;; ptr++) #endif #if defined SUPPORT_UTF || !defined COMPILE_PCRE8 { + /* For non-UCP wide characters, in a non-negative class containing \S or + similar (should_flip_negation is set), all characters greater than 255 + must be in the class. */ + + if ( +#if defined COMPILE_PCRE8 + utf && +#endif + should_flip_negation && !negate_class && (options & PCRE_UCP) == 0) + { + *class_uchardata++ = XCL_RANGE; + if (utf) /* Will always be utf in the 8-bit library */ + { + class_uchardata += PRIV(ord2utf)(0x100, class_uchardata); + class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); + } + else /* Can only happen for the 16-bit & 32-bit libraries */ + { +#if defined COMPILE_PCRE16 + *class_uchardata++ = 0x100; + *class_uchardata++ = 0xffffu; +#elif defined COMPILE_PCRE32 + *class_uchardata++ = 0x100; + *class_uchardata++ = 0xffffffffu; +#endif + } + } + *class_uchardata++ = XCL_END; /* Marks the end of extra data */ *code++ = OP_XCLASS; code += LINK_SIZE; @@ -6923,7 +6951,8 @@ for (;; ptr++) slot = cd->name_table; for (i = 0; i < cd->names_found; i++) { - if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0) break; + if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0 && + slot[IMM2_SIZE+namelen] == 0) break; slot += cd->name_entry_size; } @@ -7889,15 +7918,17 @@ for (;; ptr++) } } - /* For a forward assertion, we take the reqchar, if set. This can be - helpful if the pattern that follows the assertion doesn't set a different - char. For example, it's useful for /(?=abcde).+/. We can't set firstchar - for an assertion, however because it leads to incorrect effect for patterns - such as /(?=a)a.+/ when the "real" "a" would then become a reqchar instead - of a firstchar. This is overcome by a scan at the end if there's no - firstchar, looking for an asserted first char. */ - - else if (bravalue == OP_ASSERT && subreqcharflags >= 0) + /* For a forward assertion, we take the reqchar, if set, provided that the + group has also set a first char. This can be helpful if the pattern that + follows the assertion doesn't set a different char. For example, it's + useful for /(?=abcde).+/. We can't set firstchar for an assertion, however + because it leads to incorrect effect for patterns such as /(?=a)a.+/ when + the "real" "a" would then become a reqchar instead of a firstchar. This is + overcome by a scan at the end if there's no firstchar, looking for an + asserted first char. */ + + else if (bravalue == OP_ASSERT && subreqcharflags >= 0 && + subfirstcharflags >= 0) { reqchar = subreqchar; reqcharflags = subreqcharflags; @@ -8686,8 +8717,8 @@ matching and for non-DOTALL patterns that start with .* (which must start at the beginning or after \n). As in the case of is_anchored() (see above), we have to take account of back references to capturing brackets that contain .* because in that case we can't make the assumption. Also, the appearance of .* -inside atomic brackets or in a pattern that contains *PRUNE or *SKIP does not -count, because once again the assumption no longer holds. +inside atomic brackets or in an assertion, or in a pattern that contains *PRUNE +or *SKIP does not count, because once again the assumption no longer holds. Arguments: code points to start of expression (the bracket) @@ -8696,13 +8727,14 @@ Arguments: the less precise approach cd points to the compile data atomcount atomic group level + inassert TRUE if in an assertion Returns: TRUE or FALSE */ static BOOL is_startline(const pcre_uchar *code, unsigned int bracket_map, - compile_data *cd, int atomcount) + compile_data *cd, int atomcount, BOOL inassert) { do { const pcre_uchar *scode = first_significant_code( @@ -8729,7 +8761,7 @@ do { return FALSE; default: /* Assertion */ - if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount, TRUE)) return FALSE; do scode += GET(scode, 1); while (*scode == OP_ALT); scode += 1 + LINK_SIZE; break; @@ -8743,7 +8775,7 @@ do { if (op == OP_BRA || op == OP_BRAPOS || op == OP_SBRA || op == OP_SBRAPOS) { - if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount, inassert)) return FALSE; } /* Capturing brackets */ @@ -8753,33 +8785,33 @@ do { { int n = GET2(scode, 1+LINK_SIZE); int new_map = bracket_map | ((n < 32)? (1 << n) : 1); - if (!is_startline(scode, new_map, cd, atomcount)) return FALSE; + if (!is_startline(scode, new_map, cd, atomcount, inassert)) return FALSE; } /* Positive forward assertions */ else if (op == OP_ASSERT) { - if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount, TRUE)) return FALSE; } /* Atomic brackets */ else if (op == OP_ONCE || op == OP_ONCE_NC) { - if (!is_startline(scode, bracket_map, cd, atomcount + 1)) return FALSE; + if (!is_startline(scode, bracket_map, cd, atomcount + 1, inassert)) return FALSE; } /* .* means "start at start or after \n" if it isn't in atomic brackets or - brackets that may be referenced, as long as the pattern does not contain - *PRUNE or *SKIP, because these break the feature. Consider, for example, - /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e. not at the - start of a line. */ + brackets that may be referenced or an assertion, as long as the pattern does + not contain *PRUNE or *SKIP, because these break the feature. Consider, for + example, /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e. + not at the start of a line. */ else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR || op == OP_TYPEPOSSTAR) { if (scode[1] != OP_ANY || (bracket_map & cd->backref_map) != 0 || - atomcount > 0 || cd->had_pruneorskip) + atomcount > 0 || cd->had_pruneorskip || inassert) return FALSE; } @@ -9634,7 +9666,7 @@ if ((re->options & PCRE_ANCHORED) == 0) re->flags |= PCRE_FIRSTSET; } - else if (is_startline(codestart, 0, cd, 0)) re->flags |= PCRE_STARTLINE; + else if (is_startline(codestart, 0, cd, 0, FALSE)) re->flags |= PCRE_STARTLINE; } } diff --git a/pcre/pcre_jit_compile.c b/pcre/pcre_jit_compile.c index 4f15a27ac28..46ce6c65d54 100644 --- a/pcre/pcre_jit_compile.c +++ b/pcre/pcre_jit_compile.c @@ -4004,12 +4004,12 @@ sljit_emit_op_custom(compiler, instruction, 4); if (load_twice) { - OP1(SLJIT_MOV, TMP3, 0, TMP2, 0); + OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP2, 0); instruction[3] = 0xc0 | (tmp2_ind << 3) | 1; sljit_emit_op_custom(compiler, instruction, 4); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(SLJIT_MOV, TMP2, 0, TMP3, 0); + OP1(SLJIT_MOV, TMP2, 0, RETURN_ADDR, 0); } OP2(SLJIT_ASHR, TMP1, 0, TMP1, 0, TMP2, 0); diff --git a/pcre/pcre_jit_test.c b/pcre/pcre_jit_test.c index 9b61ec000fa..034cb52697f 100644 --- a/pcre/pcre_jit_test.c +++ b/pcre/pcre_jit_test.c @@ -687,6 +687,7 @@ static struct regression_test_case regression_test_cases[] = { { PCRE_FIRSTLINE | PCRE_NEWLINE_LF | PCRE_DOTALL, 0 | F_NOMATCH, "ab.", "ab" }, { MUA | PCRE_FIRSTLINE, 1 | F_NOMATCH, "^[a-d0-9]", "\nxx\nd" }, { PCRE_NEWLINE_ANY | PCRE_FIRSTLINE | PCRE_DOTALL, 0, "....a", "012\n0a" }, + { MUA | PCRE_FIRSTLINE, 0, "[aC]", "a" }, /* Recurse. */ { MUA, 0, "(a)(?1)", "aa" }, diff --git a/pcre/pcregrep.c b/pcre/pcregrep.c index cd53c648da2..fd2a67622ba 100644 --- a/pcre/pcregrep.c +++ b/pcre/pcregrep.c @@ -1803,6 +1803,12 @@ while (ptr < endptr) match = FALSE; if (line_buffered) fflush(stdout); rc = 0; /* Had some success */ + + /* If the current match ended past the end of the line (only possible + in multiline mode), we are done with this line. */ + + if ((unsigned int)offsets[1] > linelength) goto END_ONE_MATCH; + startoffset = offsets[1]; /* Restart after the match */ if (startoffset <= oldstartoffset) { diff --git a/pcre/pcretest.c b/pcre/pcretest.c index 78ef5177df7..5b73a918075 100644 --- a/pcre/pcretest.c +++ b/pcre/pcretest.c @@ -1982,6 +1982,7 @@ return(result); static int pchar(pcre_uint32 c, FILE *f) { int n = 0; +char tempbuffer[16]; if (PRINTOK(c)) { if (f != NULL) fprintf(f, "%c", c); @@ -2003,6 +2004,8 @@ if (c < 0x100) } if (f != NULL) n = fprintf(f, "\\x{%02x}", c); + else n = sprintf(tempbuffer, "\\x{%02x}", c); + return n >= 0 ? n : 0; } @@ -5042,7 +5045,7 @@ while (!done) if ((all_use_dfa || use_dfa) && find_match_limit) { - printf("**Match limit not relevant for DFA matching: ignored\n"); + printf("** Match limit not relevant for DFA matching: ignored\n"); find_match_limit = 0; } @@ -5255,10 +5258,17 @@ while (!done) if (do_allcaps) { - if (new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count) < 0) - goto SKIP_DATA; - count++; /* Allow for full match */ - if (count * 2 > use_size_offsets) count = use_size_offsets/2; + if (all_use_dfa || use_dfa) + { + fprintf(outfile, "** Show all captures ignored after DFA matching\n"); + } + else + { + if (new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count) < 0) + goto SKIP_DATA; + count++; /* Allow for full match */ + if (count * 2 > use_size_offsets) count = use_size_offsets/2; + } } /* Output the captured substrings. Note that, for the matched string, diff --git a/pcre/testdata/testinput1 b/pcre/testdata/testinput1 index 8379ce04d5b..93abab3c851 100644 --- a/pcre/testdata/testinput1 +++ b/pcre/testdata/testinput1 @@ -5733,4 +5733,10 @@ AbcdCBefgBhiBqz "(?|(\k'Pm')|(?'Pm'))" abcd +/(?=.*[A-Z])(?=.*[a-z])(?=.*[0-9])(?=.*[,;:])(?=.{8,16})(?!.*[\s])/ + \ Fred:099 + +/(?=.*X)X$/ + \ X + /-- End of testinput1 --/ diff --git a/pcre/testdata/testinput16 b/pcre/testdata/testinput16 index 15419e63fa6..7ccde0a8c80 100644 --- a/pcre/testdata/testinput16 +++ b/pcre/testdata/testinput16 @@ -38,4 +38,30 @@ /s+/i8SI SSss\x{17f} +/[\W\p{Any}]/BZ + abc + 123 + +/[\W\pL]/BZ + abc + ** Failers + 123 + +/[\D]/8 + \x{1d7cf} + +/[\D\P{Nd}]/8 + \x{1d7cf} + +/[^\D]/8 + a9b + ** Failers + \x{1d7cf} + +/[^\D\P{Nd}]/8 + a9b + \x{1d7cf} + ** Failers + \x{10000} + /-- End of testinput16 --/ diff --git a/pcre/testdata/testinput19 b/pcre/testdata/testinput19 index ce45afcb595..dfe8c7befb6 100644 --- a/pcre/testdata/testinput19 +++ b/pcre/testdata/testinput19 @@ -25,4 +25,21 @@ /s+/i8SI SSss\x{17f} +/[\D]/8 + \x{1d7cf} + +/[\D\P{Nd}]/8 + \x{1d7cf} + +/[^\D]/8 + a9b + ** Failers + \x{1d7cf} + +/[^\D\P{Nd}]/8 + a9b + \x{1d7cf} + ** Failers + \x{10000} + /-- End of testinput19 --/ diff --git a/pcre/testdata/testinput2 b/pcre/testdata/testinput2 index 75e402ee9b3..08c6f39a565 100644 --- a/pcre/testdata/testinput2 +++ b/pcre/testdata/testinput2 @@ -4243,4 +4243,10 @@ backtracking verbs. --/ /\N(?(?C)0?!.)*/ +/(?<RA>abc)(?(R)xyz)/BZ + +/(?<R>abc)(?(R)xyz)/BZ + +/(?=.*[A-Z])/I + /-- End of testinput2 --/ diff --git a/pcre/testdata/testinput6 b/pcre/testdata/testinput6 index a178d3d67b4..22ed1e64d57 100644 --- a/pcre/testdata/testinput6 +++ b/pcre/testdata/testinput6 @@ -1562,4 +1562,10 @@ \x{389} \x{20ac} +/(?=.*b)\pL/ + 11bb + +/(?(?=.*b)(?=.*b)\pL|.*c)/ + 11bb + /-- End of testinput6 --/ diff --git a/pcre/testdata/testinput7 b/pcre/testdata/testinput7 index 00b9738a371..f44a810f0f2 100644 --- a/pcre/testdata/testinput7 +++ b/pcre/testdata/testinput7 @@ -838,15 +838,6 @@ of case for anything other than the ASCII letters. --/ /^s?c/mi8I scat -/[\W\p{Any}]/BZ - abc - 123 - -/[\W\pL]/BZ - abc - ** Failers - 123 - /a[[:punct:]b]/WBZ /a[[:punct:]b]/8WBZ diff --git a/pcre/testdata/testinput8 b/pcre/testdata/testinput8 index 931dd717e74..7f8fa8292c5 100644 --- a/pcre/testdata/testinput8 +++ b/pcre/testdata/testinput8 @@ -4841,4 +4841,8 @@ bbb aaa +/()()a+/O= + aaa\D + a\D + /-- End of testinput8 --/ diff --git a/pcre/testdata/testoutput1 b/pcre/testdata/testoutput1 index e852ab9544c..a2b3cffe9d4 100644 --- a/pcre/testdata/testoutput1 +++ b/pcre/testdata/testoutput1 @@ -9434,4 +9434,12 @@ No match 0: 1: +/(?=.*[A-Z])(?=.*[a-z])(?=.*[0-9])(?=.*[,;:])(?=.{8,16})(?!.*[\s])/ + \ Fred:099 + 0: + +/(?=.*X)X$/ + \ X + 0: X + /-- End of testinput1 --/ diff --git a/pcre/testdata/testoutput16 b/pcre/testdata/testoutput16 index fd184cdbeee..e6ba26acfd4 100644 --- a/pcre/testdata/testoutput16 +++ b/pcre/testdata/testoutput16 @@ -138,4 +138,56 @@ Starting chars: S s \xc5 SSss\x{17f} 0: SSss\x{17f} +/[\W\p{Any}]/BZ +------------------------------------------------------------------ + Bra + [\x00-/:-@[-^`{-\xff\p{Any}] + Ket + End +------------------------------------------------------------------ + abc + 0: a + 123 + 0: 1 + +/[\W\pL]/BZ +------------------------------------------------------------------ + Bra + [\x00-/:-@[-^`{-\xff\p{L}] + Ket + End +------------------------------------------------------------------ + abc + 0: a + ** Failers + 0: * + 123 +No match + +/[\D]/8 + \x{1d7cf} + 0: \x{1d7cf} + +/[\D\P{Nd}]/8 + \x{1d7cf} + 0: \x{1d7cf} + +/[^\D]/8 + a9b + 0: 9 + ** Failers +No match + \x{1d7cf} +No match + +/[^\D\P{Nd}]/8 + a9b + 0: 9 + \x{1d7cf} + 0: \x{1d7cf} + ** Failers +No match + \x{10000} +No match + /-- End of testinput16 --/ diff --git a/pcre/testdata/testoutput19 b/pcre/testdata/testoutput19 index eb8a8f6cd34..982bea4c136 100644 --- a/pcre/testdata/testoutput19 +++ b/pcre/testdata/testoutput19 @@ -105,4 +105,30 @@ Starting chars: S s \xff SSss\x{17f} 0: SSss\x{17f} +/[\D]/8 + \x{1d7cf} + 0: \x{1d7cf} + +/[\D\P{Nd}]/8 + \x{1d7cf} + 0: \x{1d7cf} + +/[^\D]/8 + a9b + 0: 9 + ** Failers +No match + \x{1d7cf} +No match + +/[^\D\P{Nd}]/8 + a9b + 0: 9 + \x{1d7cf} + 0: \x{1d7cf} + ** Failers +No match + \x{10000} +No match + /-- End of testinput19 --/ diff --git a/pcre/testdata/testoutput2 b/pcre/testdata/testoutput2 index 5e88d1a7091..811bbefc84c 100644 --- a/pcre/testdata/testoutput2 +++ b/pcre/testdata/testoutput2 @@ -9380,7 +9380,7 @@ No need char /(?(?=.*b).*b|^d)/I Capturing subpattern count = 0 No options -First char at start or follows newline +No first char No need char /xyz/C @@ -14670,4 +14670,39 @@ No match /\N(?(?C)0?!.)*/ Failed: assertion expected after (?( or (?(?C) at offset 4 +/(?<RA>abc)(?(R)xyz)/BZ +------------------------------------------------------------------ + Bra + CBra 1 + abc + Ket + Cond + Cond recurse any + xyz + Ket + Ket + End +------------------------------------------------------------------ + +/(?<R>abc)(?(R)xyz)/BZ +------------------------------------------------------------------ + Bra + CBra 1 + abc + Ket + Cond + 1 Cond ref + xyz + Ket + Ket + End +------------------------------------------------------------------ + +/(?=.*[A-Z])/I +Capturing subpattern count = 0 +May match empty string +No options +No first char +No need char + /-- End of testinput2 --/ diff --git a/pcre/testdata/testoutput6 b/pcre/testdata/testoutput6 index b64dc0dc366..422d3833578 100644 --- a/pcre/testdata/testoutput6 +++ b/pcre/testdata/testoutput6 @@ -2573,4 +2573,12 @@ No match \x{20ac} No match +/(?=.*b)\pL/ + 11bb + 0: b + +/(?(?=.*b)(?=.*b)\pL|.*c)/ + 11bb + 0: b + /-- End of testinput6 --/ diff --git a/pcre/testdata/testoutput7 b/pcre/testdata/testoutput7 index fdfff646d3e..2b167b28d1c 100644 --- a/pcre/testdata/testoutput7 +++ b/pcre/testdata/testoutput7 @@ -2295,32 +2295,6 @@ Need char = 'c' (caseless) scat 0: sc -/[\W\p{Any}]/BZ ------------------------------------------------------------------- - Bra - [\x00-/:-@[-^`{-\xff\p{Any}] - Ket - End ------------------------------------------------------------------- - abc - 0: a - 123 - 0: 1 - -/[\W\pL]/BZ ------------------------------------------------------------------- - Bra - [\x00-/:-@[-^`{-\xff\p{L}] - Ket - End ------------------------------------------------------------------- - abc - 0: a - ** Failers - 0: * - 123 -No match - /a[[:punct:]b]/WBZ ------------------------------------------------------------------ Bra diff --git a/pcre/testdata/testoutput8 b/pcre/testdata/testoutput8 index e4fa4977561..17b667a980c 100644 --- a/pcre/testdata/testoutput8 +++ b/pcre/testdata/testoutput8 @@ -7791,4 +7791,14 @@ Matched, but offsets vector is too small to show all matches aaa No match +/()()a+/O= + aaa\D +** Show all captures ignored after DFA matching + 0: aaa + 1: aa + 2: a + a\D +** Show all captures ignored after DFA matching + 0: a + /-- End of testinput8 --/ diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index d48b6c37728..de0b7bfae8e 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -15,7 +15,7 @@ #define PLUGIN_VERSION 0x104 -#define PLUGIN_STR_VERSION "1.4.0" +#define PLUGIN_STR_VERSION "1.4.1" #define _my_thread_var loc_thread_var @@ -156,10 +156,8 @@ static File loc_open(const char *FileName, int Flags) File fd; #if defined(_WIN32) fd= my_win_open(FileName, Flags); -#elif !defined(NO_OPEN_3) - fd = open(FileName, Flags, my_umask); /* Normal unix */ #else - fd = open((char *) FileName, Flags); + fd = open(FileName, Flags, my_umask); #endif my_errno= errno; return fd; @@ -2295,10 +2293,10 @@ typedef struct loc_system_variables } LOC_SV; +static int init_done= 0; + static int server_audit_init(void *p __attribute__((unused))) { - const void *my_hash_init_ptr; - if (!serv_ver) { #ifdef _WIN32 @@ -2307,11 +2305,16 @@ static int server_audit_init(void *p __attribute__((unused))) serv_ver= server_version; #endif /*_WIN32*/ } - my_hash_init_ptr= dlsym(RTLD_DEFAULT, "_my_hash_init"); - if (!my_hash_init_ptr) + if (!mysql_57_started) { - maria_above_5= 1; - my_hash_init_ptr= dlsym(RTLD_DEFAULT, "my_hash_init2"); + const void *my_hash_init_ptr= dlsym(RTLD_DEFAULT, "_my_hash_init"); + if (!my_hash_init_ptr) + { + maria_above_5= 1; + my_hash_init_ptr= dlsym(RTLD_DEFAULT, "my_hash_init2"); + } + if (!my_hash_init_ptr) + return 1; } if(!(int_mysql_data_home= dlsym(RTLD_DEFAULT, "mysql_data_home"))) @@ -2320,7 +2323,7 @@ static int server_audit_init(void *p __attribute__((unused))) int_mysql_data_home= &default_home; } - if (!serv_ver || !my_hash_init_ptr) + if (!serv_ver) return 1; if (!started_mysql) @@ -2400,6 +2403,7 @@ static int server_audit_init(void *p __attribute__((unused))) if (logging) start_logging(); + init_done= 1; return 0; } @@ -2415,6 +2419,10 @@ static int server_audit_init_mysql(void *p) static int server_audit_deinit(void *p __attribute__((unused))) { + if (!init_done) + return 0; + + init_done= 0; coll_free(&incl_user_coll); coll_free(&excl_user_coll); @@ -2837,12 +2845,15 @@ void __attribute__ ((constructor)) audit_plugin_so_init(void) if (sc >= 24) use_event_data_for_disconnect= 1; } - else if (serv_ver[0] == '5' && serv_ver[2] == '7') + else if ((serv_ver[0] == '5' && serv_ver[2] == '7') || + (serv_ver[0] == '8' && serv_ver[2] == '0')) { mysql_57_started= 1; _mysql_plugin_declarations_[0].info= mysql_v4_descriptor; use_event_data_for_disconnect= 1; } + MYSQL_SYSVAR_NAME(loc_info).flags= PLUGIN_VAR_STR | PLUGIN_VAR_THDLOCAL | + PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC; } memset(locinfo_ini_value, 'O', sizeof(locinfo_ini_value)-1); diff --git a/scripts/mysql_secure_installation.sh b/scripts/mysql_secure_installation.sh index 2c3b2f8dc8c..57e4d43dfad 100644 --- a/scripts/mysql_secure_installation.sh +++ b/scripts/mysql_secure_installation.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2002, 2012, Oracle and/or its affiliates. +# Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -194,6 +194,20 @@ set_echo_compat() { esac } +validate_reply () { + ret=0 + if [ -z "$1" ]; then + reply=y + return $ret + fi + case $1 in + y|Y|yes|Yes|YES) reply=y ;; + n|N|no|No|NO) reply=n ;; + *) ret=1 ;; + esac + return $ret +} + prepare() { touch $config $command chmod 600 $config $command @@ -407,15 +421,18 @@ echo "Setting the root password ensures that nobody can log into the MariaDB" echo "root user without the proper authorisation." echo -if [ $hadpass -eq 0 ]; then - echo $echo_n "Set root password? [Y/n] $echo_c" -else - echo "You already have a root password set, so you can safely answer 'n'." - echo - echo $echo_n "Change the root password? [Y/n] $echo_c" -fi +while true ; do + if [ $hadpass -eq 0 ]; then + echo $echo_n "Set root password? [Y/n] $echo_c" + else + echo "You already have a root password set, so you can safely answer 'n'." + echo + echo $echo_n "Change the root password? [Y/n] $echo_c" + fi + read reply + validate_reply $reply && break +done -read reply if [ "$reply" = "n" ]; then echo " ... skipping." else @@ -439,9 +456,11 @@ echo "go a bit smoother. You should remove them before moving into a" echo "production environment." echo -echo $echo_n "Remove anonymous users? [Y/n] $echo_c" - -read reply +while true ; do + echo $echo_n "Remove anonymous users? [Y/n] $echo_c" + read reply + validate_reply $reply && break +done if [ "$reply" = "n" ]; then echo " ... skipping." else @@ -457,9 +476,11 @@ echo echo "Normally, root should only be allowed to connect from 'localhost'. This" echo "ensures that someone cannot guess at the root password from the network." echo - -echo $echo_n "Disallow root login remotely? [Y/n] $echo_c" -read reply +while true ; do + echo $echo_n "Disallow root login remotely? [Y/n] $echo_c" + read reply + validate_reply $reply && break +done if [ "$reply" = "n" ]; then echo " ... skipping." else @@ -477,8 +498,12 @@ echo "access. This is also intended only for testing, and should be removed" echo "before moving into a production environment." echo -echo $echo_n "Remove test database and access to it? [Y/n] $echo_c" -read reply +while true ; do + echo $echo_n "Remove test database and access to it? [Y/n] $echo_c" + read reply + validate_reply $reply && break +done + if [ "$reply" = "n" ]; then echo " ... skipping." else @@ -495,8 +520,12 @@ echo "Reloading the privilege tables will ensure that all changes made so far" echo "will take effect immediately." echo -echo $echo_n "Reload privilege tables now? [Y/n] $echo_c" -read reply +while true ; do + echo $echo_n "Reload privilege tables now? [Y/n] $echo_c" + read reply + validate_reply $reply && break +done + if [ "$reply" = "n" ]; then echo " ... skipping." else diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 8585edd5cb2..da5906138de 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -20,6 +20,7 @@ mysqld_ld_preload= mysqld_ld_library_path= flush_caches=0 numa_interleave=0 +unsafe_my_cnf=0 # Initial logging status: error log is not open, and not using syslog logging=init @@ -129,6 +130,18 @@ my_which () return $ret # Success } +find_in_bin() { + if test -x "$MY_BASEDIR_VERSION/bin/$1" + then + echo "$MY_BASEDIR_VERSION/bin/$1" + elif test -x "@bindir@/$1" + then + echo "@bindir@/$1" + else + echo "$1" + fi +} + log_generic () { priority="$1" shift @@ -137,7 +150,7 @@ log_generic () { echo "$msg" case $logging in init) ;; # Just echo the message, don't save it anywhere - file) echo "$msg" >> "$err_log" ;; + file) echo "$msg" | "$helper" "$user" log "$err_log" ;; syslog) logger -t "$syslog_tag_mysqld_safe" -p "$priority" "$*" ;; *) echo "Internal program error (non-fatal):" \ @@ -157,7 +170,7 @@ log_notice () { eval_log_error () { cmd="$1" case $logging in - file) cmd="$cmd >> "`shell_quote_string "$err_log"`" 2>&1" ;; + file) cmd="$cmd 2>&1 | "`shell_quote_string "$helper"`" $user log "`shell_quote_string "$err_log"` ;; syslog) # mysqld often prefixes its messages with a timestamp, which is # redundant when logging to syslog (which adds its own timestamp) @@ -191,11 +204,17 @@ shell_quote_string() { echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g' } +check_executable_location() { + if test "$unsafe_my_cnf" = 1 -a "$unrecognized_handling" != collect; then + log_error "Cannot accept $1 from a config file, when my.cnf is in the datadir" + exit 1 + fi +} + parse_arguments() { for arg do val=`echo "$arg" | sed -e "s;--[^=]*=;;"` case "$arg" in - --crash[-_]script=*) CRASH_SCRIPT="$val" ;; # these get passed explicitly to mysqld --basedir=*) MY_BASEDIR_VERSION="$val" ;; --datadir=*|--data=*) DATADIR="$val" ;; @@ -222,12 +241,14 @@ parse_arguments() { # mysqld_safe-specific options - must be set in my.cnf ([mysqld_safe])! --core[-_]file[-_]size=*) core_file_size="$val" ;; - --ledir=*) ledir="$val" ;; - --malloc[-_]lib=*) set_malloc_lib "$val" ;; - --mysqld=*) MYSQLD="$val" ;; + --ledir=*) check_executable_location "$arg" ; ledir="$val" ;; + --malloc[-_]lib=*) check_executable_location "$arg"; set_malloc_lib "$val" ;; + --crash[-_]script=*) check_executable_location "$arg"; crash_script="$val" ;; + --mysqld=*) check_executable_location "$arg"; MYSQLD="$val" ;; --mysqld[-_]version=*) if test -n "$val" then + check_executable_location "$arg" MYSQLD="mysqld-$val" PLUGIN_VARIANT="/$val" else @@ -386,15 +407,8 @@ set_malloc_lib() { # First, try to find BASEDIR and ledir (where mysqld is) # -if echo '@pkgdatadir@' | grep '^@prefix@' > /dev/null -then - relpkgdata=`echo '@pkgdatadir@' | sed -e 's,^@prefix@,,' -e 's,^/,,' -e 's,^,./,'` -else - # pkgdatadir is not relative to prefix - relpkgdata='@pkgdatadir@' -fi - -MY_PWD=`pwd` +MY_PWD=`dirname $0` +MY_PWD=`cd "$MY_PWD"/.. && pwd` # Check for the directories we would expect from a binary release install if test -n "$MY_BASEDIR_VERSION" -a -d "$MY_BASEDIR_VERSION" then @@ -410,16 +424,16 @@ then else ledir="$MY_BASEDIR_VERSION/bin" fi -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/bin/mysqld" +elif test -x "$MY_PWD/bin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where bin, share and data are ledir="$MY_PWD/bin" # Where mysqld is # Check for the directories we would expect from a source install -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/libexec/mysqld" +elif test -x "$MY_PWD/libexec/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where libexec, share and var are ledir="$MY_PWD/libexec" # Where mysqld is -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/sbin/mysqld" +elif test -x "$MY_PWD/sbin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where sbin, share and var are ledir="$MY_PWD/sbin" # Where mysqld is @@ -429,6 +443,8 @@ else ledir='@libexecdir@' fi +helper=`find_in_bin mysqld_safe_helper` +print_defaults=`find_in_bin my_print_defaults` # # Second, try to find the data directory @@ -466,6 +482,7 @@ IGNORING $DATADIR/my.cnf" log_error "WARNING: Found $DATADIR/my.cnf The data directory is a deprecated location for my.cnf, please move it to $MY_BASEDIR_VERSION/my.cnf" + unsafe_my_cnf=1 MYSQL_HOME=$DATADIR else MYSQL_HOME=$MY_BASEDIR_VERSION @@ -473,34 +490,15 @@ $MY_BASEDIR_VERSION/my.cnf" fi export MYSQL_HOME - -# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] -# and then merge with the command line arguments -if test -x "$MY_BASEDIR_VERSION/bin/my_print_defaults" -then - print_defaults="$MY_BASEDIR_VERSION/bin/my_print_defaults" -elif test -x `dirname $0`/my_print_defaults -then - print_defaults="`dirname $0`/my_print_defaults" -elif test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x @bindir@/my_print_defaults -then - print_defaults="@bindir@/my_print_defaults" -elif test -x @bindir@/mysql_print_defaults -then - print_defaults="@bindir@/mysql_print_defaults" -else - print_defaults="my_print_defaults" -fi - append_arg_to_args () { args="$args "`shell_quote_string "$1"` } args= +# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] +# and then merge with the command line arguments + SET_USER=2 parse_arguments `$print_defaults $defaults --loose-verbose --mysqld` if test $SET_USER -eq 2 @@ -609,11 +607,6 @@ then log_notice "Logging to '$err_log'." logging=file - if [ ! -f "$err_log" ]; then # if error log already exists, - touch "$err_log" # we just append. otherwise, - chmod "$fmode" "$err_log" # fix the permissions here! - fi - else if [ -n "$syslog_tag" ] then @@ -626,10 +619,6 @@ else logging=syslog fi -# close stdout and stderr, everything goes to $logging now -exec 1>&- -exec 2>&- - USER_OPTION="" if test -w / -o "$USER" = "root" then @@ -637,11 +626,6 @@ then then USER_OPTION="--user=$user" fi - # Change the err log to the right user, if it is in use - if [ $want_syslog -eq 0 ]; then - touch "$err_log" - chown $user "$err_log" - fi if test -n "$open_files" then ulimit -n $open_files @@ -885,6 +869,15 @@ max_fast_restarts=5 # flag whether a usable sleep command exists have_sleep=1 +# close stdout and stderr, everything goes to $logging now +if expr "${-}" : '.*x' > /dev/null +then + : +else + exec 1>&- + exec 2>&- +fi + while true do rm -f "$pid_file" # Some extra safety @@ -892,13 +885,6 @@ do start_time=`date +%M%S` eval_log_error "$cmd" - - if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then - touch "$err_log" # hypothetical: log was renamed but not - chown $user "$err_log" # flushed yet. we'd recreate it with - chmod "$fmode" "$err_log" # wrong owner next time we log, so set - fi # it up correctly while we can! - end_time=`date +%M%S` if test ! -f "$pid_file" # This is removed if normal shutdown @@ -962,12 +948,11 @@ do done fi log_notice "mysqld restarted" - if test -n "$CRASH_SCRIPT" + if test -n "$crash_script" then - crash_script_output=`$CRASH_SCRIPT 2>&1` + crash_script_output=`$crash_script 2>&1` log_error "$crash_script_output" fi done log_notice "mysqld from pid file $pid_file ended" - diff --git a/sql-common/client.c b/sql-common/client.c index 78c5426367c..6c1627ecb71 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1,5 +1,5 @@ /* Copyright (c) 2003, 2016, Oracle and/or its affiliates. - Copyright (c) 2009, 2016, MariaDB + Copyright (c) 2009, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1678,8 +1678,8 @@ mysql_ssl_set(MYSQL *mysql __attribute__((unused)) , mysql_options(mysql, MYSQL_OPT_SSL_CAPATH, capath) | mysql_options(mysql, MYSQL_OPT_SSL_CIPHER, cipher) ? 1 : 0); - mysql->options.use_ssl= TRUE; #endif /* HAVE_OPENSSL && !EMBEDDED_LIBRARY */ + mysql->options.use_ssl= TRUE; DBUG_RETURN(result); } @@ -2540,7 +2540,6 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, int3store(buff+2, net->max_packet_size); end= buff+5; } -#ifdef HAVE_OPENSSL /* If client uses ssl and client also has to verify the server @@ -2558,6 +2557,7 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, goto error; } +#ifdef HAVE_OPENSSL if (mysql->client_flag & CLIENT_SSL) { /* Do the SSL layering. */ @@ -3867,8 +3867,6 @@ static void mysql_close_free(MYSQL *mysql) static void mysql_prune_stmt_list(MYSQL *mysql) { LIST *element= mysql->stmts; - LIST *pruned_list= 0; - for (; element; element= element->next) { MYSQL_STMT *stmt= (MYSQL_STMT *) element->data; @@ -3878,14 +3876,9 @@ static void mysql_prune_stmt_list(MYSQL *mysql) stmt->last_errno= CR_SERVER_LOST; strmov(stmt->last_error, ER(CR_SERVER_LOST)); strmov(stmt->sqlstate, unknown_sqlstate); - } - else - { - pruned_list= list_add(pruned_list, element); + mysql->stmts= list_delete(mysql->stmts, element); } } - - mysql->stmts= pruned_list; } diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 88ec26eb6f6..2e581b1acbe 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -150,6 +150,63 @@ ELSE() SET(MYSQLD_SOURCE main.cc ${DTRACE_PROBES_ALL}) ENDIF() + +IF(MSVC AND NOT WITHOUT_DYNAMIC_PLUGINS) + + # mysqld.exe must to export symbols from some specific libs. + # These symbols are used by dynamic plugins, that "link" to mysqld. + # + # To do that, we + # + # 1. Generate mysqld_lib.def text file with all symbols from static + # libraries mysys, dbug, strings, sql. + # 2. Then we call + # lib.exe /DEF:mysqld_lib.def ... + # to create import library mysqld_lib.lib and export library mysqld_lib.exp + # 3. mysqld.exe links with mysqld_lib.exp (exporting symbols) + # 4. plugins link with mysqld_lib.lib (importing symbols) + # + # We do not not regenerate .def, .lib and .exp + # without necessity.E.g source modifications, that do not + # change list of exported symbols, will not result in a relink for plugins. + + SET(MYSQLD_DEF ${CMAKE_CURRENT_BINARY_DIR}/mysqld_lib.def) + SET(MYSQLD_EXP ${CMAKE_CURRENT_BINARY_DIR}/mysqld_lib.exp) + SET(MYSQLD_LIB ${CMAKE_CURRENT_BINARY_DIR}/mysqld_lib.lib) + SET(MYSQLD_CORELIBS sql mysys mysys_ssl dbug strings) + FOREACH (CORELIB ${MYSQLD_CORELIBS}) + GET_TARGET_PROPERTY(LOC ${CORELIB} LOCATION) + FILE(TO_NATIVE_PATH ${LOC} LOC) + SET (LIB_LOCATIONS ${LIB_LOCATIONS} ${LOC}) + ENDFOREACH (CORELIB) + + SET(_PLATFORM x86) + IF(CMAKE_SIZEOF_VOID_P EQUAL 8) + SET(_PLATFORM x64) + ENDIF() + + ADD_CUSTOM_COMMAND( + OUTPUT ${MYSQLD_DEF} + COMMAND cscript ARGS //nologo ${PROJECT_SOURCE_DIR}/win/create_def_file.js + ${_PLATFORM} /forLib ${LIB_LOCATIONS} > mysqld_lib.def.tmp + COMMAND ${CMAKE_COMMAND} -E copy_if_different mysqld_lib.def.tmp mysqld_lib.def + COMMAND ${CMAKE_COMMAND} -E remove mysqld_lib.def.tmp + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${MYSQLD_CORELIBS} + ) + + ADD_CUSTOM_COMMAND( + OUTPUT ${MYSQLD_LIB} + COMMAND lib + ARGS /NAME:mysqld.exe "/DEF:${MYSQLD_DEF}" "/MACHINE:${_PLATFORM}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${MYSQLD_DEF} + ) + ADD_CUSTOM_TARGET(gen_mysqld_lib DEPENDS ${MYSQLD_LIB}) + ADD_LIBRARY(mysqld_import_lib UNKNOWN IMPORTED GLOBAL) + SET_TARGET_PROPERTIES(mysqld_import_lib PROPERTIES IMPORTED_LOCATION ${MYSQLD_LIB}) +ENDIF() + MYSQL_ADD_EXECUTABLE(mysqld ${MYSQLD_SOURCE} DESTINATION ${INSTALL_SBINDIR} COMPONENT Server) IF(APPLE) @@ -170,25 +227,9 @@ IF(NOT WITHOUT_DYNAMIC_PLUGINS) SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS "${mysqld_link_flags} -Wl,--export-all-symbols") ENDIF() IF(MSVC) - # Set module definition file. Also use non-incremental linker, - # incremental appears to crash from time to time,if used with /DEF option - SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS "${mysqld_link_flags} /DEF:mysqld.def /INCREMENTAL:NO") - - FOREACH (CORELIB sql mysys mysys_ssl dbug strings) - GET_TARGET_PROPERTY(LOC ${CORELIB} LOCATION) - FILE(TO_NATIVE_PATH ${LOC} LOC) - SET (LIB_LOCATIONS ${LIB_LOCATIONS} ${LOC}) - ENDFOREACH (CORELIB ${MYSQLD_CORE_LIBS}) - SET(_PLATFORM x86) - IF(CMAKE_SIZEOF_VOID_P EQUAL 8) - SET(_PLATFORM x64) - ENDIF() - ADD_CUSTOM_COMMAND(TARGET mysqld PRE_LINK - COMMAND echo ${_PLATFORM} && cscript ARGS //nologo ${PROJECT_SOURCE_DIR}/win/create_def_file.js - ${_PLATFORM} ${LIB_LOCATIONS} > mysqld.def - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - ADD_DEPENDENCIES(sql GenError) - ENDIF(MSVC) + SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS "${mysqld_link_flags} \"${MYSQLD_EXP}\"") + ADD_DEPENDENCIES(mysqld gen_mysqld_lib) + ENDIF() ENDIF(NOT WITHOUT_DYNAMIC_PLUGINS) SET_TARGET_PROPERTIES(mysqld PROPERTIES ENABLE_EXPORTS TRUE) @@ -387,6 +428,7 @@ IF(WIN32) ${CMAKE_CURRENT_BINARY_DIR}/mysql_bootstrap_sql.c COMPONENT Server ) + SET_TARGET_PROPERTIES(mysql_install_db PROPERTIES COMPILE_FLAGS -DINSTALL_PLUGINDIR=${INSTALL_PLUGINDIR}) TARGET_LINK_LIBRARIES(mysql_install_db mysys) ADD_LIBRARY(winservice STATIC winservice.c) diff --git a/sql/contributors.h b/sql/contributors.h index 0359ec54022..3a771e2b493 100644 --- a/sql/contributors.h +++ b/sql/contributors.h @@ -37,16 +37,18 @@ struct show_table_contributors_st { struct show_table_contributors_st show_table_contributors[]= { /* MariaDB foundation sponsors, in contribution, size , time order */ - {"Booking.com", "http://www.booking.com", "Founding member, Platinum Sponsor of the MariaDB Foundation"}, + {"Booking.com", "https://www.booking.com", "Founding member, Platinum Sponsor of the MariaDB Foundation"}, + {"Alibaba Cloud", "https://intl.aliyun.com", "Platinum Sponsor of the MariaDB Foundation"}, {"MariaDB Corporation", "https://mariadb.com", "Founding member, Gold Sponsor of the MariaDB Foundation"}, - {"Visma", "http://visma.com", "Gold Sponsor of the MariaDB Foundation"}, - {"DBS", "http://dbs.com", "Gold Sponsor of the MariaDB Foundation"}, + {"Visma", "https://visma.com", "Gold Sponsor of the MariaDB Foundation"}, + {"DBS", "https://dbs.com", "Gold Sponsor of the MariaDB Foundation"}, {"Nexedi", "https://www.nexedi.com", "Silver Sponsor of the MariaDB Foundation"}, {"Acronis", "http://www.acronis.com", "Silver Sponsor of the MariaDB Foundation"}, {"Auttomattic", "https://automattic.com", "Bronze Sponsor of the MariaDB Foundation"}, - {"Verkkokauppa.com", "https://virtuozzo.com", "Bronze Sponsor of the MariaDB Foundation"}, - {"Virtuozzo", "https://virtuozzo.com/", "Bronze Sponsor of the MariaDB Foundation"}, - {"Tencent Game DBA", "http://tencentdba.com/about/", "Bronze Sponsor of the MariaDB Foundation"}, + {"Verkkokauppa.com", "https://www.verkkokauppa.com", "Bronze Sponsor of the MariaDB Foundation"}, + {"Virtuozzo", "https://virtuozzo.com", "Bronze Sponsor of the MariaDB Foundation"}, + {"Tencent Game DBA", "http://tencentdba.com/about", "Bronze Sponsor of the MariaDB Foundation"}, + {"Tencent TDSQL", "http://tdsql.org", "Bronze Sponsor of the MariaDB Foundation"}, /* Sponsors of important features */ {"Google", "USA", "Sponsoring encryption, parallel replication and GTID"}, diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc index 6785b46d0da..66606b18c49 100644 --- a/sql/event_db_repository.cc +++ b/sql/event_db_repository.cc @@ -18,6 +18,7 @@ #include "sql_priv.h" #include "unireg.h" #include "sql_base.h" // close_thread_tables +#include "sql_parse.h" #include "event_db_repository.h" #include "key.h" // key_copy #include "sql_db.h" // get_default_db_collation @@ -704,19 +705,17 @@ Event_db_repository::create_event(THD *thd, Event_parse_data *parse_data, restore_record(table, s->default_values); // Get default values for fields - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->dbname.str, - parse_data->dbname.str + parse_data->dbname.length) > - table->field[ET_FIELD_DB]->char_length()) + if (check_string_char_length(&parse_data->dbname, 0, + table->field[ET_FIELD_DB]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->dbname.str); goto end; } - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->name.str, - parse_data->name.str + parse_data->name.length) > - table->field[ET_FIELD_NAME]->char_length()) + if (check_string_char_length(&parse_data->name, 0, + table->field[ET_FIELD_NAME]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->name.str); goto end; diff --git a/sql/event_scheduler.cc b/sql/event_scheduler.cc index 5c4926c830c..2423fac4ca6 100644 --- a/sql/event_scheduler.cc +++ b/sql/event_scheduler.cc @@ -133,12 +133,6 @@ post_init_event_thread(THD *thd) thd->cleanup(); return TRUE; } - - thread_safe_increment32(&thread_count, &thread_count_lock); - mysql_mutex_lock(&LOCK_thread_count); - threads.append(thd); - mysql_mutex_unlock(&LOCK_thread_count); - inc_thread_running(); return FALSE; } @@ -157,7 +151,13 @@ deinit_event_thread(THD *thd) thd->proc_info= "Clearing"; DBUG_PRINT("exit", ("Event thread finishing")); - delete_running_thd(thd); + mysql_mutex_lock(&LOCK_thread_count); + thd->unlink(); + mysql_mutex_unlock(&LOCK_thread_count); + + delete thd; + thread_safe_decrement32(&thread_count, &thread_count_lock); + signal_thd_deleted(); } @@ -191,8 +191,10 @@ pre_init_event_thread(THD* thd) thd->net.read_timeout= slave_net_timeout; thd->variables.option_bits|= OPTION_AUTO_IS_NULL; thd->client_capabilities|= CLIENT_MULTI_RESULTS; + thread_safe_increment32(&thread_count, &thread_count_lock); mysql_mutex_lock(&LOCK_thread_count); thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; + threads.append(thd); mysql_mutex_unlock(&LOCK_thread_count); /* @@ -240,13 +242,8 @@ event_scheduler_thread(void *arg) my_free(arg); if (!res) scheduler->run(thd); - else - { - thd->proc_info= "Clearing"; - net_end(&thd->net); - delete thd; - } + deinit_event_thread(thd); DBUG_LEAVE; // Against gcc warnings my_thread_end(); return 0; @@ -310,6 +307,7 @@ Event_worker_thread::run(THD *thd, Event_queue_element_for_exec *event) DBUG_ENTER("Event_worker_thread::run"); DBUG_PRINT("info", ("Time is %ld, THD: 0x%lx", (long) my_time(0), (long) thd)); + inc_thread_running(); if (res) goto end; @@ -338,6 +336,7 @@ end: event->name.str)); delete event; + dec_thread_running(); deinit_event_thread(thd); DBUG_VOID_RETURN; @@ -442,13 +441,9 @@ Event_scheduler::start(int *err_no) " Can not create thread for event scheduler (errno=%d)", *err_no); - new_thd->proc_info= "Clearing"; - DBUG_ASSERT(new_thd->net.buff != 0); - net_end(&new_thd->net); - state= INITIALIZED; scheduler_thd= NULL; - delete new_thd; + deinit_event_thread(new_thd); delete scheduler_param_value; ret= true; @@ -515,7 +510,6 @@ Event_scheduler::run(THD *thd) } LOCK_DATA(); - deinit_event_thread(thd); scheduler_thd= NULL; state= INITIALIZED; DBUG_PRINT("info", ("Broadcasting COND_state back to the stoppers")); @@ -575,10 +569,7 @@ Event_scheduler::execute_top(Event_queue_element_for_exec *event_name) sql_print_error("Event_scheduler::execute_top: Can not create event worker" " thread (errno=%d). Stopping event scheduler", res); - new_thd->proc_info= "Clearing"; - DBUG_ASSERT(new_thd->net.buff != 0); - net_end(&new_thd->net); - + deinit_event_thread(new_thd); goto error; } @@ -590,9 +581,6 @@ Event_scheduler::execute_top(Event_queue_element_for_exec *event_name) error: DBUG_PRINT("error", ("Event_scheduler::execute_top() res: %d", res)); - if (new_thd) - delete new_thd; - delete event_name; DBUG_RETURN(TRUE); } diff --git a/sql/field.cc b/sql/field.cc index 49989a4a4ed..dec89e16674 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1269,12 +1269,15 @@ void Field_num::prepend_zeros(String *value) int diff; if ((diff= (int) (field_length - value->length())) > 0) { - bmove_upp((uchar*) value->ptr()+field_length, - (uchar*) value->ptr()+value->length(), - value->length()); - bfill((uchar*) value->ptr(),diff,'0'); - value->length(field_length); - (void) value->c_ptr_quick(); // Avoid warnings in purify + const bool error= value->realloc(field_length); + if (!error) + { + bmove_upp((uchar*) value->ptr()+field_length, + (uchar*) value->ptr()+value->length(), + value->length()); + bfill((uchar*) value->ptr(),diff,'0'); + value->length(field_length); + } } } diff --git a/sql/filesort.cc b/sql/filesort.cc index 6ad7bee48c6..73a6c89e53f 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -1629,7 +1629,7 @@ int merge_buffers(Sort_param *param, IO_CACHE *from_file, if (!(error= (int) read_to_buffer(from_file, buffpek, rec_length))) { - queue_remove(&queue,0); + (void) queue_remove_top(&queue); reuse_freed_buff(&queue, buffpek, rec_length); } else if (error == -1) diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 69cc78588a8..86ea5a22a5b 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -7273,6 +7273,7 @@ int ha_partition::reset(void) result= tmp; } bitmap_clear_all(&m_partitions_to_reset); + m_extra_prepare_for_update= FALSE; DBUG_RETURN(result); } diff --git a/sql/handler.cc b/sql/handler.cc index 8298d286259..e51f17f1712 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3371,6 +3371,8 @@ void handler::print_error(int error, myf errflag) textno=ER_FILE_USED; break; case ENOENT: + case ENOTDIR: + case ELOOP: textno=ER_FILE_NOT_FOUND; break; case ENOSPC: @@ -3840,7 +3842,6 @@ int handler::delete_table(const char *name) int saved_error= 0; int error= 0; int enoent_or_zero; - char buff[FN_REFLEN]; if (ht->discover_table) enoent_or_zero= 0; // the table may not exist in the engine, it's ok @@ -3849,8 +3850,7 @@ int handler::delete_table(const char *name) for (const char **ext=bas_ext(); *ext ; ext++) { - fn_format(buff, name, "", *ext, MY_UNPACK_FILENAME|MY_APPEND_EXT); - if (mysql_file_delete_with_symlink(key_file_misc, buff, MYF(0))) + if (mysql_file_delete_with_symlink(key_file_misc, name, *ext, 0)) { if (my_errno != ENOENT) { @@ -4209,7 +4209,7 @@ enum_alter_inplace_result handler::check_if_supported_inplace_alter(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { - DBUG_ENTER("check_if_supported_alter"); + DBUG_ENTER("handler::check_if_supported_inplace_alter"); HA_CREATE_INFO *create_info= ha_alter_info->create_info; @@ -4524,7 +4524,7 @@ void handler::get_dynamic_partition_info(PARTITION_STATS *stat_info, stat_info->update_time= stats.update_time; stat_info->check_time= stats.check_time; stat_info->check_sum= 0; - if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM)) + if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) stat_info->check_sum= checksum(); return; } @@ -5944,7 +5944,7 @@ int handler::ha_external_lock(THD *thd, int lock_type) MYSQL_TABLE_LOCK_WAIT(m_psi, PSI_TABLE_EXTERNAL_LOCK, lock_type, { error= external_lock(thd, lock_type); }) - if (error == 0) + if (error == 0 || lock_type == F_UNLCK) { m_lock_type= lock_type; cached_table_flags= table_flags(); diff --git a/sql/item.cc b/sql/item.cc index ecdb3a94f7f..ceb8d984d9e 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1206,7 +1206,8 @@ Item *Item_cache::safe_charset_converter(CHARSET_INFO *tocs) if (conv == example) return this; Item_cache *cache; - if (!conv || !(cache= new Item_cache_str(conv))) + if (!conv || conv->fix_fields(current_thd, (Item **) NULL) || + !(cache= new Item_cache_str(conv))) return NULL; // Safe conversion is not possible, or OEM cache->setup(conv); cache->fixed= false; // Make Item::fix_fields() happy @@ -2742,6 +2743,44 @@ void Item_field::fix_after_pullout(st_select_lex *new_parent, Item **ref) depended_from= NULL; if (context) { + bool need_change= false; + /* + Suppose there are nested selects: + + select_id=1 + select_id=2 + select_id=3 <----+ + select_id=4 -+ + select_id=5 --+ + + Suppose, pullout operation has moved anything that had select_id=4 or 5 + in to select_id=3. + + If this Item_field had a name resolution context pointing into select_lex + with id=4 or id=5, it needs a new name resolution context. + + However, it could also be that this object is a part of outer reference: + Item_ref(Item_field(field in select with select_id=1))). + - The Item_ref object has a context with select_id=5, and so needs a new + name resolution context. + - The Item_field object has a context with select_id=1, and doesn't need + a new name resolution context. + + So, the following loop walks from Item_field's current context upwards. + If we find that the select we've been pulled out to is up there, we + create the new name resolution context. Otherwise, we don't. + */ + for (Name_resolution_context *ct= context; ct; ct= ct->outer_context) + { + if (new_parent == ct->select_lex) + { + need_change= true; + break; + } + } + if (!need_change) + return; + Name_resolution_context *ctx= new Name_resolution_context(); if (context->select_lex == new_parent) { @@ -7102,19 +7141,6 @@ bool Item_ref::fix_fields(THD *thd, Item **reference) last_checked_context->select_lex->nest_level); } } - else if (ref_type() != VIEW_REF) - { - /* - It could be that we're referring to something that's in ancestor selects. - We must make an appropriate mark_as_dependent() call for each such - outside reference. - */ - Dependency_marker dep_marker; - dep_marker.current_select= current_sel; - dep_marker.thd= thd; - (*ref)->walk(&Item::enumerate_field_refs_processor, FALSE, - (uchar*)&dep_marker); - } DBUG_ASSERT(*ref); /* @@ -9438,7 +9464,10 @@ void Item_cache_row::set_null() Item_type_holder::Item_type_holder(THD *thd, Item *item) - :Item(thd, item), enum_set_typelib(0), fld_type(get_real_type(item)) + :Item(thd, item), + enum_set_typelib(0), + fld_type(get_real_type(item)), + geometry_type(Field::GEOM_GEOMETRY) { DBUG_ASSERT(item->fixed); maybe_null= item->maybe_null; diff --git a/sql/item.h b/sql/item.h index 2f8607d8fad..4d33a0eb6c1 100644 --- a/sql/item.h +++ b/sql/item.h @@ -4116,6 +4116,11 @@ public: virtual double val_real() = 0; virtual longlong val_int() = 0; virtual int save_in_field(Field *field, bool no_conversions) = 0; + bool walk(Item_processor processor, bool walk_subquery, uchar *args) + { + return (item->walk(processor, walk_subquery, args)) || + (this->*processor)(args); + } }; /** diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index e0dbf2c5a73..038a0ebabc2 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -798,7 +798,7 @@ int Arg_comparator::set_cmp_func(Item_result_field *owner_arg, Item **a1, Item **a2, Item_result type) { - thd= current_thd; + THD *thd= current_thd; owner= owner_arg; set_null= set_null && owner_arg; a= a1; @@ -868,7 +868,6 @@ Item** Arg_comparator::cache_converted_constant(THD *thd_arg, Item **value, void Arg_comparator::set_datetime_cmp_func(Item_result_field *owner_arg, Item **a1, Item **b1) { - thd= current_thd; owner= owner_arg; a= a1; b= b1; @@ -943,12 +942,10 @@ get_datetime_value(THD *thd, Item ***item_arg, Item **cache_arg, if (cache_arg && item->const_item() && !(item->type() == Item::CACHE_ITEM && item->cmp_type() == TIME_RESULT)) { - Query_arena backup; - Query_arena *save_arena= thd->switch_to_arena_for_cached_items(&backup); - Item_cache_temporal *cache= new Item_cache_temporal(f_type); - if (save_arena) - thd->set_query_arena(save_arena); + if (!thd) + thd= current_thd; + Item_cache_temporal *cache= new Item_cache_temporal(f_type); cache->store_packed(value, item); *cache_arg= cache; *item_arg= cache_arg; @@ -983,12 +980,12 @@ int Arg_comparator::compare_datetime() owner->null_value= 1; /* Get DATE/DATETIME/TIME value of the 'a' item. */ - a_value= get_datetime_value(thd, &a, &a_cache, *b, &a_is_null); + a_value= get_datetime_value(0, &a, &a_cache, *b, &a_is_null); if (a_is_null) return -1; /* Get DATE/DATETIME/TIME value of the 'b' item. */ - b_value= get_datetime_value(thd, &b, &b_cache, *a, &b_is_null); + b_value= get_datetime_value(0, &b, &b_cache, *a, &b_is_null); if (b_is_null) return -1; @@ -1006,10 +1003,10 @@ int Arg_comparator::compare_e_datetime() longlong a_value, b_value; /* Get DATE/DATETIME/TIME value of the 'a' item. */ - a_value= get_datetime_value(thd, &a, &a_cache, *b, &a_is_null); + a_value= get_datetime_value(0, &a, &a_cache, *b, &a_is_null); /* Get DATE/DATETIME/TIME value of the 'b' item. */ - b_value= get_datetime_value(thd, &b, &b_cache, *a, &b_is_null); + b_value= get_datetime_value(0, &b, &b_cache, *a, &b_is_null); return a_is_null || b_is_null ? a_is_null == b_is_null : a_value == b_value; } @@ -3663,7 +3660,7 @@ void in_datetime::set(uint pos,Item *item) bool is_null; struct packed_longlong *buff= &((packed_longlong*) base)[pos]; - buff->val= get_datetime_value(thd, &tmp_item, 0, warn_item, &is_null); + buff->val= get_datetime_value(0, &tmp_item, 0, warn_item, &is_null); buff->unsigned_flag= 1L; } @@ -3671,7 +3668,7 @@ uchar *in_datetime::get_value(Item *item) { bool is_null; Item **tmp_item= lval_cache ? &lval_cache : &item; - tmp.val= get_datetime_value(thd, &tmp_item, &lval_cache, warn_item, &is_null); + tmp.val= get_datetime_value(0, &tmp_item, &lval_cache, warn_item, &is_null); if (item->null_value) return 0; tmp.unsigned_flag= 1L; @@ -3915,7 +3912,7 @@ void cmp_item_datetime::store_value(Item *item) { bool is_null; Item **tmp_item= lval_cache ? &lval_cache : &item; - value= get_datetime_value(thd, &tmp_item, &lval_cache, warn_item, &is_null); + value= get_datetime_value(0, &tmp_item, &lval_cache, warn_item, &is_null); } @@ -3924,7 +3921,7 @@ int cmp_item_datetime::cmp(Item *arg) bool is_null; Item **tmp_item= &arg; return value != - get_datetime_value(thd, &tmp_item, 0, warn_item, &is_null); + get_datetime_value(0, &tmp_item, 0, warn_item, &is_null); } @@ -5107,6 +5104,15 @@ int Regexp_processor_pcre::default_regex_flags() return default_regex_flags_pcre(current_thd); } +void Regexp_processor_pcre::set_recursion_limit(THD *thd) +{ + long stack_used; + DBUG_ASSERT(thd == current_thd); + stack_used= available_stack_size(thd->thread_stack, &stack_used); + m_pcre_extra.match_limit_recursion= + (my_thread_stack_size - stack_used)/my_pcre_frame_size; +} + /** Convert string to lib_charset, if needed. @@ -5196,15 +5202,77 @@ void Regexp_processor_pcre::pcre_exec_warn(int rc) const */ switch (rc) { + case PCRE_ERROR_NULL: + errmsg= "pcre_exec: null arguement passed"; + break; + case PCRE_ERROR_BADOPTION: + errmsg= "pcre_exec: bad option"; + break; + case PCRE_ERROR_BADMAGIC: + errmsg= "pcre_exec: bad magic - not a compiled regex"; + break; + case PCRE_ERROR_UNKNOWN_OPCODE: + errmsg= "pcre_exec: error in compiled regex"; + break; case PCRE_ERROR_NOMEMORY: errmsg= "pcre_exec: Out of memory"; break; + case PCRE_ERROR_NOSUBSTRING: + errmsg= "pcre_exec: no substring"; + break; + case PCRE_ERROR_MATCHLIMIT: + errmsg= "pcre_exec: match limit exceeded"; + break; + case PCRE_ERROR_CALLOUT: + errmsg= "pcre_exec: callout error"; + break; case PCRE_ERROR_BADUTF8: errmsg= "pcre_exec: Invalid utf8 byte sequence in the subject string"; break; + case PCRE_ERROR_BADUTF8_OFFSET: + errmsg= "pcre_exec: Started at invalid location within utf8 byte sequence"; + break; + case PCRE_ERROR_PARTIAL: + errmsg= "pcre_exec: partial match"; + break; + case PCRE_ERROR_INTERNAL: + errmsg= "pcre_exec: internal error"; + break; + case PCRE_ERROR_BADCOUNT: + errmsg= "pcre_exec: ovesize is negative"; + break; + case PCRE_ERROR_RECURSIONLIMIT: + my_snprintf(buf, sizeof(buf), "pcre_exec: recursion limit of %ld exceeded", + m_pcre_extra.match_limit_recursion); + errmsg= buf; + break; + case PCRE_ERROR_BADNEWLINE: + errmsg= "pcre_exec: bad newline options"; + break; + case PCRE_ERROR_BADOFFSET: + errmsg= "pcre_exec: start offset negative or greater than string length"; + break; + case PCRE_ERROR_SHORTUTF8: + errmsg= "pcre_exec: ended in middle of utf8 sequence"; + break; + case PCRE_ERROR_JIT_STACKLIMIT: + errmsg= "pcre_exec: insufficient stack memory for JIT compile"; + break; case PCRE_ERROR_RECURSELOOP: errmsg= "pcre_exec: Recursion loop detected"; break; + case PCRE_ERROR_BADMODE: + errmsg= "pcre_exec: compiled pattern passed to wrong bit library function"; + break; + case PCRE_ERROR_BADENDIANNESS: + errmsg= "pcre_exec: compiled pattern passed to wrong endianness processor"; + break; + case PCRE_ERROR_JIT_BADOPTION: + errmsg= "pcre_exec: bad jit option"; + break; + case PCRE_ERROR_BADLENGTH: + errmsg= "pcre_exec: negative length"; + break; default: /* As other error codes should normally not happen, @@ -5240,7 +5308,7 @@ int Regexp_processor_pcre::pcre_exec_with_warn(const pcre *code, bool Regexp_processor_pcre::exec(const char *str, int length, int offset) { - m_pcre_exec_rc= pcre_exec_with_warn(m_pcre, NULL, str, length, offset, 0, + m_pcre_exec_rc= pcre_exec_with_warn(m_pcre, &m_pcre_extra, str, length, offset, 0, m_SubStrVec, m_subpatterns_needed * 3); return false; } @@ -5251,7 +5319,7 @@ bool Regexp_processor_pcre::exec(String *str, int offset, { if (!(str= convert_if_needed(str, &subject_converter))) return true; - m_pcre_exec_rc= pcre_exec_with_warn(m_pcre, NULL, + m_pcre_exec_rc= pcre_exec_with_warn(m_pcre, &m_pcre_extra, str->c_ptr_safe(), str->length(), offset, 0, m_SubStrVec, m_subpatterns_needed * 3); diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 9e83b732dc7..17ad1bd8c7d 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -45,7 +45,6 @@ class Arg_comparator: public Sql_alloc Arg_comparator *comparators; // used only for compare_row() double precision; /* Fields used in DATE/DATETIME comparison. */ - THD *thd; Item *a_cache, *b_cache; // Cached values of a and b items // when one of arguments is NULL. int set_compare_func(Item_result_field *owner, Item_result type); @@ -61,10 +60,10 @@ public: /* Allow owner function to use string buffers. */ String value1, value2; - Arg_comparator(): set_null(TRUE), comparators(0), thd(0), + Arg_comparator(): set_null(TRUE), comparators(0), a_cache(0), b_cache(0) {}; Arg_comparator(Item **a1, Item **a2): a(a1), b(a2), set_null(TRUE), - comparators(0), thd(0), a_cache(0), b_cache(0) {}; + comparators(0), a_cache(0), b_cache(0) {}; int set_cmp_func(Item_result_field *owner_arg, Item **a1, Item **a2, @@ -963,15 +962,13 @@ public: class in_datetime :public in_longlong { public: - THD *thd; /* An item used to issue warnings. */ Item *warn_item; /* Cache for the left item. */ Item *lval_cache; in_datetime(Item *warn_item_arg, uint elements) - :in_longlong(elements), thd(current_thd), warn_item(warn_item_arg), - lval_cache(0) {}; + :in_longlong(elements), warn_item(warn_item_arg), lval_cache(0) {}; void set(uint pos,Item *item); uchar *get_value(Item *item); Item* create_item() @@ -1131,14 +1128,13 @@ class cmp_item_datetime :public cmp_item { longlong value; public: - THD *thd; /* Item used for issuing warnings. */ Item *warn_item; /* Cache for the left item. */ Item *lval_cache; cmp_item_datetime(Item *warn_item_arg) - :thd(current_thd), warn_item(warn_item_arg), lval_cache(0) {} + : warn_item(warn_item_arg), lval_cache(0) {} void store_value(Item *item); int cmp(Item *arg); int compare(cmp_item *ci); @@ -1554,6 +1550,7 @@ public: class Regexp_processor_pcre { pcre *m_pcre; + pcre_extra m_pcre_extra; bool m_conversion_is_needed; bool m_is_const; int m_library_flags; @@ -1578,8 +1575,12 @@ public: m_data_charset(&my_charset_utf8_general_ci), m_library_charset(&my_charset_utf8_general_ci), m_subpatterns_needed(0) - {} + { + m_pcre_extra.flags= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + m_pcre_extra.match_limit_recursion= 100L; + } int default_regex_flags(); + void set_recursion_limit(THD *); void init(CHARSET_INFO *data_charset, int extra_flags, uint nsubpatterns) { m_library_flags= default_regex_flags() | extra_flags | diff --git a/sql/item_func.cc b/sql/item_func.cc index c87d4587f4d..2c43181e8bb 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2853,7 +2853,6 @@ void Item_func_min_max::fix_length_and_dec() decimals=0; max_length=0; maybe_null=0; - thd= current_thd; cmp_type=args[0]->result_type(); for (uint i=0 ; i < arg_count ; i++) @@ -2926,13 +2925,11 @@ bool Item_func_min_max::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) { Item **arg= args + i; bool is_null; - longlong res= get_datetime_value(thd, &arg, 0, compare_as_dates, &is_null); + longlong res= get_datetime_value(0, &arg, 0, compare_as_dates, &is_null); /* Check if we need to stop (because of error or KILL) and stop the loop */ - if (thd->is_error() || args[i]->null_value) - { + if (args[i]->null_value) return (null_value= 1); - } if (i == 0 || (res < min_max ? cmp_sign : -cmp_sign) > 0) min_max= res; @@ -3961,12 +3958,7 @@ longlong Item_master_pos_wait::val_int() else connection_name= thd->variables.default_master_connection; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) // master_info_index is set to NULL on shutdown. - mi= master_info_index->get_master_info(&connection_name, - Sql_condition::WARN_LEVEL_WARN); - mysql_mutex_unlock(&LOCK_active_mi); - if (!mi) + if (!(mi= get_master_info(&connection_name, Sql_condition::WARN_LEVEL_WARN))) goto err; if ((event_count = mi->rli.wait_for_pos(thd, log_name, pos, timeout)) == -2) @@ -3974,6 +3966,7 @@ longlong Item_master_pos_wait::val_int() null_value = 1; event_count=0; } + mi->release(); #endif return event_count; diff --git a/sql/item_func.h b/sql/item_func.h index 3d0cdca4472..0141619dced 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1077,7 +1077,6 @@ class Item_func_min_max :public Item_func int cmp_sign; /* An item used for issuing warnings while string to DATETIME conversion. */ Item *compare_as_dates; - THD *thd; protected: enum_field_types cached_field_type; public: diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 4ea3075e69c..9baa5d52319 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -1,6 +1,6 @@ /* - Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2009, 2013, Monty Program Ab. + Copyright (c) 2000, 2017, Oracle and/or its affiliates. + Copyright (c) 2009, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -3926,6 +3926,7 @@ String *Item_func_quote::val_str(String *str) 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + ulong max_allowed_packet= current_thd->variables.max_allowed_packet; char *from, *to, *end, *start; String *arg= args[0]->val_str(str); uint arg_length, new_length; @@ -3944,11 +3945,14 @@ String *Item_func_quote::val_str(String *str) new_length= arg_length + 2; /* for beginning and ending ' signs */ for (from= (char*) arg->ptr(), end= from + arg_length; from < end; from++) new_length+= get_esc_bit(escmask, (uchar) *from); + if (new_length > max_allowed_packet) + goto toolong; } else { new_length= (arg_length * 2) + /* For string characters */ (2 * collation.collation->mbmaxlen); /* For quotes */ + set_if_smaller(new_length, max_allowed_packet); } if (tmp_value.alloc(new_length)) @@ -3964,7 +3968,7 @@ String *Item_func_quote::val_str(String *str) /* Put leading quote */ if ((mblen= cs->cset->wc_mb(cs, '\'', (uchar *) to, to_end)) <= 0) - goto null; + goto toolong; to+= mblen; for (start= (char*) arg->ptr(), end= start + arg_length; start < end; ) @@ -3984,17 +3988,17 @@ String *Item_func_quote::val_str(String *str) if (escape) { if ((mblen= cs->cset->wc_mb(cs, '\\', (uchar*) to, to_end)) <= 0) - goto null; + goto toolong; to+= mblen; } if ((mblen= cs->cset->wc_mb(cs, wc, (uchar*) to, to_end)) <= 0) - goto null; + goto toolong; to+= mblen; } /* Put trailing quote */ if ((mblen= cs->cset->wc_mb(cs, '\'', (uchar *) to, to_end)) <= 0) - goto null; + goto toolong; to+= mblen; new_length= to - tmp_value.ptr(); goto ret; @@ -4038,6 +4042,11 @@ ret: null_value= 0; return &tmp_value; +toolong: + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_WARN_ALLOWED_PACKET_OVERFLOWED, + ER_THD(current_thd, ER_WARN_ALLOWED_PACKET_OVERFLOWED), + func_name(), max_allowed_packet); null: null_value= 1; return 0; diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index cf09b1801cf..e87db62bd98 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1,5 +1,5 @@ -/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2010, 2015, MariaDB +/* Copyright (c) 2002, 2016, Oracle and/or its affiliates. + Copyright (c) 2010, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -894,7 +894,7 @@ void Item_subselect::update_used_tables() if (!forced_const) { recalc_used_tables(parent_select, FALSE); - if (!engine->uncacheable()) + if (!(engine->uncacheable() & ~UNCACHEABLE_EXPLAIN)) { // did all used tables become static? if (!(used_tables_cache & ~engine->upper_select_const_tables())) @@ -2048,6 +2048,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, We can encounter "NULL IN (SELECT ...)". Wrap the added condition within a trig_cond. */ + disable_cond_guard_for_const_null_left_expr(0); item= new Item_func_trig_cond(item, get_cond_guard(0)); } @@ -2072,6 +2073,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, having= new Item_is_not_null_test(this, having); if (left_expr->maybe_null) { + disable_cond_guard_for_const_null_left_expr(0); if (!(having= new Item_func_trig_cond(having, get_cond_guard(0)))) DBUG_RETURN(true); @@ -2090,6 +2092,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, */ if (!abort_on_null && left_expr->maybe_null) { + disable_cond_guard_for_const_null_left_expr(0); if (!(item= new Item_func_trig_cond(item, get_cond_guard(0)))) DBUG_RETURN(true); } @@ -2116,6 +2119,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, (char *)"<result>")); if (!abort_on_null && left_expr->maybe_null) { + disable_cond_guard_for_const_null_left_expr(0); if (!(new_having= new Item_func_trig_cond(new_having, get_cond_guard(0)))) DBUG_RETURN(true); @@ -2311,6 +2315,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, Item *col_item= new Item_cond_or(item_eq, item_isnull); if (!abort_on_null && left_expr->element_index(i)->maybe_null) { + disable_cond_guard_for_const_null_left_expr(i); if (!(col_item= new Item_func_trig_cond(col_item, get_cond_guard(i)))) DBUG_RETURN(true); } @@ -2325,6 +2330,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, (char *)"<list ref>")); if (!abort_on_null && left_expr->element_index(i)->maybe_null) { + disable_cond_guard_for_const_null_left_expr(i); if (!(item_nnull_test= new Item_func_trig_cond(item_nnull_test, get_cond_guard(i)))) DBUG_RETURN(true); @@ -2381,6 +2387,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, item= new Item_cond_or(item, item_isnull); if (left_expr->element_index(i)->maybe_null) { + disable_cond_guard_for_const_null_left_expr(i); if (!(item= new Item_func_trig_cond(item, get_cond_guard(i)))) DBUG_RETURN(true); if (!(having_col_item= @@ -3424,8 +3431,12 @@ bool subselect_union_engine::is_executed() const bool subselect_union_engine::no_rows() { + bool rows_present= false; + /* Check if we got any rows when reading UNION result from temp. table: */ - return MY_TEST(!unit->fake_select_lex->join->send_records); + if (unit->fake_select_lex->join) + rows_present= MY_TEST(!unit->fake_select_lex->join->send_records); + return rows_present; } @@ -4808,9 +4819,9 @@ bool subselect_hash_sj_engine::init(List<Item> *tmp_columns, uint subquery_id) result= result_sink; /* - If the subquery has blobs, or the total key lenght is bigger than + If the subquery has blobs, or the total key length is bigger than some length, or the total number of key parts is more than the - allowed maximum (currently MAX_REF_PARTS == 16), then the created + allowed maximum (currently MAX_REF_PARTS == 32), then the created index cannot be used for lookups and we can't use hash semi join. If this is the case, delete the temporary table since it will not be used, and tell the caller we failed to initialize the @@ -6564,4 +6575,3 @@ end: void subselect_table_scan_engine::cleanup() { } - diff --git a/sql/item_subselect.h b/sql/item_subselect.h index 0c62c3f8d02..492a3d7dda5 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -619,6 +619,15 @@ public: bool expr_cache_is_needed(THD *thd); inline bool left_expr_has_null(); + void disable_cond_guard_for_const_null_left_expr(int i) + { + if (left_expr->const_item() && !left_expr->is_expensive()) + { + if (left_expr->element_index(i)->is_null()) + set_cond_guard_var(i,FALSE); + } + } + int optimize(double *out_rows, double *cost); /* Return the identifier that we could use to identify the subquery for the @@ -696,6 +705,13 @@ public: in_strategy= (SUBS_STRATEGY_CHOSEN | strategy); DBUG_VOID_RETURN; } + + bool walk(Item_processor processor, bool walk_subquery, uchar *arg) + { + return left_expr->walk(processor, walk_subquery, arg) || + Item_subselect::walk(processor, walk_subquery, arg); + } + bool exists2in_processor(uchar *opt_arg __attribute__((unused))) { return 0; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index d1ab102c709..0f10d832ba6 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -3442,7 +3442,7 @@ Item_func_group_concat::fix_fields(THD *thd, Item **ref) args[i]->fix_fields(thd, args + i)) || args[i]->check_cols(1)) return TRUE; - with_subselect|= args[i]->with_subselect; + with_subselect|= args[i]->with_subselect; } /* skip charset aggregation for order columns */ diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 866dbe65ecc..fa1c3d0da90 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -426,7 +426,7 @@ static bool extract_date_time(DATE_TIME_FORMAT *format, { if (!my_isspace(&my_charset_latin1,*val)) { - make_truncated_value_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, + make_truncated_value_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, val_begin, length, cached_timestamp_type, NullS); break; @@ -708,7 +708,7 @@ static bool get_interval_info(const char *str,uint length,CHARSET_INFO *cs, { longlong value; const char *start= str; - for (value=0; str != end && my_isdigit(cs, *str) ; str++) + for (value= 0; str != end && my_isdigit(cs, *str); str++) value= value*10 + *str - '0'; msec_length= 6 - (str - start); values[i]= value; @@ -1460,6 +1460,7 @@ void Item_temporal_func::fix_length_and_dec() time can get us to return NULL. */ maybe_null= 1; + if (decimals) { if (decimals == NOT_FIXED_DEC) diff --git a/sql/item_xmlfunc.cc b/sql/item_xmlfunc.cc index f8bf7cbf93a..c7fd923b0ae 100644 --- a/sql/item_xmlfunc.cc +++ b/sql/item_xmlfunc.cc @@ -1,4 +1,5 @@ -/* Copyright (c) 2005, 2013, Oracle and/or its affiliates. +/* Copyright (c) 2005, 2016, Oracle and/or its affiliates. + Copyright (c) 2009, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2751,9 +2752,9 @@ int xml_enter(MY_XML_PARSER *st,const char *attr, size_t len) node.parent= data->parent; // Set parent for the new node to old parent data->parent= numnodes; // Remember current node as new parent - DBUG_ASSERT(data->level <= MAX_LEVEL); + DBUG_ASSERT(data->level < MAX_LEVEL); data->pos[data->level]= numnodes; - if (data->level < MAX_LEVEL) + if (data->level < MAX_LEVEL - 1) node.level= data->level++; else return MY_XML_ERROR; diff --git a/sql/lock.cc b/sql/lock.cc index 614341fcc43..29afcc8f578 100644 --- a/sql/lock.cc +++ b/sql/lock.cc @@ -380,12 +380,15 @@ static int lock_external(THD *thd, TABLE **tables, uint count) void mysql_unlock_tables(THD *thd, MYSQL_LOCK *sql_lock, bool free_lock) { DBUG_ENTER("mysql_unlock_tables"); + bool errors= thd->is_error(); if (sql_lock->table_count) unlock_external(thd, sql_lock->table, sql_lock->table_count); if (sql_lock->lock_count) thr_multi_unlock(sql_lock->locks, sql_lock->lock_count, 0); if (free_lock) my_free(sql_lock); + if (!errors) + thd->clear_error(); DBUG_VOID_RETURN; } diff --git a/sql/log.cc b/sql/log.cc index cd215822d44..b80d65e20db 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -1294,7 +1294,7 @@ bool LOGGER::slow_log_print(THD *thd, const char *query, uint query_length, /* fill in user_host value: the format is "%s[%s] @ %s [%s]" */ user_host_len= (strxnmov(user_host_buff, MAX_USER_HOST_SIZE, - sctx->priv_user ? sctx->priv_user : "", "[", + sctx->priv_user, "[", sctx->user ? sctx->user : (thd->slave_thread ? "SQL_SLAVE" : ""), "] @ ", sctx->host ? sctx->host : "", " [", sctx->ip ? sctx->ip : "", "]", NullS) - @@ -2702,16 +2702,16 @@ int MYSQL_LOG::generate_new_name(char *new_name, const char *log_name) void MYSQL_QUERY_LOG::reopen_file() { char *save_name; - DBUG_ENTER("MYSQL_LOG::reopen_file"); + + mysql_mutex_lock(&LOCK_log); if (!is_open()) { DBUG_PRINT("info",("log is closed")); + mysql_mutex_unlock(&LOCK_log); DBUG_VOID_RETURN; } - mysql_mutex_lock(&LOCK_log); - save_name= name; name= 0; // Don't free name close(LOG_CLOSE_TO_BE_OPENED); @@ -2870,13 +2870,6 @@ bool MYSQL_QUERY_LOG::write(THD *thd, time_t current_time, DBUG_ENTER("MYSQL_QUERY_LOG::write"); mysql_mutex_lock(&LOCK_log); - - if (!is_open()) - { - mysql_mutex_unlock(&LOCK_log); - DBUG_RETURN(0); - } - if (is_open()) { // Safety agains reopen int tmp_errno= 0; @@ -3100,7 +3093,9 @@ void MYSQL_BIN_LOG::cleanup() } inited= 0; + mysql_mutex_lock(&LOCK_log); close(LOG_CLOSE_INDEX|LOG_CLOSE_STOP_EVENT); + mysql_mutex_unlock(&LOCK_log); delete description_event_for_queue; delete description_event_for_exec; @@ -3257,10 +3252,11 @@ bool MYSQL_BIN_LOG::open(const char *log_name, { File file= -1; xid_count_per_binlog *new_xid_list_entry= NULL, *b; - DBUG_ENTER("MYSQL_BIN_LOG::open"); DBUG_PRINT("enter",("log_type: %d",(int) log_type_arg)); + mysql_mutex_assert_owner(&LOCK_log); + if (!is_relay_log) { if (!binlog_state_recover_done) @@ -4129,7 +4125,7 @@ err: int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) { - int error; + int error, errcode; char *to_purge_if_included= NULL; inuse_relaylog *ir; ulonglong log_space_reclaimed= 0; @@ -4200,7 +4196,8 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) } /* Store where we are in the new file for the execution thread */ - flush_relay_log_info(rli); + if (flush_relay_log_info(rli)) + error= LOG_INFO_IO; DBUG_EXECUTE_IF("crash_before_purge_logs", DBUG_SUICIDE();); @@ -4216,11 +4213,13 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) * Need to update the log pos because purge logs has been called * after fetching initially the log pos at the begining of the method. */ - if((error=find_log_pos(&rli->linfo, rli->event_relay_log_name, 0))) + if ((errcode= find_log_pos(&rli->linfo, rli->event_relay_log_name, 0))) { char buff[22]; + if (!error) + error= errcode; sql_print_error("next log error: %d offset: %s log: %s included: %d", - error, + errcode, llstr(rli->linfo.index_file_offset,buff), rli->group_relay_log_name, included); @@ -4839,21 +4838,21 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) bool delay_close= false; File old_file; LINT_INIT(old_file); - DBUG_ENTER("MYSQL_BIN_LOG::new_file_impl"); + + if (need_lock) + mysql_mutex_lock(&LOCK_log); + mysql_mutex_assert_owner(&LOCK_log); + if (!is_open()) { DBUG_PRINT("info",("log is closed")); + mysql_mutex_unlock(&LOCK_log); DBUG_RETURN(error); } - if (need_lock) - mysql_mutex_lock(&LOCK_log); mysql_mutex_lock(&LOCK_index); - mysql_mutex_assert_owner(&LOCK_log); - mysql_mutex_assert_owner(&LOCK_index); - /* Reuse old name if not binlog and not update log */ new_name_ptr= name; @@ -4986,9 +4985,9 @@ end: new_name_ptr, errno); } + mysql_mutex_unlock(&LOCK_index); if (need_lock) mysql_mutex_unlock(&LOCK_log); - mysql_mutex_unlock(&LOCK_index); DBUG_RETURN(error); } @@ -7772,9 +7771,11 @@ int MYSQL_BIN_LOG::wait_for_update_bin_log(THD* thd, void MYSQL_BIN_LOG::close(uint exiting) { // One can't set log_type here! bool failed_to_save_state= false; - DBUG_ENTER("MYSQL_BIN_LOG::close"); DBUG_PRINT("enter",("exiting: %d", (int) exiting)); + + mysql_mutex_assert_owner(&LOCK_log); + if (log_state == LOG_OPENED) { #ifdef HAVE_REPLICATION diff --git a/sql/log_event.cc b/sql/log_event.cc index d311297f8a8..43e8df7b801 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -6244,9 +6244,11 @@ bool Rotate_log_event::write(IO_CACHE* file) @retval 0 ok + 1 error */ int Rotate_log_event::do_update_pos(rpl_group_info *rgi) { + int error= 0; Relay_log_info *rli= rgi->rli; DBUG_ENTER("Rotate_log_event::do_update_pos"); #ifndef DBUG_OFF @@ -6298,7 +6300,7 @@ int Rotate_log_event::do_update_pos(rpl_group_info *rgi) (ulong) rli->group_master_log_pos)); mysql_mutex_unlock(&rli->data_lock); rpl_global_gtid_slave_state->record_and_update_gtid(thd, rgi); - flush_relay_log_info(rli); + error= flush_relay_log_info(rli); /* Reset thd->variables.option_bits and sql_mode etc, because this could @@ -6316,8 +6318,7 @@ int Rotate_log_event::do_update_pos(rpl_group_info *rgi) else rgi->inc_event_relay_log_pos(); - - DBUG_RETURN(0); + DBUG_RETURN(error); } @@ -6915,6 +6916,7 @@ Gtid_list_log_event::do_apply_event(rpl_group_info *rgi) rli->abort_slave= true; rli->stop_for_until= true; } + free_root(thd->mem_root, MYF(MY_KEEP_PREALLOC)); return ret; } @@ -8173,6 +8175,7 @@ void Stop_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) int Stop_log_event::do_update_pos(rpl_group_info *rgi) { + int error= 0; Relay_log_info *rli= rgi->rli; DBUG_ENTER("Stop_log_event::do_update_pos"); /* @@ -8188,9 +8191,10 @@ int Stop_log_event::do_update_pos(rpl_group_info *rgi) { rpl_global_gtid_slave_state->record_and_update_gtid(thd, rgi); rli->inc_group_relay_log_pos(0, rgi); - flush_relay_log_info(rli); + if (flush_relay_log_info(rli)) + error= 1; } - DBUG_RETURN(0); + DBUG_RETURN(error); } #endif /* !MYSQL_CLIENT */ @@ -10177,8 +10181,8 @@ int Rows_log_event::do_update_pos(rpl_group_info *rgi) { Relay_log_info *rli= rgi->rli; - DBUG_ENTER("Rows_log_event::do_update_pos"); int error= 0; + DBUG_ENTER("Rows_log_event::do_update_pos"); DBUG_PRINT("info", ("flags: %s", get_flags(STMT_END_F) ? "STMT_END_F " : "")); @@ -10190,7 +10194,7 @@ Rows_log_event::do_update_pos(rpl_group_info *rgi) Step the group log position if we are not in a transaction, otherwise increase the event log position. */ - rli->stmt_done(log_pos, thd, rgi); + error= rli->stmt_done(log_pos, thd, rgi); /* Clear any errors in thd->net.last_err*. It is not known if this is needed or not. It is believed that any errors that may exist in diff --git a/sql/log_event_old.cc b/sql/log_event_old.cc index 22ef970dab9..73869e82f72 100644 --- a/sql/log_event_old.cc +++ b/sql/log_event_old.cc @@ -1681,8 +1681,7 @@ int Old_rows_log_event::do_apply_event(rpl_group_info *rgi) default: rli->report(ERROR_LEVEL, thd->net.last_errno, NULL, "Error in %s event: row application failed. %s", - get_type_str(), - thd->net.last_error ? thd->net.last_error : ""); + get_type_str(), thd->net.last_error); thd->is_slave_error= 1; break; } @@ -1721,8 +1720,7 @@ int Old_rows_log_event::do_apply_event(rpl_group_info *rgi) "Error in %s event: error during transaction execution " "on table %s.%s. %s", get_type_str(), table->s->db.str, - table->s->table_name.str, - thd->net.last_error ? thd->net.last_error : ""); + table->s->table_name.str, thd->net.last_error); /* If one day we honour --skip-slave-errors in row-based replication, and @@ -1843,8 +1841,8 @@ int Old_rows_log_event::do_update_pos(rpl_group_info *rgi) { Relay_log_info *rli= rgi->rli; - DBUG_ENTER("Old_rows_log_event::do_update_pos"); int error= 0; + DBUG_ENTER("Old_rows_log_event::do_update_pos"); DBUG_PRINT("info", ("flags: %s", get_flags(STMT_END_F) ? "STMT_END_F " : "")); @@ -1856,7 +1854,7 @@ Old_rows_log_event::do_update_pos(rpl_group_info *rgi) Step the group log position if we are not in a transaction, otherwise increase the event log position. */ - rli->stmt_done(log_pos, thd, rgi); + error= rli->stmt_done(log_pos, thd, rgi); /* Clear any errors in thd->net.last_err*. It is not known if this is needed or not. It is believed that any errors that may exist in diff --git a/sql/log_slow.h b/sql/log_slow.h index 3ae2060cc27..aea5b149263 100644 --- a/sql/log_slow.h +++ b/sql/log_slow.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2009 Monty Program Ab +/* Copyright (C) 2009, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,23 +16,22 @@ /* Defining what to log to slow log */ #define LOG_SLOW_VERBOSITY_INIT 0 -#define LOG_SLOW_VERBOSITY_INNODB (1 << 0) -#define LOG_SLOW_VERBOSITY_QUERY_PLAN (1 << 1) -#define LOG_SLOW_VERBOSITY_EXPLAIN (1 << 2) +#define LOG_SLOW_VERBOSITY_INNODB (1U << 0) +#define LOG_SLOW_VERBOSITY_QUERY_PLAN (1U << 1) +#define LOG_SLOW_VERBOSITY_EXPLAIN (1U << 2) #define QPLAN_INIT QPLAN_QC_NO -#define QPLAN_ADMIN (1 << 0) -#define QPLAN_FILESORT (1 << 1) -#define QPLAN_FILESORT_DISK (1 << 2) -#define QPLAN_FULL_JOIN (1 << 3) -#define QPLAN_FULL_SCAN (1 << 4) -#define QPLAN_QC (1 << 5) -#define QPLAN_QC_NO (1 << 6) -#define QPLAN_TMP_DISK (1 << 7) -#define QPLAN_TMP_TABLE (1 << 8) -#define QPLAN_FILESORT_PRIORITY_QUEUE (1 << 9) - +#define QPLAN_ADMIN (1U << 0) +#define QPLAN_FILESORT (1U << 1) +#define QPLAN_FILESORT_DISK (1U << 2) +#define QPLAN_FULL_JOIN (1U << 3) +#define QPLAN_FULL_SCAN (1U << 4) +#define QPLAN_QC (1U << 5) +#define QPLAN_QC_NO (1U << 6) +#define QPLAN_TMP_DISK (1U << 7) +#define QPLAN_TMP_TABLE (1U << 8) +#define QPLAN_FILESORT_PRIORITY_QUEUE (1U << 9) + /* ... */ -#define QPLAN_MAX (((ulong) 1) << 31) /* reserved as placeholder */ - +#define QPLAN_MAX (1U << 31) /* reserved as placeholder */ diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index c39789f7c97..de799874a8f 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -233,6 +233,20 @@ static void get_basedir(char *basedir, int size, const char *mysqld_path) } } +#define STR(s) _STR(s) +#define _STR(s) #s + +static char *get_plugindir() +{ + static char plugin_dir[2*MAX_PATH]; + get_basedir(plugin_dir, sizeof(plugin_dir), mysqld_path); + strcat(plugin_dir, "/" STR(INSTALL_PLUGINDIR)); + + if (access(plugin_dir, 0) == 0) + return plugin_dir; + + return NULL; +} /** Allocate and initialize command line for mysqld --bootstrap. @@ -313,6 +327,10 @@ static int create_myini() fprintf(myini,"protocol=pipe\n"); else if (opt_port) fprintf(myini,"port=%d\n",opt_port); + + char *plugin_dir = get_plugindir(); + if (plugin_dir) + fprintf(myini, "plugin-dir=%s\n", plugin_dir); fclose(myini); return 0; } @@ -368,8 +386,8 @@ static int register_service() CloseServiceHandle(sc_manager); die("CreateService failed (%u)", GetLastError()); } - - SERVICE_DESCRIPTION sd= { "MariaDB database server" }; + char description[] = "MariaDB database server"; + SERVICE_DESCRIPTION sd= { description }; ChangeServiceConfig2(sc_service, SERVICE_CONFIG_DESCRIPTION, &sd); CloseServiceHandle(sc_service); CloseServiceHandle(sc_manager); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index fda4fab9d26..a8fef1bd8a3 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -21,7 +21,7 @@ #ifndef __WIN__ #include <netdb.h> // getservbyname, servent #endif -#include "sql_parse.h" // test_if_data_home_dir +#include "sql_parse.h" // path_starts_from_data_home_dir #include "sql_cache.h" // query_cache, query_cache_* #include "sql_locale.h" // MY_LOCALES, my_locales, my_locale_by_name #include "sql_show.h" // free_status_vars, add_status_vars, @@ -100,6 +100,7 @@ #include "sp_rcontext.h" #include "sp_cache.h" #include "sql_reload.h" // reload_acl_and_cache +#include "pcre.h" #ifdef HAVE_POLL_H #include <poll.h> @@ -123,10 +124,7 @@ extern "C" { // Because of SCO 3.2V4.2 #include <sysent.h> #endif #ifdef HAVE_PWD_H -#include <pwd.h> // For getpwent -#endif -#ifdef HAVE_GRP_H -#include <grp.h> +#include <pwd.h> // For struct passwd #endif #include <my_net.h> @@ -465,9 +463,7 @@ ulong opt_binlog_rows_event_max_size; my_bool opt_master_verify_checksum= 0; my_bool opt_slave_sql_verify_checksum= 1; const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS}; -#ifdef HAVE_INITGROUPS volatile sig_atomic_t calling_initgroups= 0; /**< Used in SIGSEGV handler. */ -#endif uint mysqld_port, test_flags, select_errors, dropping_tables, ha_open_options; uint mysqld_extra_port; uint mysqld_port_timeout; @@ -707,12 +703,15 @@ mysql_mutex_t LOCK_delayed_insert, LOCK_delayed_status, LOCK_delayed_create, LOCK_crypt, LOCK_global_system_variables, - LOCK_user_conn, LOCK_slave_list, LOCK_active_mi, + LOCK_user_conn, LOCK_slave_list, LOCK_connection_count, LOCK_error_messages, LOCK_slave_init; mysql_mutex_t LOCK_stats, LOCK_global_user_client_stats, LOCK_global_table_stats, LOCK_global_index_stats; +/* This protects against changes in master_info_index */ +mysql_mutex_t LOCK_active_mi; + /** The below lock protects access to two global server variables: max_prepared_stmt_count and prepared_stmt_count. These variables @@ -865,7 +864,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_system_variables_hash, key_LOCK_thd_data, key_LOCK_user_conn, key_LOCK_uuid_short_generator, key_LOG_LOCK_log, key_master_info_data_lock, key_master_info_run_lock, - key_master_info_sleep_lock, + key_master_info_sleep_lock, key_master_info_start_stop_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_rpl_group_info_sleep_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, @@ -935,6 +934,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_uuid_short_generator, "LOCK_uuid_short_generator", PSI_FLAG_GLOBAL}, { &key_LOG_LOCK_log, "LOG::LOCK_log", 0}, { &key_master_info_data_lock, "Master_info::data_lock", 0}, + { &key_master_info_start_stop_lock, "Master_info::start_stop_lock", 0}, { &key_master_info_run_lock, "Master_info::run_lock", 0}, { &key_master_info_sleep_lock, "Master_info::sleep_lock", 0}, { &key_mutex_slave_reporting_capability_err_lock, "Slave_reporting_capability::err_lock", 0}, @@ -1419,7 +1419,7 @@ ulong query_cache_min_res_unit= QUERY_CACHE_MIN_RESULT_DATA_SIZE; Query_cache query_cache; #endif #ifdef HAVE_SMEM -char *shared_memory_base_name= default_shared_memory_base_name; +const char *shared_memory_base_name= default_shared_memory_base_name; my_bool opt_enable_shared_memory; HANDLE smem_event_connect_request= 0; #endif @@ -1656,7 +1656,7 @@ static void close_connections(void) mysql_mutex_unlock(&LOCK_thread_count); // For unlink from list Events::deinit(); - end_slave(); + slave_prepare_for_shutdown(); /* Give threads time to die. @@ -1705,6 +1705,7 @@ static void close_connections(void) DBUG_PRINT("quit",("Unlocking LOCK_thread_count")); mysql_mutex_unlock(&LOCK_thread_count); } + end_slave(); /* All threads has now been aborted */ DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count)); mysql_mutex_lock(&LOCK_thread_count); @@ -2223,59 +2224,18 @@ static void set_ports() static struct passwd *check_user(const char *user) { -#if !defined(__WIN__) - struct passwd *tmp_user_info; - uid_t user_id= geteuid(); + myf flags= 0; + if (global_system_variables.log_warnings) + flags|= MY_WME; + if (!opt_bootstrap && !opt_help) + flags|= MY_FAE; - // Don't bother if we aren't superuser - if (user_id) - { - if (user) - { - /* Don't give a warning, if real user is same as given with --user */ - /* purecov: begin tested */ - tmp_user_info= getpwnam(user); - if ((!tmp_user_info || user_id != tmp_user_info->pw_uid) && - global_system_variables.log_warnings) - sql_print_warning( - "One can only use the --user switch if running as root\n"); - /* purecov: end */ - } - return NULL; - } - if (!user) - { - if (!opt_bootstrap && !opt_help) - { - sql_print_error("Fatal error: Please consult the Knowledge Base " - "to find out how to run mysqld as root!\n"); - unireg_abort(1); - } - return NULL; - } - /* purecov: begin tested */ - if (!strcmp(user,"root")) - return NULL; // Avoid problem with dynamic libraries + struct passwd *tmp_user_info= my_check_user(user, MYF(flags)); - if (!(tmp_user_info= getpwnam(user))) - { - // Allow a numeric uid to be used - const char *pos; - for (pos= user; my_isdigit(mysqld_charset,*pos); pos++) ; - if (*pos) // Not numeric id - goto err; - if (!(tmp_user_info= getpwuid(atoi(user)))) - goto err; - } + if (!tmp_user_info && my_errno==EINVAL && (flags & MY_FAE)) + unireg_abort(1); return tmp_user_info; - /* purecov: end */ - -err: - sql_print_error("Fatal error: Can't change to run as user '%s' ; Please check that the user exists!\n",user); - unireg_abort(1); -#endif - return NULL; } static inline void allow_coredumps() @@ -2292,10 +2252,6 @@ static inline void allow_coredumps() static void set_user(const char *user, struct passwd *user_info_arg) { - /* purecov: begin tested */ -#if !defined(__WIN__) - DBUG_ASSERT(user_info_arg != 0); -#ifdef HAVE_INITGROUPS /* We can get a SIGSEGV when calling initgroups() on some systems when NSS is configured to use LDAP and the server is statically linked. We set @@ -2303,22 +2259,11 @@ static void set_user(const char *user, struct passwd *user_info_arg) output a specific message to help the user resolve this problem. */ calling_initgroups= 1; - initgroups((char*) user, user_info_arg->pw_gid); + int res= my_set_user(user, user_info_arg, MYF(MY_WME)); calling_initgroups= 0; -#endif - if (setgid(user_info_arg->pw_gid) == -1) - { - sql_perror("setgid"); - unireg_abort(1); - } - if (setuid(user_info_arg->pw_uid) == -1) - { - sql_perror("setuid"); + if (res) unireg_abort(1); - } allow_coredumps(); -#endif - /* purecov: end */ } @@ -2704,23 +2649,6 @@ void dec_connection_count(THD *thd) /* - Delete THD and decrement thread counters, including thread_running -*/ - -void delete_running_thd(THD *thd) -{ - mysql_mutex_lock(&LOCK_thread_count); - thd->unlink(); - mysql_mutex_unlock(&LOCK_thread_count); - - delete thd; - dec_thread_running(); - thread_safe_decrement32(&thread_count, &thread_count_lock); - signal_thd_deleted(); -} - - -/* Send a signal to unblock close_conneciton() if there is no more threads running with a THD attached @@ -3569,6 +3497,7 @@ static void init_libstrings() #endif } +ulonglong my_pcre_frame_size; static void init_pcre() { @@ -3576,6 +3505,8 @@ static void init_pcre() pcre_free= pcre_stack_free= my_str_free_mysqld; #ifndef EMBEDDED_LIBRARY pcre_stack_guard= check_enough_stack_size_slow; + /* See http://pcre.org/original/doc/html/pcrestack.html */ + my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) + 16; #endif } @@ -4265,19 +4196,24 @@ static int init_common_variables() default_charset_info= default_collation; } /* Set collactions that depends on the default collation */ - global_system_variables.collation_server= default_charset_info; - global_system_variables.collation_database= default_charset_info; - global_system_variables.collation_connection= default_charset_info; - global_system_variables.character_set_results= default_charset_info; - if (default_charset_info->mbminlen > 1) - { - global_system_variables.character_set_client= &my_charset_latin1; - sql_print_warning("Cannot use %s as character_set_client, %s will be used instead", - default_charset_info->csname, - global_system_variables.character_set_client->csname); + global_system_variables.collation_server= default_charset_info; + global_system_variables.collation_database= default_charset_info; + if (is_supported_parser_charset(default_charset_info)) + { + global_system_variables.collation_connection= default_charset_info; + global_system_variables.character_set_results= default_charset_info; + global_system_variables.character_set_client= default_charset_info; } else - global_system_variables.character_set_client= default_charset_info; + { + sql_print_warning("'%s' can not be used as client character set. " + "'%s' will be used as default client character set.", + default_charset_info->csname, + my_charset_latin1.csname); + global_system_variables.collation_connection= &my_charset_latin1; + global_system_variables.character_set_results= &my_charset_latin1; + global_system_variables.character_set_client= &my_charset_latin1; + } if (!(character_set_filesystem= get_charset_by_csname(character_set_filesystem_name, @@ -4979,9 +4915,17 @@ static int init_server_components() unireg_abort(1); } - if (opt_bin_log && mysql_bin_log.open(opt_bin_logname, LOG_BIN, 0, - WRITE_CACHE, max_binlog_size, 0, TRUE)) - unireg_abort(1); + if (opt_bin_log) + { + int error; + mysql_mutex_t *log_lock= mysql_bin_log.get_log_lock(); + mysql_mutex_lock(log_lock); + error= mysql_bin_log.open(opt_bin_logname, LOG_BIN, 0, + WRITE_CACHE, max_binlog_size, 0, TRUE); + mysql_mutex_unlock(log_lock); + if (error) + unireg_abort(1); + } #ifdef HAVE_REPLICATION if (opt_bin_log && expire_logs_days) @@ -7270,17 +7214,14 @@ static int show_slave_running(THD *thd, SHOW_VAR *var, char *buff) var->type= SHOW_MY_BOOL; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE))) { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE); - if (mi) - tmp= (my_bool) (mi->slave_running == MYSQL_SLAVE_RUN_READING && - mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN); + tmp= (my_bool) (mi->slave_running == MYSQL_SLAVE_RUN_READING && + mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); if (mi) *((my_bool *)buff)= tmp; else @@ -7312,38 +7253,26 @@ static int show_slaves_running(THD *thd, SHOW_VAR *var, char *buff) { var->type= SHOW_LONGLONG; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) - *((longlong *)buff)= master_info_index->any_slave_sql_running(); - else - *((longlong *)buff)= 0; + *((longlong *)buff)= any_slave_sql_running(); - mysql_mutex_unlock(&LOCK_active_mi); return 0; } static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff) { - Master_info *mi= NULL; - longlong tmp; - LINT_INIT(tmp); + Master_info *mi; var->type= SHOW_LONGLONG; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE))) { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE); - if (mi) - tmp= mi->received_heartbeats; + *((longlong *)buff)= mi->received_heartbeats; + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); - if (mi) - *((longlong *)buff)= tmp; else var->type= SHOW_UNDEF; return 0; @@ -7353,23 +7282,16 @@ static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff) static int show_heartbeat_period(THD *thd, SHOW_VAR *var, char *buff) { Master_info *mi= NULL; - float tmp; - LINT_INIT(tmp); var->type= SHOW_CHAR; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE))) { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE); - if (mi) - tmp= mi->heartbeat_period; + sprintf(buff, "%.3f", mi->heartbeat_period); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); - if (mi) - sprintf(buff, "%.3f", tmp); else var->type= SHOW_UNDEF; return 0; @@ -8195,7 +8117,7 @@ static int mysql_init_variables(void) mysql_home[0]= pidfile_name[0]= log_error_file[0]= 0; #if defined(HAVE_REALPATH) && !defined(HAVE_valgrind) && !defined(HAVE_BROKEN_REALPATH) /* We can only test for sub paths if my_symlink.c is using realpath */ - myisam_test_invalid_symlink= test_if_data_home_dir; + mysys_test_invalid_symlink= path_starts_from_data_home_dir; #endif opt_log= opt_slow_log= 0; opt_bin_log= opt_bin_log_used= 0; diff --git a/sql/mysqld.h b/sql/mysqld.h index 732f00c887a..e771a0c0da2 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -59,7 +59,6 @@ void kill_mysql(void); void close_connection(THD *thd, uint sql_errno= 0); void handle_connection_in_main_thread(THD *thd); void create_thread_to_handle_connection(THD *thd); -void delete_running_thd(THD *thd); void signal_thd_deleted(); void unlink_thd(THD *thd); bool one_thread_per_connection_end(THD *thd, bool put_in_cache); @@ -118,7 +117,8 @@ extern my_bool sp_automatic_privileges, opt_noacl; extern ulong use_stat_tables; extern my_bool opt_old_style_user_limits, trust_function_creators; extern uint opt_crash_binlog_innodb; -extern char *shared_memory_base_name, *mysqld_unix_port; +extern const char *shared_memory_base_name; +extern char *mysqld_unix_port; extern my_bool opt_enable_shared_memory; extern ulong opt_replicate_events_marked_for_skip; extern char *default_tz_name; @@ -265,7 +265,7 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_thd_data, key_LOCK_user_conn, key_LOG_LOCK_log, key_master_info_data_lock, key_master_info_run_lock, - key_master_info_sleep_lock, + key_master_info_sleep_lock, key_master_info_start_stop_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, key_rpl_group_info_sleep_lock, @@ -488,6 +488,8 @@ extern pthread_t signal_thread; extern struct st_VioSSLFd * ssl_acceptor_fd; #endif /* HAVE_OPENSSL */ +extern ulonglong my_pcre_frame_size; + /* The following variables were under INNODB_COMPABILITY_HOOKS */ diff --git a/sql/nt_servc.cc b/sql/nt_servc.cc index d6a8eac7ed5..e05e43a0a59 100644 --- a/sql/nt_servc.cc +++ b/sql/nt_servc.cc @@ -508,7 +508,7 @@ BOOL NTService::IsService(LPCSTR ServiceName) } /* ------------------------------------------------------------------------ -------------------------------------------------------------------------- */ -BOOL NTService::got_service_option(char **argv, char *service_option) +BOOL NTService::got_service_option(char **argv, const char *service_option) { char *option; for (option= argv[1]; *option; option++) diff --git a/sql/nt_servc.h b/sql/nt_servc.h index 949499d8d7f..6781fe0ddfa 100644 --- a/sql/nt_servc.h +++ b/sql/nt_servc.h @@ -61,7 +61,7 @@ class NTService BOOL SeekStatus(LPCSTR szInternName, int OperationType); BOOL Remove(LPCSTR szInternName); BOOL IsService(LPCSTR ServiceName); - BOOL got_service_option(char **argv, char *service_option); + BOOL got_service_option(char **argv, const char *service_option); BOOL is_super_user(); /* diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 5d6891a1edf..8f9d5abfa4d 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -8067,8 +8067,15 @@ static SEL_TREE *get_mm_tree(RANGE_OPT_PARAM *param,COND *cond) if (cond_func->functype() == Item_func::BETWEEN || cond_func->functype() == Item_func::IN_FUNC) inv= ((Item_func_opt_neg *) cond_func)->negated; - else if (cond_func->select_optimize() == Item_func::OPTIMIZE_NONE) - DBUG_RETURN(0); + else + { + MEM_ROOT *tmp_root= param->mem_root; + param->thd->mem_root= param->old_root; + Item_func::optimize_type opt_res= cond_func->select_optimize(); + param->thd->mem_root= tmp_root; + if (opt_res == Item_func::OPTIMIZE_NONE) + DBUG_RETURN(0); + } param->cond= cond; @@ -9923,6 +9930,13 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) if (!tmp->next_key_part) { + if (key2->use_count) + { + SEL_ARG *key2_cpy= new SEL_ARG(*key2); + if (key2_cpy) + return 0; + key2= key2_cpy; + } /* tmp->next_key_part is empty: cut the range that is covered by tmp from key2. @@ -9954,13 +9968,6 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) key2: [---] tmp: [---------] */ - if (key2->use_count) - { - SEL_ARG *key2_cpy= new SEL_ARG(*key2); - if (key2_cpy) - return 0; - key2= key2_cpy; - } key2->copy_max_to_min(tmp); continue; } diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index a81b091461f..564a108c766 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -4044,13 +4044,13 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) share->db_plugin= ha_lock_engine(0, TMP_ENGINE_HTON); table->file= get_new_handler(share, &table->mem_root, share->db_type()); - DBUG_ASSERT(uniq_tuple_length_arg <= table->file->max_key_length()); } else { share->db_plugin= ha_lock_engine(0, heap_hton); table->file= get_new_handler(share, &table->mem_root, share->db_type()); + DBUG_ASSERT(uniq_tuple_length_arg <= table->file->max_key_length()); } if (!table->file) goto err; diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 1607b1937df..1eee5df2bc5 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -27,7 +27,7 @@ #include "sql_partition.h" // partition_info.h: LIST_PART_ENTRY // NOT_A_PARTITION_ID #include "partition_info.h" -#include "sql_parse.h" // test_if_data_home_dir +#include "sql_parse.h" #include "sql_acl.h" // *_ACL #include "sql_base.h" // fill_record diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 249bf7608e5..bff0abe8198 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -40,7 +40,9 @@ Master_info::Master_info(LEX_STRING *connection_name_arg, sync_counter(0), heartbeat_period(0), received_heartbeats(0), master_id(0), prev_master_id(0), using_gtid(USE_GTID_NO), events_queued_since_last_gtid(0), - gtid_reconnect_event_skip_count(0), gtid_event_seen(false) + gtid_reconnect_event_skip_count(0), gtid_event_seen(false), + in_start_all_slaves(0), in_stop_all_slaves(0), + users(0), killed(0) { host[0] = 0; user[0] = 0; password[0] = 0; ssl_ca[0]= 0; ssl_capath[0]= 0; ssl_cert[0]= 0; @@ -78,6 +80,8 @@ Master_info::Master_info(LEX_STRING *connection_name_arg, bzero((char*) &file, sizeof(file)); mysql_mutex_init(key_master_info_run_lock, &run_lock, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_master_info_data_lock, &data_lock, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_master_info_start_stop_lock, &start_stop_lock, + MY_MUTEX_INIT_SLOW); mysql_mutex_setflags(&run_lock, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_setflags(&data_lock, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_init(key_master_info_sleep_lock, &sleep_lock, MY_MUTEX_INIT_FAST); @@ -87,8 +91,27 @@ Master_info::Master_info(LEX_STRING *connection_name_arg, mysql_cond_init(key_master_info_sleep_cond, &sleep_cond, NULL); } + +/** + Wait until no one is using Master_info +*/ + +void Master_info::wait_until_free() +{ + mysql_mutex_lock(&sleep_lock); + killed= 1; + while (users) + mysql_cond_wait(&sleep_cond, &sleep_lock); + mysql_mutex_unlock(&sleep_lock); +} + +/** + Delete master_info +*/ + Master_info::~Master_info() { + wait_until_free(); rpl_filters.delete_element(connection_name.str, connection_name.length, (void (*)(const char*, uchar*)) free_rpl_filter); my_free(connection_name.str); @@ -96,6 +119,7 @@ Master_info::~Master_info() mysql_mutex_destroy(&run_lock); mysql_mutex_destroy(&data_lock); mysql_mutex_destroy(&sleep_lock); + mysql_mutex_destroy(&start_stop_lock); mysql_cond_destroy(&data_cond); mysql_cond_destroy(&start_cond); mysql_cond_destroy(&stop_cond); @@ -706,12 +730,28 @@ uchar *get_key_master_info(Master_info *mi, size_t *length, return (uchar*) mi->cmp_connection_name.str; } +/* + Delete a master info + + Called from my_hash_delete(&master_info_hash) + Stops associated slave threads and frees master_info +*/ + void free_key_master_info(Master_info *mi) { DBUG_ENTER("free_key_master_info"); + mysql_mutex_unlock(&LOCK_active_mi); + + /* Ensure that we are not in reset_slave while this is done */ + mi->lock_slave_threads(); terminate_slave_threads(mi,SLAVE_FORCE_ALL); + /* We use 2 here instead of 1 just to make it easier when debugging */ + mi->killed= 2; end_master_info(mi); + mi->unlock_slave_threads(); delete mi; + + mysql_mutex_lock(&LOCK_active_mi); DBUG_VOID_RETURN; } @@ -867,9 +907,28 @@ Master_info_index::Master_info_index() index_file.file= -1; } + +/** + Free all connection threads + + This is done during early stages of shutdown + to give connection threads and slave threads time + to die before ~Master_info_index is called +*/ + +void Master_info_index::free_connections() +{ + mysql_mutex_assert_owner(&LOCK_active_mi); + my_hash_reset(&master_info_hash); +} + + +/** + Free all connection threads and free structures +*/ + Master_info_index::~Master_info_index() { - /* This will close connection for all objects in the cache */ my_hash_free(&master_info_hash); end_io_cache(&index_file); if (index_file.file >= 0) @@ -890,9 +949,9 @@ bool Master_info_index::init_all_master_info() int err_num= 0, succ_num= 0; // The number of success read Master_info char sign[MAX_CONNECTION_NAME+1]; File index_file_nr; + THD *thd; DBUG_ENTER("init_all_master_info"); - mysql_mutex_assert_owner(&LOCK_active_mi); DBUG_ASSERT(master_info_index); if ((index_file_nr= my_open(index_file_name, @@ -922,6 +981,10 @@ bool Master_info_index::init_all_master_info() DBUG_RETURN(1); } + thd= new THD; /* Needed by start_slave_threads */ + thd->thread_stack= (char*) &thd; + thd->store_globals(); + reinit_io_cache(&index_file, READ_CACHE, 0L,0,0); while (!init_strvar_from_file(sign, sizeof(sign), &index_file, NULL)) @@ -937,10 +1000,9 @@ bool Master_info_index::init_all_master_info() mi->error()) { delete mi; - DBUG_RETURN(1); + goto error; } - lock_slave_threads(mi); init_thread_mask(&thread_mask,mi,0 /*not inverse*/); create_logfile_name_with_suffix(buf_master_info_file, @@ -955,6 +1017,7 @@ bool Master_info_index::init_all_master_info() sql_print_information("Reading Master_info: '%s' Relay_info:'%s'", buf_master_info_file, buf_relay_log_info_file); + mi->lock_slave_threads(); if (init_master_info(mi, buf_master_info_file, buf_relay_log_info_file, 0, thread_mask)) { @@ -966,16 +1029,17 @@ bool Master_info_index::init_all_master_info() { /* Master_info is not in HASH; Add it */ if (master_info_index->add_master_info(mi, FALSE)) - DBUG_RETURN(1); + goto error; succ_num++; - unlock_slave_threads(mi); + mi->unlock_slave_threads(); } else { /* Master_info already in HASH */ sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), + (int) connection_name.length, connection_name.str, (int) connection_name.length, connection_name.str); - unlock_slave_threads(mi); + mi->unlock_slave_threads(); delete mi; } continue; @@ -991,22 +1055,22 @@ bool Master_info_index::init_all_master_info() { /* Master_info was already registered */ sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), + (int) connection_name.length, connection_name.str, (int) connection_name.length, connection_name.str); - unlock_slave_threads(mi); + mi->unlock_slave_threads(); delete mi; continue; } /* Master_info was not registered; add it */ if (master_info_index->add_master_info(mi, FALSE)) - DBUG_RETURN(1); + goto error; succ_num++; - unlock_slave_threads(mi); if (!opt_skip_slave_start) { if (start_slave_threads(1 /* need mutex */, - 0 /* no wait for start*/, + 1 /* wait for start*/, mi, buf_master_info_file, buf_relay_log_info_file, @@ -1022,8 +1086,11 @@ bool Master_info_index::init_all_master_info() (int) connection_name.length, connection_name.str); } + mi->unlock_slave_threads(); } } + thd->reset_globals(); + delete thd; if (!err_num) // No Error on read Master_info { @@ -1031,16 +1098,19 @@ bool Master_info_index::init_all_master_info() sql_print_information("Reading of all Master_info entries succeded"); DBUG_RETURN(0); } - else if (succ_num) // Have some Error and some Success + if (succ_num) // Have some Error and some Success { sql_print_warning("Reading of some Master_info entries failed"); DBUG_RETURN(2); } - else // All failed - { - sql_print_error("Reading of all Master_info entries failed!"); - DBUG_RETURN(1); - } + + sql_print_error("Reading of all Master_info entries failed!"); + DBUG_RETURN(1); + +error: + thd->reset_globals(); + delete thd; + DBUG_RETURN(1); } @@ -1073,6 +1143,71 @@ bool Master_info_index::write_master_name_to_index_file(LEX_STRING *name, /** + Get Master_info for a connection and lock the object from deletion + + @param + connection_name Connection name + warning WARN_LEVEL_NOTE -> Don't print anything + WARN_LEVEL_WARN -> Issue warning if not exists + WARN_LEVEL_ERROR-> Issue error if not exists +*/ + +Master_info *get_master_info(LEX_STRING *connection_name, + Sql_condition::enum_warning_level warning) +{ + Master_info *mi; + DBUG_ENTER("get_master_info"); + + /* Protect against inserts into hash */ + mysql_mutex_lock(&LOCK_active_mi); + /* + The following can only be true during shutdown when slave has been killed + but some other threads are still trying to access slave statistics. + */ + if (unlikely(!master_info_index)) + { + if (warning != Sql_condition::WARN_LEVEL_NOTE) + my_error(WARN_NO_MASTER_INFO, + MYF(warning == Sql_condition::WARN_LEVEL_WARN ? + ME_JUST_WARNING : 0), + (int) connection_name->length, connection_name->str); + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(0); + } + if ((mi= master_info_index->get_master_info(connection_name, warning))) + { + /* + We have to use sleep_lock here. If we would use LOCK_active_mi + then we would take locks in wrong order in Master_info::release() + */ + mysql_mutex_lock(&mi->sleep_lock); + mi->users++; + DBUG_PRINT("info",("users: %d", mi->users)); + mysql_mutex_unlock(&mi->sleep_lock); + } + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(mi); +} + + +/** + Release master info. + Signals ~Master_info that it's now safe to delete it +*/ + +void Master_info::release() +{ + mysql_mutex_lock(&sleep_lock); + if (!--users && killed) + { + /* Signal ~Master_info that it's ok to now free it */ + mysql_cond_signal(&sleep_cond); + } + mysql_mutex_unlock(&sleep_lock); +} + + +/** Get Master_info for a connection @param @@ -1094,8 +1229,6 @@ Master_info_index::get_master_info(LEX_STRING *connection_name, ("connection_name: '%.*s'", (int) connection_name->length, connection_name->str)); - mysql_mutex_assert_owner(&LOCK_active_mi); - /* Make name lower case for comparison */ res= strmake(buff, connection_name->str, connection_name->length); my_casedn_str(system_charset_info, buff); @@ -1161,7 +1294,12 @@ bool Master_info_index::check_duplicate_master_info(LEX_STRING *name_arg, /* Add a Master_info class to Hash Table */ bool Master_info_index::add_master_info(Master_info *mi, bool write_to_file) { - if (!my_hash_insert(&master_info_hash, (uchar*) mi)) + /* + We have to protect against shutdown to ensure we are not calling + my_hash_insert() while my_hash_free() is in progress + */ + if (unlikely(shutdown_in_progress) || + !my_hash_insert(&master_info_hash, (uchar*) mi)) { if (global_system_variables.log_warnings > 1) sql_print_information("Added new Master_info '%.*s' to hash table", @@ -1187,105 +1325,131 @@ bool Master_info_index::add_master_info(Master_info *mi, bool write_to_file) atomic */ -bool Master_info_index::remove_master_info(LEX_STRING *name) +bool Master_info_index::remove_master_info(Master_info *mi) { - Master_info* mi; DBUG_ENTER("remove_master_info"); + mysql_mutex_assert_owner(&LOCK_active_mi); - if ((mi= get_master_info(name, Sql_condition::WARN_LEVEL_WARN))) + // Delete Master_info and rewrite others to file + if (!my_hash_delete(&master_info_hash, (uchar*) mi)) { - // Delete Master_info and rewrite others to file - if (!my_hash_delete(&master_info_hash, (uchar*) mi)) + File index_file_nr; + + // Close IO_CACHE and FILE handler fisrt + end_io_cache(&index_file); + my_close(index_file.file, MYF(MY_WME)); + + // Reopen File and truncate it + if ((index_file_nr= my_open(index_file_name, + O_RDWR | O_CREAT | O_TRUNC | O_BINARY , + MYF(MY_WME))) < 0 || + init_io_cache(&index_file, index_file_nr, + IO_SIZE, WRITE_CACHE, + my_seek(index_file_nr,0L,MY_SEEK_END,MYF(0)), + 0, MYF(MY_WME | MY_WAIT_IF_FULL))) { - File index_file_nr; - - // Close IO_CACHE and FILE handler fisrt - end_io_cache(&index_file); - my_close(index_file.file, MYF(MY_WME)); - - // Reopen File and truncate it - if ((index_file_nr= my_open(index_file_name, - O_RDWR | O_CREAT | O_TRUNC | O_BINARY , - MYF(MY_WME))) < 0 || - init_io_cache(&index_file, index_file_nr, - IO_SIZE, WRITE_CACHE, - my_seek(index_file_nr,0L,MY_SEEK_END,MYF(0)), - 0, MYF(MY_WME | MY_WAIT_IF_FULL))) - { - int error= my_errno; - if (index_file_nr >= 0) - my_close(index_file_nr,MYF(0)); - - sql_print_error("Create of Master Info Index file '%s' failed with " - "error: %M", - index_file_name, error); - DBUG_RETURN(TRUE); - } + int error= my_errno; + if (index_file_nr >= 0) + my_close(index_file_nr,MYF(0)); - // Rewrite Master_info.index - for (uint i= 0; i< master_info_hash.records; ++i) - { - Master_info *tmp_mi; - tmp_mi= (Master_info *) my_hash_element(&master_info_hash, i); - write_master_name_to_index_file(&tmp_mi->connection_name, 0); - } - my_sync(index_file_nr, MYF(MY_WME)); + sql_print_error("Create of Master Info Index file '%s' failed with " + "error: %M", + index_file_name, error); + DBUG_RETURN(TRUE); } + + // Rewrite Master_info.index + for (uint i= 0; i< master_info_hash.records; ++i) + { + Master_info *tmp_mi; + tmp_mi= (Master_info *) my_hash_element(&master_info_hash, i); + write_master_name_to_index_file(&tmp_mi->connection_name, 0); + } + if (my_sync(index_file_nr, MYF(MY_WME))) + DBUG_RETURN(TRUE); } DBUG_RETURN(FALSE); } /** - Master_info_index::give_error_if_slave_running() + give_error_if_slave_running() + + @param + already_locked 0 if we need to lock, 1 if we have LOCK_active_mi_locked @return TRUE If some slave is running. An error is printed FALSE No slave is running */ -bool Master_info_index::give_error_if_slave_running() +bool give_error_if_slave_running(bool already_locked) { + bool ret= 0; DBUG_ENTER("give_error_if_slave_running"); - mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + if (!already_locked) + mysql_mutex_lock(&LOCK_active_mi); + if (!master_info_index) { - Master_info *mi; - mi= (Master_info *) my_hash_element(&master_info_hash, i); - if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + my_error(ER_SERVER_SHUTDOWN, MYF(0)); + ret= 1; + } + else + { + HASH *hash= &master_info_index->master_info_hash; + for (uint i= 0; i< hash->records; ++i) { - my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, - mi->connection_name.str); - DBUG_RETURN(TRUE); + Master_info *mi; + mi= (Master_info *) my_hash_element(hash, i); + if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + { + my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, + mi->connection_name.str); + ret= 1; + break; + } } } - DBUG_RETURN(FALSE); + if (!already_locked) + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(ret); } /** - Master_info_index::any_slave_sql_running() - - The LOCK_active_mi must be held while calling this function. + any_slave_sql_running() @return 0 No Slave SQL thread is running # Number of slave SQL thread running + + Note that during shutdown we return 1. This is needed to ensure we + don't try to resize thread pool during shutdown as during shutdown + master_info_hash may be freeing the hash and during that time + hash entries can't be accessed. */ -uint Master_info_index::any_slave_sql_running() +uint any_slave_sql_running() { uint count= 0; + HASH *hash; DBUG_ENTER("any_slave_sql_running"); - mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + mysql_mutex_lock(&LOCK_active_mi); + if (unlikely(shutdown_in_progress || !master_info_index)) + { + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(1); + } + hash= &master_info_index->master_info_hash; + for (uint i= 0; i< hash->records; ++i) { - Master_info *mi= (Master_info *)my_hash_element(&master_info_hash, i); + Master_info *mi= (Master_info *)my_hash_element(hash, i); if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) count++; } + mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(count); } @@ -1298,15 +1462,25 @@ uint Master_info_index::any_slave_sql_running() @return TRUE Error FALSE Everything ok. + + This code is written so that we don't keep LOCK_active_mi active + while we are starting a slave. */ bool Master_info_index::start_all_slaves(THD *thd) { bool result= FALSE; - DBUG_ENTER("warn_if_slave_running"); + DBUG_ENTER("start_all_slaves"); mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + for (uint i= 0; i< master_info_hash.records; i++) + { + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + mi->in_start_all_slaves= 0; + } + + for (uint i= 0; i< master_info_hash.records; ) { int error; Master_info *mi; @@ -1316,25 +1490,40 @@ bool Master_info_index::start_all_slaves(THD *thd) Try to start all slaves that are configured (host is defined) and are not already running */ - if ((mi->slave_running == MYSQL_SLAVE_NOT_RUN || - !mi->rli.slave_running) && *mi->host) + if (!((mi->slave_running == MYSQL_SLAVE_NOT_RUN || + !mi->rli.slave_running) && *mi->host) || + mi->in_start_all_slaves) { - if ((error= start_slave(thd, mi, 1))) - { - my_error(ER_CANT_START_STOP_SLAVE, MYF(0), - "START", - (int) mi->connection_name.length, - mi->connection_name.str); - result= 1; - if (error < 0) // fatal error - break; - } - else - push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, - ER_SLAVE_STARTED, ER(ER_SLAVE_STARTED), - (int) mi->connection_name.length, - mi->connection_name.str); + i++; + continue; + } + mi->in_start_all_slaves= 1; + + mysql_mutex_lock(&mi->sleep_lock); + mi->users++; // Mark used + mysql_mutex_unlock(&mi->sleep_lock); + mysql_mutex_unlock(&LOCK_active_mi); + error= start_slave(thd, mi, 1); + mi->release(); + mysql_mutex_lock(&LOCK_active_mi); + if (error) + { + my_error(ER_CANT_START_STOP_SLAVE, MYF(0), + "START", + (int) mi->connection_name.length, + mi->connection_name.str); + result= 1; + if (error < 0) // fatal error + break; } + else + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_SLAVE_STARTED, ER(ER_SLAVE_STARTED), + (int) mi->connection_name.length, + mi->connection_name.str); + /* Restart from first element as master_info_hash may have changed */ + i= 0; + continue; } DBUG_RETURN(result); } @@ -1348,38 +1537,63 @@ bool Master_info_index::start_all_slaves(THD *thd) @return TRUE Error FALSE Everything ok. + + This code is written so that we don't keep LOCK_active_mi active + while we are stopping a slave. */ bool Master_info_index::stop_all_slaves(THD *thd) { bool result= FALSE; - DBUG_ENTER("warn_if_slave_running"); + DBUG_ENTER("stop_all_slaves"); mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + for (uint i= 0; i< master_info_hash.records; i++) + { + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + mi->in_stop_all_slaves= 0; + } + + for (uint i= 0; i< master_info_hash.records ;) { int error; Master_info *mi; mi= (Master_info *) my_hash_element(&master_info_hash, i); - if ((mi->slave_running != MYSQL_SLAVE_NOT_RUN || - mi->rli.slave_running)) + if (!(mi->slave_running != MYSQL_SLAVE_NOT_RUN || + mi->rli.slave_running) || + mi->in_stop_all_slaves) { - if ((error= stop_slave(thd, mi, 1))) - { - my_error(ER_CANT_START_STOP_SLAVE, MYF(0), - "STOP", - (int) mi->connection_name.length, - mi->connection_name.str); - result= 1; - if (error < 0) // Fatal error - break; - } - else - push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, - ER_SLAVE_STOPPED, ER(ER_SLAVE_STOPPED), - (int) mi->connection_name.length, - mi->connection_name.str); + i++; + continue; } + mi->in_stop_all_slaves= 1; // Protection for loops + + mysql_mutex_lock(&mi->sleep_lock); + mi->users++; // Mark used + mysql_mutex_unlock(&mi->sleep_lock); + mysql_mutex_unlock(&LOCK_active_mi); + error= stop_slave(thd, mi, 1); + mi->release(); + mysql_mutex_lock(&LOCK_active_mi); + if (error) + { + my_error(ER_CANT_START_STOP_SLAVE, MYF(0), + "STOP", + (int) mi->connection_name.length, + mi->connection_name.str); + result= 1; + if (error < 0) // Fatal error + break; + } + else + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_SLAVE_STOPPED, ER(ER_SLAVE_STOPPED), + (int) mi->connection_name.length, + mi->connection_name.str); + /* Restart from first element as master_info_hash may have changed */ + i= 0; + continue; } DBUG_RETURN(result); } diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index e58df0150f5..5c42378ca2c 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -79,6 +79,10 @@ class Master_info : public Slave_reporting_capability { return opt_slave_parallel_threads > 0; } + void release(); + void wait_until_free(); + void lock_slave_threads(); + void unlock_slave_threads(); /* the variables below are needed because we can change masters on the fly */ char master_log_name[FN_REFLEN+6]; /* Room for multi-*/ @@ -97,7 +101,7 @@ class Master_info : public Slave_reporting_capability File fd; // we keep the file open, so we need to remember the file pointer IO_CACHE file; - mysql_mutex_t data_lock, run_lock, sleep_lock; + mysql_mutex_t data_lock, run_lock, sleep_lock, start_stop_lock; mysql_cond_t data_cond, start_cond, stop_cond, sleep_cond; THD *io_thd; MYSQL* mysql; @@ -182,7 +186,11 @@ class Master_info : public Slave_reporting_capability uint64 gtid_reconnect_event_skip_count; /* gtid_event_seen is false until we receive first GTID event from master. */ bool gtid_event_seen; + bool in_start_all_slaves, in_stop_all_slaves; + uint users; /* Active user for object */ + uint killed; }; + int init_master_info(Master_info* mi, const char* master_info_fname, const char* slave_info_fname, bool abort_if_no_master_info_file, @@ -218,13 +226,12 @@ public: bool check_duplicate_master_info(LEX_STRING *connection_name, const char *host, uint port); bool add_master_info(Master_info *mi, bool write_to_file); - bool remove_master_info(LEX_STRING *connection_name); + bool remove_master_info(Master_info *mi); Master_info *get_master_info(LEX_STRING *connection_name, Sql_condition::enum_warning_level warning); - bool give_error_if_slave_running(); - uint any_slave_sql_running(); bool start_all_slaves(THD *thd); bool stop_all_slaves(THD *thd); + void free_connections(); }; @@ -237,6 +244,8 @@ public: }; +Master_info *get_master_info(LEX_STRING *connection_name, + Sql_condition::enum_warning_level warning); bool check_master_connection_name(LEX_STRING *name); void create_logfile_name_with_suffix(char *res_file_name, size_t length, const char *info_file, @@ -246,7 +255,8 @@ void create_logfile_name_with_suffix(char *res_file_name, size_t length, uchar *get_key_master_info(Master_info *mi, size_t *length, my_bool not_used __attribute__((unused))); void free_key_master_info(Master_info *mi); - +uint any_slave_sql_running(); +bool give_error_if_slave_running(bool already_lock); #endif /* HAVE_REPLICATION */ #endif /* RPL_MI_H */ diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 23f61a82a90..43e52c00e8d 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -1109,7 +1109,8 @@ handle_rpl_parallel_thread(void *arg) thd->wait_for_commit_ptr= &rgi->commit_orderer; - if (opt_gtid_ignore_duplicates) + if (opt_gtid_ignore_duplicates && + rgi->rli->mi->using_gtid != Master_info::USE_GTID_NO) { int res= rpl_global_gtid_slave_state->check_duplicate_gtid(&rgi->current_gtid, @@ -1253,39 +1254,16 @@ handle_rpl_parallel_thread(void *arg) */ rpt->batch_free(); - for (;;) + if ((events= rpt->event_queue) != NULL) { - if ((events= rpt->event_queue) != NULL) - { - /* - Take next group of events from the replication pool. - This is faster than having to wakeup the pool manager thread to give - us a new event. - */ - rpt->dequeue1(events); - mysql_mutex_unlock(&rpt->LOCK_rpl_thread); - goto more_events; - } - if (!rpt->pause_for_ftwrl || - (in_event_group && !group_rgi->parallel_entry->force_abort)) - break; /* - We are currently in the delicate process of pausing parallel - replication while FLUSH TABLES WITH READ LOCK is starting. We must - not de-allocate the thread (setting rpt->current_owner= NULL) until - rpl_unpause_after_ftwrl() has woken us up. + Take next group of events from the replication pool. + This is faster than having to wakeup the pool manager thread to give + us a new event. */ - mysql_mutex_lock(&rpt->current_entry->LOCK_parallel_entry); + rpt->dequeue1(events); mysql_mutex_unlock(&rpt->LOCK_rpl_thread); - if (rpt->pause_for_ftwrl) - mysql_cond_wait(&rpt->current_entry->COND_parallel_entry, - &rpt->current_entry->LOCK_parallel_entry); - mysql_mutex_unlock(&rpt->current_entry->LOCK_parallel_entry); - mysql_mutex_lock(&rpt->LOCK_rpl_thread); - /* - Now loop to check again for more events available, since we released - and re-aquired the LOCK_rpl_thread mutex. - */ + goto more_events; } rpt->inuse_relaylog_refcount_update(); @@ -1312,11 +1290,35 @@ handle_rpl_parallel_thread(void *arg) } if (!in_event_group) { + /* If we are in a FLUSH TABLES FOR READ LOCK, wait for it */ + while (rpt->current_entry && rpt->pause_for_ftwrl) + { + /* + We are currently in the delicate process of pausing parallel + replication while FLUSH TABLES WITH READ LOCK is starting. We must + not de-allocate the thread (setting rpt->current_owner= NULL) until + rpl_unpause_after_ftwrl() has woken us up. + */ + rpl_parallel_entry *e= rpt->current_entry; + /* + Wait for rpl_unpause_after_ftwrl() to wake us up. + Note that rpl_pause_for_ftwrl() may wait for 'e->pause_sub_id' + to change. This should happen eventually in finish_event_group() + */ + mysql_mutex_lock(&e->LOCK_parallel_entry); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + if (rpt->pause_for_ftwrl) + mysql_cond_wait(&e->COND_parallel_entry, &e->LOCK_parallel_entry); + mysql_mutex_unlock(&e->LOCK_parallel_entry); + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + } + rpt->current_owner= NULL; /* Tell wait_for_done() that we are done, if it is waiting. */ if (likely(rpt->current_entry) && unlikely(rpt->current_entry->force_abort)) mysql_cond_broadcast(&rpt->COND_rpl_thread_stop); + rpt->current_entry= NULL; if (!rpt->stop) rpt->pool->release_thread(rpt); @@ -1355,10 +1357,24 @@ dealloc_gco(group_commit_orderer *gco) my_free(gco); } +/** + Change thread count for global parallel worker threads + + @param pool parallel thread pool + @param new_count Number of threads to be in pool. 0 in shutdown + @param force Force thread count to new_count even if slave + threads are running + + By default we don't resize pool of there are running threads. + However during shutdown we will always do it. + This is needed as any_slave_sql_running() returns 1 during shutdown + as we don't want to access master_info while + Master_info_index::free_connections are running. +*/ static int rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, - uint32 new_count) + uint32 new_count, bool force) { uint32 i; rpl_parallel_thread **new_list= NULL; @@ -1369,6 +1385,28 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, if ((res= pool_mark_busy(pool, current_thd))) return res; + /* Protect against parallel pool resizes */ + if (pool->count == new_count) + { + pool_mark_not_busy(pool); + return 0; + } + + /* + If we are about to delete pool, do an extra check that there are no new + slave threads running since we marked pool busy + */ + if (!new_count && !force) + { + if (any_slave_sql_running()) + { + DBUG_PRINT("warning", + ("SQL threads running while trying to reset parallel pool")); + pool_mark_not_busy(pool); + return 0; // Ok to not resize pool + } + } + /* Allocate the new list of threads up-front. That way, if we fail half-way, we only need to free whatever we managed @@ -1382,7 +1420,7 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, { my_error(ER_OUTOFMEMORY, MYF(0), (int(new_count*sizeof(*new_list) + new_count*sizeof(*rpt_array)))); - goto err;; + goto err; } for (i= 0; i < new_count; ++i) @@ -1503,12 +1541,26 @@ err: return 1; } +/* + Deactivate the parallel replication thread pool, if there are now no more + SQL threads running. +*/ + +int rpl_parallel_resize_pool_if_no_slaves(void) +{ + /* master_info_index is set to NULL on shutdown */ + if (opt_slave_parallel_threads > 0 && !any_slave_sql_running()) + return rpl_parallel_inactivate_pool(&global_rpl_thread_pool); + return 0; +} + int rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool) { if (!pool->count) - return rpl_parallel_change_thread_count(pool, opt_slave_parallel_threads); + return rpl_parallel_change_thread_count(pool, opt_slave_parallel_threads, + 0); return 0; } @@ -1516,7 +1568,7 @@ rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool) int rpl_parallel_inactivate_pool(rpl_parallel_thread_pool *pool) { - return rpl_parallel_change_thread_count(pool, 0); + return rpl_parallel_change_thread_count(pool, 0, 0); } @@ -1795,7 +1847,7 @@ rpl_parallel_thread_pool::destroy() { if (!inited) return; - rpl_parallel_change_thread_count(this, 0); + rpl_parallel_change_thread_count(this, 0, 1); mysql_mutex_destroy(&LOCK_rpl_thread_pool); mysql_cond_destroy(&COND_rpl_thread_pool); inited= false; @@ -1814,6 +1866,7 @@ rpl_parallel_thread_pool::get_thread(rpl_parallel_thread **owner, { rpl_parallel_thread *rpt; + DBUG_ASSERT(count > 0); mysql_mutex_lock(&LOCK_rpl_thread_pool); while (unlikely(busy) || !(rpt= free_list)) mysql_cond_wait(&COND_rpl_thread_pool, &LOCK_rpl_thread_pool); @@ -2042,6 +2095,11 @@ rpl_parallel::find(uint32 domain_id) return e; } +/** + Wait until all sql worker threads has stopped processing + + This is called when sql thread has been killed/stopped +*/ void rpl_parallel::wait_for_done(THD *thd, Relay_log_info *rli) diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h index a02c1af3b3e..0d7cd4f2e9b 100644 --- a/sql/rpl_parallel.h +++ b/sql/rpl_parallel.h @@ -342,6 +342,7 @@ struct rpl_parallel { extern struct rpl_parallel_thread_pool global_rpl_thread_pool; +extern int rpl_parallel_resize_pool_if_no_slaves(void); extern int rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool); extern int rpl_parallel_inactivate_pool(rpl_parallel_thread_pool *pool); extern bool process_gtid_for_restart_pos(Relay_log_info *rli, rpl_gtid *gtid); diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index 183248ad1b8..5e48cfb02e5 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -390,6 +390,12 @@ unpack_row(rpl_group_info *rgi, } /* + Add Extra slave persistent columns + */ + if ((error= fill_extra_persistent_columns(table, cols->n_bits))) + DBUG_RETURN(error); + + /* We should now have read all the null bytes, otherwise something is really wrong. */ @@ -461,5 +467,30 @@ int prepare_record(TABLE *const table, const uint skip, const bool check) DBUG_RETURN(0); } +/** + Fills @c table->record[0] with computed values of extra persistent column which are present on slave but not on master. + @param table Table whose record[0] buffer is prepared. + @param master_cols No of columns on master + @returns 0 on success + */ +int fill_extra_persistent_columns(TABLE *table, int master_cols) +{ + int error= 0; + Field **vfield_ptr, *vfield; + if (!table->vfield) + return 0; + for (vfield_ptr= table->vfield; *vfield_ptr; ++vfield_ptr) + { + vfield= *vfield_ptr; + if (vfield->field_index >= master_cols && vfield->stored_in_db) + { + /*Set bitmap for writing*/ + bitmap_set_bit(table->vcol_set, vfield->field_index); + error= vfield->vcol_info->expr_item->save_in_field(vfield,0); + bitmap_clear_bit(table->vcol_set, vfield->field_index); + } + } + return error; +} #endif // HAVE_REPLICATION diff --git a/sql/rpl_record.h b/sql/rpl_record.h index c10eb8225b0..be69716d9d5 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -38,6 +38,7 @@ int unpack_row(rpl_group_info *rgi, // Fill table's record[0] with default values. int prepare_record(TABLE *const table, const uint skip, const bool check); +int fill_extra_persistent_columns(TABLE *table, int master_cols); #endif #endif diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index c570105cdf6..928fbd3d7c1 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -208,6 +208,7 @@ a file name for --relay-log-index option", opt_relaylog_index_name); Master_info* mi= rli->mi; char buf_relay_logname[FN_REFLEN], buf_relaylog_index_name_buff[FN_REFLEN]; char *buf_relaylog_index_name= opt_relaylog_index_name; + mysql_mutex_t *log_lock; create_logfile_name_with_suffix(buf_relay_logname, sizeof(buf_relay_logname), @@ -227,14 +228,18 @@ a file name for --relay-log-index option", opt_relaylog_index_name); note, that if open() fails, we'll still have index file open but a destructor will take care of that */ + log_lock= rli->relay_log.get_log_lock(); + mysql_mutex_lock(log_lock); if (rli->relay_log.open_index_file(buf_relaylog_index_name, ln, TRUE) || rli->relay_log.open(ln, LOG_BIN, 0, SEQ_READ_APPEND, mi->rli.max_relay_log_size, 1, TRUE)) { + mysql_mutex_unlock(log_lock); mysql_mutex_unlock(&rli->data_lock); sql_print_error("Failed when trying to open logs for '%s' in init_relay_log_info(). Error: %M", ln, my_errno); DBUG_RETURN(1); } + mysql_mutex_unlock(log_lock); } /* if file does not exist */ @@ -432,7 +437,7 @@ Failed to open the existing relay log info file '%s' (errno %d)", } rli->inited= 1; mysql_mutex_unlock(&rli->data_lock); - DBUG_RETURN(error); + DBUG_RETURN(0); err: sql_print_error("%s", msg); @@ -1304,9 +1309,10 @@ bool Relay_log_info::is_until_satisfied(THD *thd, Log_event *ev) } -void Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, +bool Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, rpl_group_info *rgi) { + int error= 0; DBUG_ENTER("Relay_log_info::stmt_done"); DBUG_ASSERT(rgi->rli == this); @@ -1358,10 +1364,11 @@ void Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, } DBUG_EXECUTE_IF("inject_crash_before_flush_rli", DBUG_SUICIDE();); if (mi->using_gtid == Master_info::USE_GTID_NO) - flush_relay_log_info(this); + if (flush_relay_log_info(this)) + error= 1; DBUG_EXECUTE_IF("inject_crash_after_flush_rli", DBUG_SUICIDE();); } - DBUG_VOID_RETURN; + DBUG_RETURN(error); } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index efcec83b880..7fc41786957 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -416,7 +416,7 @@ public: relay log info and used to produce information for <code>SHOW SLAVE STATUS</code>. */ - void stmt_done(my_off_t event_log_pos, THD *thd, rpl_group_info *rgi); + bool stmt_done(my_off_t event_log_pos, THD *thd, rpl_group_info *rgi); int alloc_inuse_relaylog(const char *name); void free_inuse_relaylog(inuse_relaylog *ir); void reset_inuse_relaylog(); diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index d8c64a9d534..de0daf759fe 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -4841,7 +4841,7 @@ WARN_DATA_TRUNCATED 01000 spa "Datos truncados para columna '%s' en la lÃnea %lu" ER_WARN_USING_OTHER_HANDLER eng "Using storage engine %s for table '%s'" - ger "Für Tabelle '%s' wird Speicher-Engine %s benutzt" + ger "Speicher-Engine %s wird für Tabelle '%s' benutzt" jpn "ストレージエンジン %s ãŒè¡¨ '%s' ã«åˆ©ç”¨ã•ã‚Œã¦ã„ã¾ã™ã€‚" por "Usando engine de armazenamento %s para tabela '%s'" spa "Usando motor de almacenamiento %s para tabla '%s'" diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index c3f25848e8a..2029c52d17d 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -149,7 +149,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) if (opt_stack_trace) { - my_safe_printf_stderr("Thread pointer: 0x%p\n", thd); + my_safe_printf_stderr("Thread pointer: %p\n", thd); my_safe_printf_stderr("%s", "Attempting backtrace. You can use the following " "information to find out\n" diff --git a/sql/slave.cc b/sql/slave.cc index 65bcdc48c6a..3dd17fd7276 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -228,16 +228,14 @@ void init_thread_mask(int* mask,Master_info* mi,bool inverse) /* - lock_slave_threads() + lock_slave_threads() against other threads doing STOP, START or RESET SLAVE + */ -void lock_slave_threads(Master_info* mi) +void Master_info::lock_slave_threads() { DBUG_ENTER("lock_slave_threads"); - - //TODO: see if we can do this without dual mutex - mysql_mutex_lock(&mi->run_lock); - mysql_mutex_lock(&mi->rli.run_lock); + mysql_mutex_lock(&start_stop_lock); DBUG_VOID_RETURN; } @@ -246,13 +244,10 @@ void lock_slave_threads(Master_info* mi) unlock_slave_threads() */ -void unlock_slave_threads(Master_info* mi) +void Master_info::unlock_slave_threads() { DBUG_ENTER("unlock_slave_threads"); - - //TODO: see if we can do this without dual mutex - mysql_mutex_unlock(&mi->rli.run_lock); - mysql_mutex_unlock(&mi->run_lock); + mysql_mutex_unlock(&start_stop_lock); DBUG_VOID_RETURN; } @@ -374,7 +369,6 @@ int init_slave() accepted. However bootstrap may conflict with us if it does START SLAVE. So it's safer to take the lock. */ - mysql_mutex_lock(&LOCK_active_mi); if (pthread_key_create(&RPL_MASTER_INFO, NULL)) goto err; @@ -383,7 +377,6 @@ int init_slave() if (!master_info_index || master_info_index->init_all_master_info()) { sql_print_error("Failed to initialize multi master structures"); - mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(1); } if (!(active_mi= new Master_info(&default_master_connection_name, @@ -428,12 +421,21 @@ int init_slave() if (active_mi->host[0] && !opt_skip_slave_start) { - if (start_slave_threads(1 /* need mutex */, - 0 /* no wait for start*/, - active_mi, - master_info_file, - relay_log_info_file, - SLAVE_IO | SLAVE_SQL)) + int error; + THD *thd= new THD; + thd->thread_stack= (char*) &thd; + thd->store_globals(); + + error= start_slave_threads(1 /* need mutex */, + 1 /* wait for start*/, + active_mi, + master_info_file, + relay_log_info_file, + SLAVE_IO | SLAVE_SQL); + + thd->reset_globals(); + delete thd; + if (error) { sql_print_error("Failed to create slave threads"); goto err; @@ -441,7 +443,6 @@ int init_slave() } end: - mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(error); err: @@ -614,6 +615,7 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) if (!mi->inited) DBUG_RETURN(0); /* successfully do nothing */ int error,force_all = (thread_mask & SLAVE_FORCE_ALL); + int retval= 0; mysql_mutex_t *sql_lock = &mi->rli.run_lock, *io_lock = &mi->run_lock; mysql_mutex_t *log_lock= mi->rli.relay_log.get_log_lock(); @@ -633,24 +635,19 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) skip_lock)) && !force_all) DBUG_RETURN(error); + retval= error; mysql_mutex_lock(log_lock); DBUG_PRINT("info",("Flushing relay-log info file.")); if (current_thd) THD_STAGE_INFO(current_thd, stage_flushing_relay_log_info_file); - if (flush_relay_log_info(&mi->rli)) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); - - if (my_sync(mi->rli.info_fd, MYF(MY_WME))) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); + if (flush_relay_log_info(&mi->rli) || + my_sync(mi->rli.info_fd, MYF(MY_WME))) + retval= ER_ERROR_DURING_FLUSH_LOGS; mysql_mutex_unlock(log_lock); } - if (opt_slave_parallel_threads > 0 && - master_info_index &&// master_info_index is set to NULL on server shutdown - !master_info_index->any_slave_sql_running()) - rpl_parallel_inactivate_pool(&global_rpl_thread_pool); if (thread_mask & (SLAVE_IO|SLAVE_FORCE_ALL)) { DBUG_PRINT("info",("Terminating IO thread")); @@ -661,25 +658,26 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) skip_lock)) && !force_all) DBUG_RETURN(error); + if (!retval) + retval= error; mysql_mutex_lock(log_lock); DBUG_PRINT("info",("Flushing relay log and master info file.")); if (current_thd) THD_STAGE_INFO(current_thd, stage_flushing_relay_log_and_master_info_repository); - if (flush_master_info(mi, TRUE, FALSE)) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); - + if (likely(mi->fd >= 0)) + { + if (flush_master_info(mi, TRUE, FALSE) || my_sync(mi->fd, MYF(MY_WME))) + retval= ER_ERROR_DURING_FLUSH_LOGS; + } if (mi->rli.relay_log.is_open() && my_sync(mi->rli.relay_log.get_log_file()->file, MYF(MY_WME))) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); - - if (my_sync(mi->fd, MYF(MY_WME))) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); + retval= ER_ERROR_DURING_FLUSH_LOGS; mysql_mutex_unlock(log_lock); } - DBUG_RETURN(0); + DBUG_RETURN(retval); } @@ -842,6 +840,15 @@ int start_slave_thread( mysql_mutex_unlock(start_lock); DBUG_RETURN(ER_SLAVE_THREAD); } + + /* + In the following loop we can't check for thd->killed as we have to + wait until THD structures for the slave thread are created + before we can return. + This should be ok as there is no major work done in the slave + threads before they signal that we can stop waiting. + */ + if (start_cond && cond_lock) // caller has cond_lock { THD* thd = current_thd; @@ -859,16 +866,9 @@ int start_slave_thread( registered, we could otherwise go waiting though thd->killed is set. */ - if (!thd->killed) - mysql_cond_wait(start_cond, cond_lock); + mysql_cond_wait(start_cond, cond_lock); thd->EXIT_COND(& saved_stage); mysql_mutex_lock(cond_lock); // re-acquire it as exit_cond() released - if (thd->killed) - { - if (start_lock) - mysql_mutex_unlock(start_lock); - DBUG_RETURN(thd->killed_errno()); - } } } if (start_lock) @@ -956,10 +956,7 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start, mi); if (!error && (thread_mask & SLAVE_SQL)) { - if (opt_slave_parallel_threads > 0) - error= rpl_parallel_activate_pool(&global_rpl_thread_pool); - if (!error) - error= start_slave_thread( + error= start_slave_thread( #ifdef HAVE_PSI_INTERFACE key_thread_slave_sql, #endif @@ -975,10 +972,18 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start, /* - Release slave threads at time of executing shutdown. + Kill slaves preparing for shutdown +*/ - SYNOPSIS - end_slave() +void slave_prepare_for_shutdown() +{ + mysql_mutex_lock(&LOCK_active_mi); + master_info_index->free_connections(); + mysql_mutex_unlock(&LOCK_active_mi); +} + +/* + Release slave threads at time of executing shutdown. */ void end_slave() @@ -996,7 +1001,10 @@ void end_slave() startup parameter to the server was wrong. */ mysql_mutex_lock(&LOCK_active_mi); - /* This will call terminate_slave_threads() on all connections */ + /* + master_info_index should not have any threads anymore as they where + killed as part of slave_prepare_for_shutdown() + */ delete master_info_index; master_info_index= 0; active_mi= 0; @@ -2657,7 +2665,9 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, mysql_mutex_lock(&mi->data_lock); mysql_mutex_lock(&mi->rli.data_lock); + /* err_lock is to protect mi->last_error() */ mysql_mutex_lock(&mi->err_lock); + /* err_lock is to protect mi->rli.last_error() */ mysql_mutex_lock(&mi->rli.err_lock); protocol->store(mi->host, &my_charset_bin); protocol->store(mi->user, &my_charset_bin); @@ -3602,7 +3612,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, DBUG_RETURN(1); } - if (opt_gtid_ignore_duplicates) + if (opt_gtid_ignore_duplicates && + rli->mi->using_gtid != Master_info::USE_GTID_NO) { int res= rpl_global_gtid_slave_state->check_duplicate_gtid (&serial_rgi->current_gtid, serial_rgi); @@ -4493,6 +4504,16 @@ pthread_handler_t handle_slave_sql(void *arg) rli->slave_running= MYSQL_SLAVE_RUN_NOT_CONNECT; pthread_detach_this_thread(); + + if (opt_slave_parallel_threads > 0 && + rpl_parallel_activate_pool(&global_rpl_thread_pool)) + { + mysql_cond_broadcast(&rli->start_cond); + rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, + "Failed during parallel slave pool activation"); + goto err_during_init; + } + if (init_slave_thread(thd, mi, SLAVE_THD_SQL)) { /* @@ -4574,11 +4595,11 @@ pthread_handler_t handle_slave_sql(void *arg) { rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, "Error initializing relay log position: %s", errmsg); - goto err; + goto err_before_start; } rli->reset_inuse_relaylog(); if (rli->alloc_inuse_relaylog(rli->group_relay_log_name)) - goto err; + goto err_before_start; strcpy(rli->future_event_master_log_name, rli->group_master_log_name); THD_CHECK_SENTRY(thd); @@ -4738,6 +4759,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, } } + err: if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); @@ -4757,15 +4779,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, tmp.c_ptr_safe()); } - err: - - /* - Once again, in case we aborted with an error and skipped the first one. - (We want the first one to be before the printout of stop position to - get the correct position printed.) - */ - if (mi->using_parallel()) - rli->parallel.wait_for_done(thd, rli); + err_before_start: /* Some events set some playgrounds, which won't be cleared because thread @@ -4786,8 +4800,15 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, if (rli->mi->using_gtid != Master_info::USE_GTID_NO) { ulong domain_count; + my_bool save_log_all_errors= thd->log_all_errors; + /* + We don't need to check return value for flush_relay_log_info() + as any errors should be logged to stderr + */ + thd->log_all_errors= 1; flush_relay_log_info(rli); + thd->log_all_errors= save_log_all_errors; if (mi->using_parallel()) { /* @@ -4869,17 +4890,7 @@ err_during_init: DBUG_EXECUTE_IF("simulate_slave_delay_at_terminate_bug38694", sleep(5);); mysql_mutex_unlock(&rli->run_lock); // tell the world we are done - /* - Deactivate the parallel replication thread pool, if there are now no more - SQL threads running. Do this here, when we have released all locks, but - while our THD (and current_thd) is still valid. - */ - mysql_mutex_lock(&LOCK_active_mi); - if (opt_slave_parallel_threads > 0 && - master_info_index &&// master_info_index is set to NULL on server shutdown - !master_info_index->any_slave_sql_running()) - rpl_parallel_inactivate_pool(&global_rpl_thread_pool); - mysql_mutex_unlock(&LOCK_active_mi); + rpl_parallel_resize_pool_if_no_slaves(); mysql_mutex_lock(&LOCK_thread_count); delete thd; @@ -5636,8 +5647,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) TODO: handling `when' for SHOW SLAVE STATUS' snds behind */ - if ((memcmp(mi->master_log_name, hb.get_log_ident(), hb.get_ident_len()) - && mi->master_log_name != NULL) + if (memcmp(mi->master_log_name, hb.get_log_ident(), hb.get_ident_len()) || mi->master_log_pos > hb.log_pos) { /* missed events of heartbeat from the past */ @@ -5989,6 +5999,7 @@ err: void end_relay_log_info(Relay_log_info* rli) { + mysql_mutex_t *log_lock; DBUG_ENTER("end_relay_log_info"); if (!rli->inited) @@ -6006,8 +6017,11 @@ void end_relay_log_info(Relay_log_info* rli) rli->cur_log_fd = -1; } rli->inited = 0; + log_lock= rli->relay_log.get_log_lock(); + mysql_mutex_lock(log_lock); rli->relay_log.close(LOG_CLOSE_INDEX | LOG_CLOSE_STOP_EVENT); rli->relay_log.harvest_bytes_written(&rli->log_space_total); + mysql_mutex_unlock(log_lock); /* Delete the slave's temporary tables from memory. In the future there will be other actions than this, to ensure persistance @@ -6138,7 +6152,7 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, mysql_options(mysql, MYSQL_PLUGIN_DIR, opt_plugin_dir_ptr); /* we disallow empty users */ - if (mi->user == NULL || mi->user[0] == 0) + if (mi->user[0] == 0) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER(ER_SLAVE_FATAL_ERROR), @@ -6159,7 +6173,7 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, suppress_warnings= 0; mi->report(ERROR_LEVEL, last_errno, NULL, "error %s to master '%s@%s:%d'" - " - retry-time: %d retries: %lu message: %s", + " - retry-time: %d maximum-retries: %lu message: %s", (reconnect ? "reconnecting" : "connecting"), mi->user, mi->host, mi->port, mi->connect_retry, master_retry_count, @@ -6723,9 +6737,12 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) } rli->event_relay_log_pos = BIN_LOG_HEADER_SIZE; strmake_buf(rli->event_relay_log_name,rli->linfo.log_file_name); - flush_relay_log_info(rli); + if (flush_relay_log_info(rli)) + { + errmsg= "error flushing relay log"; + goto err; + } } - /* Now we want to open this next log. To know if it's a hot log (the one being written by the I/O thread now) or a cold log, we can use diff --git a/sql/slave.h b/sql/slave.h index c6b78b96aca..abcb4df984b 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -213,13 +213,12 @@ bool rpl_master_erroneous_autoinc(THD* thd); const char *print_slave_db_safe(const char *db); void skip_load_data_infile(NET* net); +void slave_prepare_for_shutdown(); void end_slave(); /* release slave threads */ void close_active_mi(); /* clean up slave threads data */ void clear_until_condition(Relay_log_info* rli); void clear_slave_error(Relay_log_info* rli); void end_relay_log_info(Relay_log_info* rli); -void lock_slave_threads(Master_info* mi); -void unlock_slave_threads(Master_info* mi); void init_thread_mask(int* mask,Master_info* mi,bool inverse); Format_description_log_event * read_relay_log_description_event(IO_CACHE *cur_log, ulonglong start_pos, diff --git a/sql/sp.cc b/sql/sp.cc index 4b65f835f70..52d3c04cbdf 100644 --- a/sql/sp.cc +++ b/sql/sp.cc @@ -1,6 +1,6 @@ /* - Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2009, 2015, MariaDB + Copyright (c) 2002, 2016, Oracle and/or its affiliates. + Copyright (c) 2009, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/sql/sp_head.cc b/sql/sp_head.cc index b90108ed944..fb0f3132816 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -522,12 +522,8 @@ check_routine_name(LEX_STRING *ident) my_error(ER_SP_WRONG_NAME, MYF(0), ident->str); return TRUE; } - if (check_string_char_length(ident, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + if (check_ident_length(ident)) return TRUE; - } return FALSE; } @@ -3120,23 +3116,23 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) thd->query_length()) <= 0) { res= m_lex_keeper.reset_lex_and_exec_core(thd, nextp, FALSE, this); + bool log_slow= !res && thd->enable_slow_log; - if (thd->get_stmt_da()->is_eof()) - { - /* Finalize server status flags after executing a statement. */ + /* Finalize server status flags after executing a statement. */ + if (log_slow || thd->get_stmt_da()->is_eof()) thd->update_server_status(); + if (thd->get_stmt_da()->is_eof()) thd->protocol->end_statement(); - } query_cache_end_of_result(thd); mysql_audit_general(thd, MYSQL_AUDIT_GENERAL_STATUS, - thd->get_stmt_da()->is_error() ? - thd->get_stmt_da()->sql_errno() : 0, - command_name[COM_QUERY].str); + thd->get_stmt_da()->is_error() ? + thd->get_stmt_da()->sql_errno() : 0, + command_name[COM_QUERY].str); - if (!res && unlikely(thd->enable_slow_log)) + if (log_slow) log_slow_statement(thd); } else diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index be1425e294a..c7e47c84db0 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -8935,13 +8935,13 @@ static int handle_grant_struct(enum enum_acl_lists struct_no, bool drop, case USER_ACL: acl_user->user.str= strdup_root(&acl_memroot, user_to->user.str); acl_user->user.length= user_to->user.length; - acl_user->host.hostname= strdup_root(&acl_memroot, user_to->host.str); - acl_user->hostname_length= user_to->host.length; + update_hostname(&acl_user->host, strdup_root(&acl_memroot, user_to->host.str)); + acl_user->hostname_length= strlen(acl_user->host.hostname); break; case DB_ACL: acl_db->user= strdup_root(&acl_memroot, user_to->user.str); - acl_db->host.hostname= strdup_root(&acl_memroot, user_to->host.str); + update_hostname(&acl_db->host, strdup_root(&acl_memroot, user_to->host.str)); break; case COLUMN_PRIVILEGES_HASH: @@ -10639,12 +10639,6 @@ void fill_effective_table_privileges(THD *thd, GRANT_INFO *grant, /* global privileges */ grant->privilege= sctx->master_access; - if (!sctx->priv_user) - { - DBUG_PRINT("info", ("privilege 0x%lx", grant->privilege)); - DBUG_VOID_RETURN; // it is slave - } - if (!thd->db || strcmp(db, thd->db)) { /* db privileges */ diff --git a/sql/sql_alter.h b/sql/sql_alter.h index 526442e83e2..7114694124b 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -70,59 +70,56 @@ public: // Set for DISABLE KEYS | ENABLE KEYS static const uint ALTER_KEYS_ONOFF = 1L << 9; - // Set for CONVERT TO CHARACTER SET - static const uint ALTER_CONVERT = 1L << 10; - // Set for FORCE // Set for ENGINE(same engine) // Set by mysql_recreate_table() - static const uint ALTER_RECREATE = 1L << 11; + static const uint ALTER_RECREATE = 1L << 10; // Set for ADD PARTITION - static const uint ALTER_ADD_PARTITION = 1L << 12; + static const uint ALTER_ADD_PARTITION = 1L << 11; // Set for DROP PARTITION - static const uint ALTER_DROP_PARTITION = 1L << 13; + static const uint ALTER_DROP_PARTITION = 1L << 12; // Set for COALESCE PARTITION - static const uint ALTER_COALESCE_PARTITION = 1L << 14; + static const uint ALTER_COALESCE_PARTITION = 1L << 13; // Set for REORGANIZE PARTITION ... INTO - static const uint ALTER_REORGANIZE_PARTITION = 1L << 15; + static const uint ALTER_REORGANIZE_PARTITION = 1L << 14; // Set for partition_options - static const uint ALTER_PARTITION = 1L << 16; + static const uint ALTER_PARTITION = 1L << 15; // Set for LOAD INDEX INTO CACHE ... PARTITION // Set for CACHE INDEX ... PARTITION - static const uint ALTER_ADMIN_PARTITION = 1L << 17; + static const uint ALTER_ADMIN_PARTITION = 1L << 16; // Set for REORGANIZE PARTITION - static const uint ALTER_TABLE_REORG = 1L << 18; + static const uint ALTER_TABLE_REORG = 1L << 17; // Set for REBUILD PARTITION - static const uint ALTER_REBUILD_PARTITION = 1L << 19; + static const uint ALTER_REBUILD_PARTITION = 1L << 18; // Set for partitioning operations specifying ALL keyword - static const uint ALTER_ALL_PARTITION = 1L << 20; + static const uint ALTER_ALL_PARTITION = 1L << 19; // Set for REMOVE PARTITIONING - static const uint ALTER_REMOVE_PARTITIONING = 1L << 21; + static const uint ALTER_REMOVE_PARTITIONING = 1L << 20; // Set for ADD FOREIGN KEY - static const uint ADD_FOREIGN_KEY = 1L << 22; + static const uint ADD_FOREIGN_KEY = 1L << 21; // Set for DROP FOREIGN KEY - static const uint DROP_FOREIGN_KEY = 1L << 23; + static const uint DROP_FOREIGN_KEY = 1L << 22; // Set for EXCHANGE PARITION - static const uint ALTER_EXCHANGE_PARTITION = 1L << 24; + static const uint ALTER_EXCHANGE_PARTITION = 1L << 23; // Set by Sql_cmd_alter_table_truncate_partition::execute() - static const uint ALTER_TRUNCATE_PARTITION = 1L << 25; + static const uint ALTER_TRUNCATE_PARTITION = 1L << 24; // Set for ADD [COLUMN] FIRST | AFTER - static const uint ALTER_COLUMN_ORDER = 1L << 26; + static const uint ALTER_COLUMN_ORDER = 1L << 25; enum enum_enable_or_disable { LEAVE_AS_IS, ENABLE, DISABLE }; diff --git a/sql/sql_audit.cc b/sql/sql_audit.cc index b659054a50b..60a75cb06e7 100644 --- a/sql/sql_audit.cc +++ b/sql/sql_audit.cc @@ -369,8 +369,7 @@ int initialize_audit_plugin(st_plugin_int *plugin) { st_mysql_audit *data= (st_mysql_audit*) plugin->plugin->info; - if (!data->class_mask || !data->event_notify || - !data->class_mask[0]) + if (!data->event_notify || !data->class_mask[0]) { sql_print_error("Plugin '%s' has invalid data.", plugin->name.str); diff --git a/sql/sql_base.cc b/sql/sql_base.cc index b9cc4e5d69a..73f71a07f15 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -6628,7 +6628,7 @@ find_field_in_tables(THD *thd, Item_ident *item, if (!table_ref->belong_to_view && !table_ref->belong_to_derived) { - SELECT_LEX *current_sel= thd->lex->current_select; + SELECT_LEX *current_sel= item->context->select_lex; SELECT_LEX *last_select= table_ref->select_lex; bool all_merged= TRUE; for (SELECT_LEX *sl= current_sel; sl && sl!=last_select; @@ -8714,9 +8714,7 @@ fill_record(THD * thd, TABLE *table_arg, List<Item> &fields, List<Item> &values, /* Update virtual fields*/ thd->abort_on_warning= FALSE; if (vcol_table && vcol_table->vfield && - update_virtual_fields(thd, vcol_table, - vcol_table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE)) + update_virtual_fields(thd, vcol_table, VCOL_UPDATE_FOR_WRITE)) goto err; thd->abort_on_warning= save_abort_on_warning; thd->no_errors= save_no_errors; @@ -8777,9 +8775,7 @@ fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, List<Item> &fields, if (item_field && item_field->field && table && table->vfield) { DBUG_ASSERT(table == item_field->field->table); - result= update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE); + result= update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE); } } } @@ -8864,9 +8860,7 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values, /* Update virtual fields*/ thd->abort_on_warning= FALSE; if (table->vfield && - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE)) + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE)) goto err; thd->abort_on_warning= abort_on_warning_saved; DBUG_RETURN(thd->is_error()); @@ -8917,9 +8911,7 @@ fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, Field **ptr, { DBUG_ASSERT(table == (*ptr)->table); if (table->vfield) - result= update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE); + result= update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE); } return result; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index eff70eb27ba..666cdb3afc3 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2008, 2016, MariaDB + Copyright (c) 2008, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1054,7 +1054,6 @@ THD::THD() m_internal_handler= NULL; m_binlog_invoker= INVOKER_NONE; - arena_for_cached_items= 0; memset(&invoker_user, 0, sizeof(invoker_user)); memset(&invoker_host, 0, sizeof(invoker_host)); prepare_derived_at_open= FALSE; @@ -6517,7 +6516,13 @@ wait_for_commit::reinit() So in this case, do a re-init of the mutex. In release builds, we want to avoid the overhead of a re-init though. + + To ensure that no one is locking the mutex, we take a lock of it first. + For full explanation, see wait_for_commit::~wait_for_commit() */ + mysql_mutex_lock(&LOCK_wait_commit); + mysql_mutex_unlock(&LOCK_wait_commit); + mysql_mutex_destroy(&LOCK_wait_commit); mysql_mutex_init(key_LOCK_wait_commit, &LOCK_wait_commit, MY_MUTEX_INIT_FAST); #endif diff --git a/sql/sql_class.h b/sql/sql_class.h index f52a2cd5999..6d2c9ef89fd 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3676,26 +3676,7 @@ public: } } -private: - /* - This reference points to the table arena when the expression - for a virtual column is being evaluated - */ - Query_arena *arena_for_cached_items; - public: - void reset_arena_for_cached_items(Query_arena *new_arena) - { - arena_for_cached_items= new_arena; - } - Query_arena *switch_to_arena_for_cached_items(Query_arena *backup) - { - if (!arena_for_cached_items) - return 0; - set_n_backup_active_arena(arena_for_cached_items, backup); - return backup; - } - void clear_wakeup_ready() { wakeup_ready= false; } /* Sleep waiting for others to wake us up with signal_wakeup_ready(). diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc index c5a0d1464e7..63f9bfae47a 100644 --- a/sql/sql_connect.cc +++ b/sql/sql_connect.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2007, 2013, Oracle and/or its affiliates. - Copyright (c) 2008, 2014, SkySQL Ab. + Copyright (c) 2008, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -831,6 +831,7 @@ void update_global_user_stats(THD *thd, bool create_user, time_t now) bool thd_init_client_charset(THD *thd, uint cs_number) { + SV *gv=&global_system_variables; CHARSET_INFO *cs; /* Use server character set and collation if @@ -841,12 +842,10 @@ bool thd_init_client_charset(THD *thd, uint cs_number) if (!opt_character_set_client_handshake || !(cs= get_charset(cs_number, MYF(0)))) { - thd->variables.character_set_client= - global_system_variables.character_set_client; - thd->variables.collation_connection= - global_system_variables.collation_connection; - thd->variables.character_set_results= - global_system_variables.character_set_results; + DBUG_ASSERT(is_supported_parser_charset(gv->character_set_client)); + thd->variables.character_set_client= gv->character_set_client; + thd->variables.collation_connection= gv->collation_connection; + thd->variables.character_set_results= gv->character_set_results; } else { diff --git a/sql/sql_const.h b/sql/sql_const.h index 5c3a6d9a7cf..da2eef1b441 100644 --- a/sql/sql_const.h +++ b/sql/sql_const.h @@ -69,6 +69,7 @@ #define RAND_TABLE_BIT (((table_map) 1) << (sizeof(table_map)*8-1)) #define PSEUDO_TABLE_BITS (PARAM_TABLE_BIT | OUTER_REF_TABLE_BIT | \ RAND_TABLE_BIT) +#define CONNECT_STRING_MAXLEN 65535 /* stored in 2 bytes in .frm */ #define MAX_FIELDS 4096 /* Limit in the .frm file */ #define MAX_PARTITIONS 8192 diff --git a/sql/sql_db.cc b/sql/sql_db.cc index 0a3ff64113f..6c8c384e544 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -815,7 +815,8 @@ bool mysql_rm_db(THD *thd,char *db,bool if_exists, bool silent) if there exists a table with the name 'db', so let's just do it separately. We know this file exists and needs to be deleted anyway. */ - if (my_delete_with_symlink(path, MYF(0)) && my_errno != ENOENT) + if (mysql_file_delete_with_symlink(key_file_misc, path, "", MYF(0)) && + my_errno != ENOENT) { my_error(EE_DELETE, MYF(0), path, my_errno); DBUG_RETURN(true); @@ -1116,9 +1117,9 @@ static bool find_db_tables_and_rm_known_files(THD *thd, MY_DIR *dirp, strxmov(filePath, path, "/", file->name, NullS); /* We ignore ENOENT error in order to skip files that was deleted - by concurrently running statement like REAPIR TABLE ... + by concurrently running statement like REPAIR TABLE ... */ - if (my_delete_with_symlink(filePath, MYF(0)) && + if (mysql_file_delete_with_symlink(key_file_misc, filePath, "", MYF(0)) && my_errno != ENOENT) { my_error(EE_DELETE, MYF(0), filePath, my_errno); @@ -1234,7 +1235,7 @@ long mysql_rm_arc_files(THD *thd, MY_DIR *dirp, const char *org_path) continue; } strxmov(filePath, org_path, "/", file->name, NullS); - if (mysql_file_delete_with_symlink(key_file_misc, filePath, MYF(MY_WME))) + if (mysql_file_delete_with_symlink(key_file_misc, filePath, "", MYF(MY_WME))) { goto err; } diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index ce7da3cd33a..6b0135d92d9 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -548,9 +548,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, ! thd->is_error()) { if (table->vfield) - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_READ); + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_READ); thd->inc_examined_row_count(1); // thd->is_error() is tested to disallow delete row on error if (!select || select->skip_record(thd) > 0) @@ -1297,4 +1295,3 @@ bool multi_delete::send_eof() } return 0; } - diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index f2674cb8dab..af5b016df9d 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -447,6 +447,9 @@ bool mysql_derived_merge(THD *thd, LEX *lex, TABLE_LIST *derived) { Item *expr= derived->on_expr; expr= and_conds(expr, dt_select->join ? dt_select->join->conds : 0); + if (expr) + expr->top_level_item(); + if (expr && (derived->prep_on_expr || expr != derived->on_expr)) { derived->on_expr= expr; diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc index f701c424f63..7584b42c904 100644 --- a/sql/sql_join_cache.cc +++ b/sql/sql_join_cache.cc @@ -589,6 +589,11 @@ void JOIN_CACHE::create_remaining_fields() { MY_BITMAP *rem_field_set; TABLE *table= tab->table; +#if MYSQL_VERSION_ID < 100204 + empty_record(table); +#else +#error remove +#endif if (all_read_fields) rem_field_set= table->read_set; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 9d5f4cfcb5b..de450b8ede8 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -1943,6 +1943,7 @@ void st_select_lex::init_select() m_agg_func_used= false; name_visibility_map= 0; join= 0; + lock_type= TL_READ_DEFAULT; } /* @@ -3157,6 +3158,9 @@ void LEX::first_lists_tables_same() if (query_tables_last == &first_table->next_global) query_tables_last= first_table->prev_global; + if (query_tables_own_last == &first_table->next_global) + query_tables_own_last= first_table->prev_global; + if ((next= *first_table->prev_global= first_table->next_global)) next->prev_global= first_table->prev_global; /* include in new place */ @@ -4270,6 +4274,12 @@ bool st_select_lex::is_merged_child_of(st_select_lex *ancestor) { continue; } + + if (sl->master_unit()->derived && + sl->master_unit()->derived->is_merged_derived()) + { + continue; + } all_merged= FALSE; break; } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 019231e2004..c4fd109009f 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -846,6 +846,9 @@ public: /* namp of nesting SELECT visibility (for aggregate functions check) */ nesting_map name_visibility_map; + /* it is for correct printing SELECT options */ + thr_lock_type lock_type; + void init_query(); void init_select(); st_select_lex_unit* master_unit(); diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 51a284964e1..c391cfba00c 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -1060,7 +1060,7 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, { uint length; uchar *pos; - Item *real_item; + Item_field *real_item; if (read_info.read_field()) break; @@ -1072,16 +1072,26 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, pos=read_info.row_start; length=(uint) (read_info.row_end-pos); - real_item= item->real_item(); + real_item= item->field_for_view_update(); if ((!read_info.enclosed && (enclosed_length && length == 4 && !memcmp(pos, STRING_WITH_LEN("NULL")))) || (length == 1 && read_info.found_null)) { - if (real_item->type() == Item::FIELD_ITEM) + if (item->type() == Item::STRING_ITEM) { - Field *field= ((Item_field *)real_item)->field; + ((Item_user_var_as_out_param *)item)->set_null_value( + read_info.read_charset); + } + else if (!real_item) + { + my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); + DBUG_RETURN(1); + } + else + { + Field *field= real_item->field; if (field->reset()) { my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field->field_name, @@ -1104,23 +1114,23 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, /* Do not auto-update this field. */ field->set_has_explicit_value(); } - else if (item->type() == Item::STRING_ITEM) - { - ((Item_user_var_as_out_param *)item)->set_null_value( - read_info.read_charset); - } - else - { - my_error(ER_LOAD_DATA_INVALID_COLUMN, MYF(0), item->full_name()); - DBUG_RETURN(1); - } continue; } - if (real_item->type() == Item::FIELD_ITEM) + if (item->type() == Item::STRING_ITEM) + { + ((Item_user_var_as_out_param *)item)->set_value((char*) pos, length, + read_info.read_charset); + } + else if (!real_item) { - Field *field= ((Item_field *)real_item)->field; + my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); + DBUG_RETURN(1); + } + else + { + Field *field= real_item->field; field->set_notnull(); read_info.row_end[0]=0; // Safe to change end marker if (field == table->next_number_field) @@ -1128,16 +1138,6 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, field->store((char*) pos, length, read_info.read_charset); field->set_has_explicit_value(); } - else if (item->type() == Item::STRING_ITEM) - { - ((Item_user_var_as_out_param *)item)->set_value((char*) pos, length, - read_info.read_charset); - } - else - { - my_error(ER_LOAD_DATA_INVALID_COLUMN, MYF(0), item->full_name()); - DBUG_RETURN(1); - } } if (thd->is_error()) @@ -1157,10 +1157,20 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, break; for (; item ; item= it++) { - Item *real_item= item->real_item(); - if (real_item->type() == Item::FIELD_ITEM) + Item_field *real_item= item->field_for_view_update(); + if (item->type() == Item::STRING_ITEM) { - Field *field= ((Item_field *)real_item)->field; + ((Item_user_var_as_out_param *)item)->set_null_value( + read_info.read_charset); + } + else if (!real_item) + { + my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name); + DBUG_RETURN(1); + } + else + { + Field *field= real_item->field; if (field->reset()) { my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0),field->field_name, @@ -1182,16 +1192,6 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list, ER(ER_WARN_TOO_FEW_RECORDS), thd->get_stmt_da()->current_row_for_warning()); } - else if (item->type() == Item::STRING_ITEM) - { - ((Item_user_var_as_out_param *)item)->set_null_value( - read_info.read_charset); - } - else - { - my_error(ER_LOAD_DATA_INVALID_COLUMN, MYF(0), item->full_name()); - DBUG_RETURN(1); - } } } @@ -1482,7 +1482,7 @@ READ_INFO::READ_INFO(File file_par, uint tot_length, CHARSET_INFO *cs, stack=stack_pos=(int*) sql_alloc(sizeof(int)*length); if (!(buffer=(uchar*) my_malloc(buff_length+1,MYF(MY_WME | MY_THREAD_SPECIFIC)))) - error=1; /* purecov: inspected */ + error= 1; /* purecov: inspected */ else { end_of_buff=buffer+buff_length; @@ -1748,7 +1748,7 @@ int READ_INFO::read_field() ** We come here if buffer is too small. Enlarge it and continue */ if (!(new_buffer=(uchar*) my_realloc((char*) buffer,buff_length+1+IO_SIZE, - MYF(MY_WME | MY_THREAD_SPECIFIC)))) + MYF(MY_WME | MY_THREAD_SPECIFIC)))) return (error=1); to=new_buffer + (to-buffer); buffer=new_buffer; @@ -1984,15 +1984,7 @@ int READ_INFO::read_value(int delim, String *val) for (chr= GET; my_tospace(chr) != delim && chr != my_b_EOF;) { #ifdef USE_MB - uint ml= my_mbcharlen(read_charset, chr); - if (ml == 0) - { - chr= my_b_EOF; - val->length(0); - return chr; - } - - if (ml > 1) + if (my_mbcharlen(read_charset, chr) > 1) { DBUG_PRINT("read_xml",("multi byte")); int i, ml= my_mbcharlen(read_charset, chr); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 70511fcd849..e57756217ca 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1,5 +1,5 @@ -/* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2008, 2015, MariaDB +/* Copyright (c) 2000, 2017, Oracle and/or its affiliates. + Copyright (c) 2008, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1909,12 +1909,12 @@ int prepare_schema_table(THD *thd, LEX *lex, Table_ident *table_ident, #endif case SCH_COLUMNS: case SCH_STATISTICS: - { #ifdef DONT_ALLOW_SHOW_COMMANDS my_message(ER_NOT_ALLOWED_COMMAND, ER(ER_NOT_ALLOWED_COMMAND), MYF(0)); /* purecov: inspected */ DBUG_RETURN(1); #else + { DBUG_ASSERT(table_ident); TABLE_LIST **query_tables_last= lex->query_tables_last; schema_select_lex= new SELECT_LEX(); @@ -2207,7 +2207,7 @@ err: int mysql_execute_command(THD *thd) { - int res= FALSE; + int res= 0; int up_result= 0; LEX *lex= thd->lex; /* first SELECT_LEX (have special meaning for many of non-SELECTcommands) */ @@ -2702,10 +2702,17 @@ case SQLCOM_PREPARE: if (check_global_access(thd, SUPER_ACL)) goto error; + /* + In this code it's ok to use LOCK_active_mi as we are adding new things + into master_info_index + */ mysql_mutex_lock(&LOCK_active_mi); - if (!master_info_index) + { + mysql_mutex_unlock(&LOCK_active_mi); + my_error(ER_SERVER_SHUTDOWN, MYF(0)); goto error; + } mi= master_info_index->get_master_info(&lex_mi->connection_name, Sql_condition::WARN_LEVEL_NOTE); @@ -2734,7 +2741,7 @@ case SQLCOM_PREPARE: If new master was not added, we still need to free mi. */ if (master_info_added) - master_info_index->remove_master_info(&lex_mi->connection_name); + master_info_index->remove_master_info(mi); else delete mi; } @@ -2752,22 +2759,24 @@ case SQLCOM_PREPARE: /* Accept one of two privileges */ if (check_global_access(thd, SUPER_ACL | REPL_CLIENT_ACL)) goto error; - mysql_mutex_lock(&LOCK_active_mi); if (lex->verbose) + { + mysql_mutex_lock(&LOCK_active_mi); res= show_all_master_info(thd); + mysql_mutex_unlock(&LOCK_active_mi); + } else { LEX_MASTER_INFO *lex_mi= &thd->lex->mi; Master_info *mi; - mi= master_info_index->get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR); - if (mi != NULL) + if ((mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) { res= show_master_info(thd, mi, 0); + mi->release(); } } - mysql_mutex_unlock(&LOCK_active_mi); break; } case SQLCOM_SHOW_MASTER_STAT: @@ -3091,22 +3100,23 @@ end_with_restore_list: load_error= rpl_load_gtid_slave_state(thd); - mysql_mutex_lock(&LOCK_active_mi); - - if ((mi= (master_info_index-> - get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) + /* + We don't need to ensure that only one user is using master_info + as start_slave is protected against simultaneous usage + */ + if ((mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) { if (load_error) { /* - We cannot start a slave using GTID if we cannot load the GTID position - from the mysql.gtid_slave_pos table. But we can allow non-GTID - replication (useful eg. during upgrade). + We cannot start a slave using GTID if we cannot load the + GTID position from the mysql.gtid_slave_pos table. But we + can allow non-GTID replication (useful eg. during upgrade). */ if (mi->using_gtid != Master_info::USE_GTID_NO) { - mysql_mutex_unlock(&LOCK_active_mi); + mi->release(); break; } else @@ -3114,8 +3124,8 @@ end_with_restore_list: } if (!start_slave(thd, mi, 1 /* net report*/)) my_ok(thd); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); break; } case SQLCOM_SLAVE_STOP: @@ -3145,13 +3155,17 @@ end_with_restore_list: } lex_mi= &thd->lex->mi; - mysql_mutex_lock(&LOCK_active_mi); - if ((mi= (master_info_index-> - get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) - if (!stop_slave(thd, mi, 1/* net report*/)) + if ((mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) + { + if (stop_slave(thd, mi, 1/* net report*/)) + res= 1; + mi->release(); + if (rpl_parallel_resize_pool_if_no_slaves()) + res= 1; + if (!res) my_ok(thd); - mysql_mutex_unlock(&LOCK_active_mi); + } break; } case SQLCOM_SLAVE_ALL_START: @@ -4317,11 +4331,13 @@ end_with_restore_list: reload_acl_and_cache binlog interactions failed */ res= 1; - } + } if (!res) my_ok(thd); } + else + res= 1; // reload_acl_and_cache failed #ifdef HAVE_REPLICATION if (lex->type & REFRESH_READ_LOCK) rpl_unpause_after_ftwrl(thd); @@ -4921,11 +4937,8 @@ create_sp_error: } case SQLCOM_SHOW_CREATE_TRIGGER: { - if (lex->spname->m_name.length > NAME_LEN) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), lex->spname->m_name.str); + if (check_ident_length(&lex->spname->m_name)) goto error; - } if (show_create_trigger(thd, lex->spname)) goto error; /* Error has been already logged. */ @@ -6150,12 +6163,6 @@ bool check_fk_parent_table_access(THD *thd, ****************************************************************************/ -#if STACK_DIRECTION < 0 -#define used_stack(A,B) (long) (A - B) -#else -#define used_stack(A,B) (long) (B - A) -#endif - #ifndef DBUG_OFF long max_stack_used; #endif @@ -6172,7 +6179,7 @@ bool check_stack_overrun(THD *thd, long margin, { long stack_used; DBUG_ASSERT(thd == current_thd); - if ((stack_used=used_stack(thd->thread_stack,(char*) &stack_used)) >= + if ((stack_used= available_stack_size(thd->thread_stack, &stack_used)) >= (long) (my_thread_stack_size - margin)) { thd->is_fatal_error= 1; @@ -6474,7 +6481,7 @@ void mysql_init_multi_delete(LEX *lex) /* - When you modify mysql_parse(), you may need to mofify + When you modify mysql_parse(), you may need to modify mysql_test_parse_for_slave() in this same file. */ @@ -6663,12 +6670,9 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, uint8 datetime_precision= length ? atoi(length) : 0; DBUG_ENTER("add_field_to_list"); - if (check_string_char_length(field_name, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), field_name->str); /* purecov: inspected */ + if (check_ident_length(field_name)) DBUG_RETURN(1); /* purecov: inspected */ - } + if (type_modifier & PRI_KEY_FLAG) { Key *key; @@ -8443,48 +8447,36 @@ bool check_string_char_length(LEX_STRING *str, const char *err_msg, return TRUE; } -C_MODE_START + +bool check_ident_length(LEX_STRING *ident) +{ + if (check_string_char_length(ident, 0, NAME_CHAR_LEN, system_charset_info, 1)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + return 1; + } + return 0; +} + /* Check if path does not contain mysql data home directory SYNOPSIS - test_if_data_home_dir() - dir directory + path_starts_from_data_home_dir() + dir directory, with all symlinks resolved RETURN VALUES 0 ok 1 error ; Given path contains data directory */ +extern "C" { -int test_if_data_home_dir(const char *dir) +int path_starts_from_data_home_dir(const char *path) { - char path[FN_REFLEN]; - int dir_len; - DBUG_ENTER("test_if_data_home_dir"); + int dir_len= strlen(path); + DBUG_ENTER("path_starts_from_data_home_dir"); - if (!dir) - DBUG_RETURN(0); - - /* - data_file_name and index_file_name include the table name without - extension. Mostly this does not refer to an existing file. When - comparing data_file_name or index_file_name against the data - directory, we try to resolve all symbolic links. On some systems, - we use realpath(3) for the resolution. This returns ENOENT if the - resolved path does not refer to an existing file. my_realpath() - does then copy the requested path verbatim, without symlink - resolution. Thereafter the comparison can fail even if the - requested path is within the data directory. E.g. if symlinks to - another file system are used. To make realpath(3) return the - resolved path, we strip the table name and compare the directory - path only. If the directory doesn't exist either, table creation - will fail anyway. - */ - - (void) fn_format(path, dir, "", "", - (MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS)); - dir_len= strlen(path); if (mysql_unpacked_real_data_home_len<= dir_len) { if (dir_len > mysql_unpacked_real_data_home_len && @@ -8512,7 +8504,31 @@ int test_if_data_home_dir(const char *dir) DBUG_RETURN(0); } -C_MODE_END +} + +/* + Check if path does not contain mysql data home directory + + SYNOPSIS + test_if_data_home_dir() + dir directory + + RETURN VALUES + 0 ok + 1 error ; Given path contains data directory +*/ + +int test_if_data_home_dir(const char *dir) +{ + char path[FN_REFLEN]; + DBUG_ENTER("test_if_data_home_dir"); + + if (!dir) + DBUG_RETURN(0); + + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + DBUG_RETURN(path_starts_from_data_home_dir(path)); +} int error_if_data_home_dir(const char *path, const char *what) diff --git a/sql/sql_parse.h b/sql/sql_parse.h index 7a55d8ac4a4..fa414911093 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -33,7 +33,8 @@ enum enum_mysql_completiontype { COMMIT_RELEASE=-1, COMMIT=0, COMMIT_AND_CHAIN=6 }; -extern "C" int test_if_data_home_dir(const char *dir); +extern "C" int path_starts_from_data_home_dir(const char *dir); +int test_if_data_home_dir(const char *dir); int error_if_data_home_dir(const char *path, const char *what); bool multi_update_precheck(THD *thd, TABLE_LIST *tables); @@ -76,6 +77,7 @@ bool check_string_byte_length(LEX_STRING *str, const char *err_msg, bool check_string_char_length(LEX_STRING *str, const char *err_msg, uint max_char_length, CHARSET_INFO *cs, bool no_error); +bool check_ident_length(LEX_STRING *ident); CHARSET_INFO* merge_charset_and_collation(CHARSET_INFO *cs, CHARSET_INFO *cl); bool check_host_name(LEX_STRING *str); bool check_identifier_name(LEX_STRING *str, uint max_char_length, diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 5fe30f21b32..03624647970 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -2010,7 +2010,7 @@ static int mysql_test_handler_read(Prepared_statement *stmt, THD *thd= stmt->thd; LEX *lex= stmt->lex; SQL_HANDLER *ha_table; - DBUG_ENTER("mysql_test_select"); + DBUG_ENTER("mysql_test_handler_read"); lex->select_lex.context.resolve_in_select_list= TRUE; diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc index bcf2585ca2a..f9c5757f721 100644 --- a/sql/sql_reload.cc +++ b/sql/sql_reload.cc @@ -174,24 +174,20 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, slave is not likely to have the same connection names. */ tmp_write_to_binlog= 0; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if (!(mi= (get_master_info(&connection_name, + Sql_condition::WARN_LEVEL_ERROR)))) { - if (!(mi= (master_info_index-> - get_master_info(&connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) - { - result= 1; - } - else - { - mysql_mutex_lock(&mi->data_lock); - if (rotate_relay_log(mi)) - *write_to_binlog= -1; - mysql_mutex_unlock(&mi->data_lock); - } + result= 1; + } + else + { + mysql_mutex_lock(&mi->data_lock); + if (rotate_relay_log(mi)) + *write_to_binlog= -1; + mysql_mutex_unlock(&mi->data_lock); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); #endif } #ifdef HAVE_QUERY_CACHE @@ -349,27 +345,33 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, LEX_MASTER_INFO* lex_mi= &thd->lex->mi; Master_info *mi; tmp_write_to_binlog= 0; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if (!(mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) { - if (!(mi= (master_info_index-> - get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) - { - result= 1; - } - else if (reset_slave(thd, mi)) + result= 1; + } + else + { + /* The following will fail if slave is running */ + if (reset_slave(thd, mi)) { + mi->release(); /* NOTE: my_error() has been already called by reset_slave(). */ result= 1; } else if (mi->connection_name.length && thd->lex->reset_slave_info.all) { /* If not default connection and 'all' is used */ - master_info_index->remove_master_info(&mi->connection_name); + mi->release(); + mysql_mutex_lock(&LOCK_active_mi); + if (master_info_index->remove_master_info(mi)) + result= 1; + mysql_mutex_unlock(&LOCK_active_mi); } + else + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); } #endif if (options & REFRESH_USER_RESOURCES) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 4505ad69880..560b7ede183 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2833,7 +2833,16 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) relay_log_info_file, 0, &mi->cmp_connection_name); - lock_slave_threads(mi); // this allows us to cleanly read slave_running + mi->lock_slave_threads(); + if (mi->killed) + { + /* connection was deleted while we waited for lock_slave_threads */ + mi->unlock_slave_threads(); + my_error(WARN_NO_MASTER_INFO, mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(-1); + } + // Get a mask of _stopped_ threads init_thread_mask(&thread_mask,mi,1 /* inverse */); @@ -2968,7 +2977,7 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) ER(ER_UNTIL_COND_IGNORED)); if (!slave_errno) - slave_errno = start_slave_threads(0 /*no mutex */, + slave_errno = start_slave_threads(1, 1 /* wait for start */, mi, master_info_file_tmp, @@ -2984,7 +2993,7 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) } err: - unlock_slave_threads(mi); + mi->unlock_slave_threads(); if (slave_errno) { @@ -3024,8 +3033,12 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) DBUG_RETURN(-1); THD_STAGE_INFO(thd, stage_killing_slave); int thread_mask; - lock_slave_threads(mi); - // Get a mask of _running_ threads + mi->lock_slave_threads(); + /* + Get a mask of _running_ threads. + We don't have to test for mi->killed as the thread_mask will take care + of checking if threads exists + */ init_thread_mask(&thread_mask,mi,0 /* not inverse*/); /* Below we will stop all running threads. @@ -3038,8 +3051,7 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) if (thread_mask) { - slave_errno= terminate_slave_threads(mi,thread_mask, - 1 /*skip lock */); + slave_errno= terminate_slave_threads(mi,thread_mask, 0 /* get lock */); } else { @@ -3048,7 +3060,8 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, ER_SLAVE_WAS_NOT_RUNNING, ER(ER_SLAVE_WAS_NOT_RUNNING)); } - unlock_slave_threads(mi); + + mi->unlock_slave_threads(); if (slave_errno) { @@ -3083,11 +3096,20 @@ int reset_slave(THD *thd, Master_info* mi) char relay_log_info_file_tmp[FN_REFLEN]; DBUG_ENTER("reset_slave"); - lock_slave_threads(mi); + mi->lock_slave_threads(); + if (mi->killed) + { + /* connection was deleted while we waited for lock_slave_threads */ + mi->unlock_slave_threads(); + my_error(WARN_NO_MASTER_INFO, mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(-1); + } + init_thread_mask(&thread_mask,mi,0 /* not inverse */); if (thread_mask) // We refuse if any slave thread is running { - unlock_slave_threads(mi); + mi->unlock_slave_threads(); my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, mi->connection_name.str); DBUG_RETURN(ER_SLAVE_MUST_STOP); @@ -3152,7 +3174,7 @@ int reset_slave(THD *thd, Master_info* mi) RUN_HOOK(binlog_relay_io, after_reset_slave, (thd, mi)); err: - unlock_slave_threads(mi); + mi->unlock_slave_threads(); if (error) my_error(sql_errno, MYF(0), errmsg); DBUG_RETURN(error); @@ -3265,8 +3287,8 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) LEX_MASTER_INFO* lex_mi= &thd->lex->mi; DBUG_ENTER("change_master"); - mysql_mutex_assert_owner(&LOCK_active_mi); DBUG_ASSERT(master_info_index); + mysql_mutex_assert_owner(&LOCK_active_mi); *master_info_added= false; /* @@ -3286,7 +3308,16 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) lex_mi->port)) DBUG_RETURN(TRUE); - lock_slave_threads(mi); + mi->lock_slave_threads(); + if (mi->killed) + { + /* connection was deleted while we waited for lock_slave_threads */ + mi->unlock_slave_threads(); + my_error(WARN_NO_MASTER_INFO, mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(TRUE); + } + init_thread_mask(&thread_mask,mi,0 /*not inverse*/); if (thread_mask) // We refuse if any slave thread is running { @@ -3588,12 +3619,13 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) in-memory value at restart (thus causing errors, as the old relay log does not exist anymore). */ - flush_relay_log_info(&mi->rli); + if (flush_relay_log_info(&mi->rli)) + ret= 1; mysql_cond_broadcast(&mi->data_cond); mysql_mutex_unlock(&mi->rli.data_lock); err: - unlock_slave_threads(mi); + mi->unlock_slave_threads(); if (ret == FALSE) my_ok(thd); DBUG_RETURN(ret); @@ -3646,7 +3678,6 @@ bool mysql_show_binlog_events(THD* thd) int old_max_allowed_packet= thd->variables.max_allowed_packet; Master_info *mi= 0; LOG_INFO linfo; - DBUG_ENTER("mysql_show_binlog_events"); Log_event::init_show_field_list(&field_list); @@ -3674,13 +3705,9 @@ bool mysql_show_binlog_events(THD* thd) } else /* showing relay log contents */ { - mysql_mutex_lock(&LOCK_active_mi); - if (!master_info_index || - !(mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR))) + if (!(mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR))) { - mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(TRUE); } binary_log= &(mi->rli.relay_log); @@ -3700,7 +3727,7 @@ bool mysql_show_binlog_events(THD* thd) if (mi) { /* We can unlock the mutex as we have a lock on the file */ - mysql_mutex_unlock(&LOCK_active_mi); + mi->release(); mi= 0; } @@ -3722,6 +3749,7 @@ bool mysql_show_binlog_events(THD* thd) goto err; } + /* These locks is here to enable syncronization with log_in_use() */ mysql_mutex_lock(&LOCK_thread_count); thd->current_linfo = &linfo; mysql_mutex_unlock(&LOCK_thread_count); @@ -3799,7 +3827,7 @@ bool mysql_show_binlog_events(THD* thd) mysql_mutex_unlock(log_lock); } else if (mi) - mysql_mutex_unlock(&LOCK_active_mi); + mi->release(); // Check that linfo is still on the function scope. DEBUG_SYNC(thd, "after_show_binlog_events"); @@ -3820,8 +3848,9 @@ err: else my_eof(thd); + /* These locks is here to enable syncronization with log_in_use() */ mysql_mutex_lock(&LOCK_thread_count); - thd->current_linfo = 0; + thd->current_linfo= 0; mysql_mutex_unlock(&LOCK_thread_count); thd->variables.max_allowed_packet= old_max_allowed_packet; DBUG_RETURN(ret); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index f345d3c9687..20456931136 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -777,10 +777,15 @@ JOIN::prepare(Item ***rref_pointer_array, if (mixed_implicit_grouping && tbl->table) tbl->table->maybe_null= 1; } + + uint real_og_num= og_num; + if (skip_order_by && + select_lex != select_lex->master_unit()->global_parameters) + real_og_num+= select_lex->order_list.elements; if ((wild_num && setup_wild(thd, tables_list, fields_list, &all_fields, wild_num)) || - select_lex->setup_ref_array(thd, og_num) || + select_lex->setup_ref_array(thd, real_og_num) || setup_fields(thd, (*rref_pointer_array), fields_list, MARK_COLUMNS_READ, &all_fields, 1) || setup_without_group(thd, (*rref_pointer_array), tables_list, @@ -3082,7 +3087,7 @@ void JOIN::exec_inner() *curr_fields_list), Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF); error= do_select(curr_join, curr_fields_list, NULL, procedure); - thd->limit_found_rows= curr_join->send_records; + thd->limit_found_rows= curr_join->send_records - curr_join->duplicate_rows; if (curr_join->order && curr_join->sortorder && curr_join->filesort_found_rows) { @@ -8438,8 +8443,6 @@ get_best_combination(JOIN *join) join->full_join=0; join->hash_join= FALSE; - used_tables= OUTER_REF_TABLE_BIT; // Outer row is already read - fix_semijoin_strategies_for_picked_join_order(join); JOIN_TAB_RANGE *root_range; @@ -8503,10 +8506,7 @@ get_best_combination(JOIN *join) j->bush_root_tab= sjm_nest_root; form=join->table[tablenr]=j->table; - used_tables|= form->map; form->reginfo.join_tab=j; - if (!*j->on_expr_ref) - form->reginfo.not_exists_optimize=0; // Only with LEFT JOIN DBUG_PRINT("info",("type: %d", j->type)); if (j->type == JT_CONST) goto loop_end; // Handled in make_join_stat.. @@ -8532,9 +8532,6 @@ get_best_combination(JOIN *join) join->best_positions[tablenr].loosescan_picker.loosescan_key); j->index= join->best_positions[tablenr].loosescan_picker.loosescan_key; }*/ - - if (keyuse && create_ref_for_key(join, j, keyuse, TRUE, used_tables)) - DBUG_RETURN(TRUE); // Something went wrong if ((j->type == JT_REF || j->type == JT_EQ_REF) && is_hash_join_key_no(j->ref.key)) @@ -8560,6 +8557,23 @@ get_best_combination(JOIN *join) } root_range->end= j; + used_tables= OUTER_REF_TABLE_BIT; // Outer row is already read + for (j=join_tab, tablenr=0 ; tablenr < table_count ; tablenr++,j++) + { + if (j->bush_children) + j= j->bush_children->start; + + used_tables|= j->table->map; + if (j->type != JT_CONST && j->type != JT_SYSTEM) + { + if ((keyuse= join->best_positions[tablenr].key) && + create_ref_for_key(join, j, keyuse, TRUE, used_tables)) + DBUG_RETURN(TRUE); // Something went wrong + } + if (j->last_leaf_in_bush) + j= j->bush_root_tab; + } + join->top_join_tab_count= join->join_tab_ranges.head()->end - join->join_tab_ranges.head()->start; /* @@ -9350,7 +9364,10 @@ make_outerjoin_info(JOIN *join) tab->cond_equal= tbl->cond_equal; if (embedding && !embedding->is_active_sjm()) tab->first_upper= embedding->nested_join->first_nested; - } + } + else if (!embedding) + tab->table->reginfo.not_exists_optimize= 0; + for ( ; embedding ; embedding= embedding->embedding) { if (embedding->is_active_sjm()) @@ -9360,7 +9377,10 @@ make_outerjoin_info(JOIN *join) } /* Ignore sj-nests: */ if (!(embedding->on_expr && embedding->outer_join)) + { + tab->table->reginfo.not_exists_optimize= 0; continue; + } NESTED_JOIN *nested_join= embedding->nested_join; if (!nested_join->counter) { @@ -9376,17 +9396,10 @@ make_outerjoin_info(JOIN *join) } if (!tab->first_inner) tab->first_inner= nested_join->first_nested; - if (tab->table->reginfo.not_exists_optimize) - tab->first_inner->table->reginfo.not_exists_optimize= 1; if (++nested_join->counter < nested_join->n_tables) break; /* Table tab is the last inner table for nested join. */ nested_join->first_nested->last_inner= tab; - if (tab->first_inner->table->reginfo.not_exists_optimize) - { - for (JOIN_TAB *join_tab= tab->first_inner; join_tab <= tab; join_tab++) - join_tab->table->reginfo.not_exists_optimize= 1; - } } } DBUG_RETURN(FALSE); @@ -9518,7 +9531,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) It solve problem with select like SELECT * FROM t1 WHERE rand() > 0.5 */ if (tab == join->join_tab + join->top_join_tab_count - 1) - current_map|= OUTER_REF_TABLE_BIT | RAND_TABLE_BIT; + current_map|= RAND_TABLE_BIT; used_tables|=current_map; if (tab->type == JT_REF && tab->quick && @@ -10147,7 +10160,7 @@ void JOIN::drop_unused_derived_keys() continue; if (!table->pos_in_table_list->is_materialized_derived()) continue; - if (table->max_keys > 1) + if (table->max_keys > 1 && !tab->is_ref_for_hash_join()) table->use_index(tab->ref.key); if (table->s->keys) { @@ -12765,6 +12778,9 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, COND_EQUAL cond_equal; cond_equal.upper_levels= inherited; + if (check_stack_overrun(thd, STACK_MIN_SIZE, NULL)) + return cond; // Fatal error flag is set! + if (cond->type() == Item::COND_ITEM) { List<Item> eq_list; @@ -15502,7 +15518,9 @@ static Field *create_tmp_field_from_item(THD *thd, Item *item, TABLE *table, if (new_field) new_field->init(table); - if (copy_func && item->real_item()->is_result_field()) + if (copy_func && + (item->is_result_field() || + (item->real_item()->is_result_field()))) *((*copy_func)++) = item; // Save for copy_funcs if (modify_item) item->set_result_field(new_field); @@ -17453,7 +17471,7 @@ do_select(JOIN *join,List<Item> *fields,TABLE *table,Procedure *procedure) join->join_tab[join->top_join_tab_count - 1].next_select= end_select; join_tab=join->join_tab+join->const_tables; } - join->send_records=0; + join->duplicate_rows= join->send_records=0; if (join->table_count == join->const_tables) { /* @@ -17974,32 +17992,41 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, first_unmatched->found= 1; for (JOIN_TAB *tab= first_unmatched; tab <= join_tab; tab++) { + /* + Check whether 'not exists' optimization can be used here. + If tab->table->reginfo.not_exists_optimize is set to true + then WHERE contains a conjunctive predicate IS NULL over + a non-nullable field of tab. When activated this predicate + will filter out all records with matches for the left part + of the outer join whose inner tables start from the + first_unmatched table and include table tab. To safely use + 'not exists' optimization we have to check that the + IS NULL predicate is really activated, i.e. all guards + that wrap it are in the 'open' state. + */ + bool not_exists_opt_is_applicable= + tab->table->reginfo.not_exists_optimize; + for (JOIN_TAB *first_upper= first_unmatched->first_upper; + not_exists_opt_is_applicable && first_upper; + first_upper= first_upper->first_upper) + { + if (!first_upper->found) + not_exists_opt_is_applicable= false; + } /* Check all predicates that has just been activated. */ /* Actually all predicates non-guarded by first_unmatched->found will be re-evaluated again. It could be fixed, but, probably, it's not worth doing now. */ - /* - not_exists_optimize has been created from a - select_cond containing 'is_null'. This 'is_null' - predicate is still present on any 'tab' with - 'not_exists_optimize'. Furthermore, the usual rules - for condition guards also applies for - 'not_exists_optimize' -> When 'is_null==false' we - know all cond. guards are open and we can apply - the 'not_exists_optimize'. - */ - DBUG_ASSERT(!(tab->table->reginfo.not_exists_optimize && - !tab->select_cond)); - if (tab->select_cond && !tab->select_cond->val_int()) { /* The condition attached to table tab is false */ - if (tab == join_tab) { found= 0; + if (not_exists_opt_is_applicable) + DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS); } else { @@ -18008,21 +18035,10 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, not to the last table of the current nest level. */ join->return_tab= tab; - } - - if (tab->table->reginfo.not_exists_optimize) - { - /* - When not_exists_optimize is set: No need to further - explore more rows of 'tab' for this partial result. - Any found 'tab' matches are known to evaluate to 'false'. - Returning .._NO_MORE_ROWS will skip rem. 'tab' rows. - */ - DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS); - } - else if (tab != join_tab) - { - DBUG_RETURN(NESTED_LOOP_OK); + if (not_exists_opt_is_applicable) + DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS); + else + DBUG_RETURN(NESTED_LOOP_OK); } } } @@ -18978,7 +18994,12 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), int error; /* result < 0 if row was not accepted and should not be counted */ if ((error= join->result->send_data(*join->fields))) - DBUG_RETURN(error < 0 ? NESTED_LOOP_OK : NESTED_LOOP_ERROR); + { + if (error > 0) + DBUG_RETURN(NESTED_LOOP_ERROR); + // error < 0 => duplicate row + join->duplicate_rows++; + } } ++join->send_records; @@ -19112,7 +19133,7 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), if (error < 0) { /* Duplicate row, don't count */ - join->send_records--; + join->duplicate_rows++; error= 0; } } @@ -24592,6 +24613,12 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) // limit print_limit(thd, str, query_type); + // lock type + if (lock_type == TL_READ_WITH_SHARED_LOCKS) + str->append(" lock in share mode"); + else if (lock_type == TL_WRITE) + str->append(" for update"); + // PROCEDURE unsupported here } diff --git a/sql/sql_select.h b/sql/sql_select.h index cbad287c5e5..d51bca785a5 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1015,7 +1015,8 @@ public: table_map outer_join; /* Bitmap of tables used in the select list items */ table_map select_list_used_tables; - ha_rows send_records,found_records,examined_rows,row_limit, select_limit; + ha_rows send_records, found_records, examined_rows, + row_limit, select_limit, duplicate_rows; /** Used to fetch no more than given amount of rows per one fetch operation of server side cursor. @@ -1281,7 +1282,7 @@ public: sort_and_group= 0; first_record= 0; do_send_rows= 1; - send_records= 0; + duplicate_rows= send_records= 0; found_records= 0; fetch_limit= HA_POS_ERROR; examined_rows= 0; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 62dff1b5928..503048f9f6a 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2497,7 +2497,19 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, int frm_delete_error, trigger_drop_error= 0; /* Delete the table definition file */ strmov(end,reg_ext); - frm_delete_error= mysql_file_delete(key_file_frm, path, MYF(MY_WME)); + if (table_type && table_type != view_pseudo_hton && + table_type->discover_table) + { + /* + Table type is using discovery and may not need a .frm file. + Delete it silently if it exists + */ + (void) mysql_file_delete(key_file_frm, path, MYF(0)); + frm_delete_error= 0; + } + else + frm_delete_error= mysql_file_delete(key_file_frm, path, + MYF(MY_WME)); if (frm_delete_error) frm_delete_error= my_errno; else @@ -2513,7 +2525,7 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, else if (frm_delete_error && if_exists) thd->clear_error(); } - non_tmp_error= error ? TRUE : non_tmp_error; + non_tmp_error|= MY_TEST(error); } if (error) { @@ -3240,6 +3252,21 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, bool tmp_table= create_table_mode == C_ALTER_TABLE; DBUG_ENTER("mysql_prepare_create_table"); + LEX_STRING* connect_string = &create_info->connect_string; + if (connect_string->length != 0 && + connect_string->length > CONNECT_STRING_MAXLEN && + (system_charset_info->cset->charpos(system_charset_info, + connect_string->str, + (connect_string->str + + connect_string->length), + CONNECT_STRING_MAXLEN) + < connect_string->length)) + { + my_error(ER_WRONG_STRING_LENGTH, MYF(0), + connect_string->str, "CONNECTION", CONNECT_STRING_MAXLEN); + DBUG_RETURN(TRUE); + } + select_field_pos= alter_info->create_list.elements - select_field_count; null_fields=blob_columns=0; create_info->varchar= 0; @@ -3504,7 +3531,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, sql_field->pack_length= dup_field->pack_length; sql_field->key_length= dup_field->key_length; sql_field->decimals= dup_field->decimals; - sql_field->create_length_to_internal_length(); sql_field->unireg_check= dup_field->unireg_check; /* We're making one field from two, the result field will have @@ -3514,6 +3540,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (!(sql_field->flags & NOT_NULL_FLAG)) null_fields--; sql_field->flags= dup_field->flags; + sql_field->create_length_to_internal_length(); sql_field->interval= dup_field->interval; sql_field->vcol_info= dup_field->vcol_info; sql_field->stored_in_db= dup_field->stored_in_db; @@ -3640,12 +3667,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, my_error(ER_TOO_MANY_KEY_PARTS,MYF(0),tmp); DBUG_RETURN(TRUE); } - if (check_string_char_length(&key->name, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), key->name.str); + if (check_ident_length(&key->name)) DBUG_RETURN(TRUE); - } key_iterator2.rewind (); if (key->type != Key::FOREIGN_KEY) { @@ -4141,7 +4164,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, Field::utype type= (Field::utype) MTYP_TYPENR(sql_field->unireg_check); if (thd->variables.sql_mode & MODE_NO_ZERO_DATE && - !sql_field->def && + !sql_field->def && !sql_field->vcol_info && is_timestamp_type(sql_field->sql_type) && (sql_field->flags & NOT_NULL_FLAG) && (type == Field::NONE || type == Field::TIMESTAMP_UN_FIELD)) @@ -5931,7 +5954,7 @@ drop_create_field: while ((f_key= fk_key_it++)) { if (my_strcasecmp(system_charset_info, f_key->foreign_id->str, - key->name.str) == 0) + keyname) == 0) goto remove_key; } } @@ -6421,6 +6444,9 @@ static bool fill_alter_inplace_info(THD *thd, new_key->user_defined_key_parts)) goto index_changed; + if (table_key->block_size != new_key->block_size) + goto index_changed; + if (engine_options_differ(table_key->option_struct, new_key->option_struct, table->file->ht->index_options)) goto index_changed; diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 198f414eb85..b17775abb7c 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -2224,6 +2224,9 @@ void Table_triggers_list::mark_fields_used(trg_event_type event) bitmap_set_bit(trigger_table->read_set, trg_field->field_idx); if (trg_field->get_settable_routine_parameter()) bitmap_set_bit(trigger_table->write_set, trg_field->field_idx); + if (trigger_table->field[trg_field->field_idx]->vcol_info) + trigger_table->mark_virtual_col(trigger_table-> + field[trg_field->field_idx]); } } } diff --git a/sql/sql_udf.cc b/sql/sql_udf.cc index 74d2f6bc252..456a6bd4058 100644 --- a/sql/sql_udf.cc +++ b/sql/sql_udf.cc @@ -453,12 +453,8 @@ int mysql_create_function(THD *thd,udf_func *udf) my_message(ER_UDF_NO_PATHS, ER(ER_UDF_NO_PATHS), MYF(0)); DBUG_RETURN(1); } - if (check_string_char_length(&udf->name, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), udf->name.str); + if (check_ident_length(&udf->name)) DBUG_RETURN(1); - } tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"), "func", TL_WRITE); diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 55dc26ef043..9ca96bd702e 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -629,9 +629,7 @@ int mysql_update(THD *thd, while (!(error=info.read_record(&info)) && !thd->killed) { if (table->vfield) - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_READ); + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_READ); thd->inc_examined_row_count(1); if (!select || (error= select->skip_record(thd)) > 0) { @@ -744,9 +742,7 @@ int mysql_update(THD *thd, while (!(error=info.read_record(&info)) && !thd->killed) { if (table->vfield) - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_READ); + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_READ); thd->inc_examined_row_count(1); if (!select || select->skip_record(thd) > 0) { @@ -2399,8 +2395,7 @@ int multi_update::do_updates() if (table->default_field && (error= table->update_default_fields())) goto err2; if (table->vfield && - update_virtual_fields(thd, table, - (table->triggers ? VCOL_UPDATE_ALL : VCOL_UPDATE_FOR_WRITE))) + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE)) goto err2; if ((error= cur_table->view_check_option(thd, ignore)) != VIEW_CHECK_OK) diff --git a/sql/sql_view.cc b/sql/sql_view.cc index f3717e3ded2..9fe4dd4849d 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -430,6 +430,15 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + /* + ignore lock specs for CREATE statement + */ + if (lex->current_select->lock_type != TL_READ_DEFAULT) + { + lex->current_select->set_lock_for_tables(TL_READ_DEFAULT); + view->mdl_request.set_type(MDL_EXCLUSIVE); + } + if (open_temporary_tables(thd, lex->query_tables) || open_and_lock_tables(thd, lex->query_tables, TRUE, 0)) { diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c9fd000141a..cd5b9657730 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -959,7 +959,7 @@ static bool sp_create_assignment_instr(THD *thd, bool no_lookahead) bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %} -%pure_parser /* We have threads */ +%pure-parser /* We have threads */ %parse-param { THD *thd } %lex-param { THD *thd } /* @@ -5117,6 +5117,8 @@ part_name: { partition_info *part_info= Lex->part_info; partition_element *p_elem= part_info->curr_part_elem; + if (check_ident_length(&$1)) + MYSQL_YYABORT; p_elem->partition_name= $1.str; } ; @@ -5411,7 +5413,11 @@ sub_part_definition: sub_name: ident_or_text - { Lex->part_info->curr_part_elem->partition_name= $1.str; } + { + if (check_ident_length(&$1)) + MYSQL_YYABORT; + Lex->part_info->curr_part_elem->partition_name= $1.str; + } ; opt_part_options: @@ -7637,7 +7643,7 @@ alter_list_item: LEX *lex= Lex; if (lex->create_info.add_alter_list_item_convert_to_charset($5)) MYSQL_YYABORT; - lex->alter_info.flags|= Alter_info::ALTER_CONVERT; + lex->alter_info.flags|= Alter_info::ALTER_OPTIONS; } | create_table_options_space_separated { @@ -8491,12 +8497,14 @@ select_lock_type: | FOR_SYM UPDATE_SYM { LEX *lex=Lex; + lex->current_select->lock_type= TL_WRITE; lex->current_select->set_lock_for_tables(TL_WRITE); lex->safe_to_cache_query=0; } | LOCK_SYM IN_SYM SHARE_SYM MODE_SYM { LEX *lex=Lex; + lex->current_select->lock_type= TL_READ_WITH_SHARED_LOCKS; lex->current_select-> set_lock_for_tables(TL_READ_WITH_SHARED_LOCKS); lex->safe_to_cache_query=0; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 089b6cd9c7e..5ff79a2f235 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -34,6 +34,7 @@ #include "sql_plugin.h" // Includes my_global.h #include "sql_priv.h" #include "sql_class.h" // set_var.h: THD +#include "sql_parse.h" #include "sys_vars.h" #include "events.h" @@ -613,7 +614,7 @@ static bool check_cs_client(sys_var *self, THD *thd, set_var *var) return true; // Currently, UCS-2 cannot be used as a client character set - if (((CHARSET_INFO *)(var->save_result.ptr))->mbminlen > 1) + if (!is_supported_parser_charset((CHARSET_INFO *)(var->save_result.ptr))) return true; return false; @@ -1526,7 +1527,6 @@ bool Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var) { String str, *res; - bool running; DBUG_ASSERT(var->type == OPT_GLOBAL); @@ -1537,11 +1537,7 @@ Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var) return true; } - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) + if (give_error_if_slave_running(0)) return true; if (!(res= var->value->val_str(&str))) return true; @@ -1579,7 +1575,7 @@ Sys_var_gtid_slave_pos::global_update(THD *thd, set_var *var) mysql_mutex_unlock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_active_mi); - if (!master_info_index || master_info_index->give_error_if_slave_running()) + if (give_error_if_slave_running(1)) err= true; else err= rpl_gtid_pos_update(thd, var->save_result.string_value.str, @@ -1765,16 +1761,7 @@ Sys_var_last_gtid::session_value_ptr(THD *thd, LEX_STRING *base) static bool check_slave_parallel_threads(sys_var *self, THD *thd, set_var *var) { - bool running; - - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) - return true; - - return false; + return give_error_if_slave_running(0); } static bool @@ -1783,10 +1770,7 @@ fix_slave_parallel_threads(sys_var *self, THD *thd, enum_var_type type) bool err; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - err= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); + err= give_error_if_slave_running(0); mysql_mutex_lock(&LOCK_global_system_variables); return err; @@ -1809,16 +1793,7 @@ static Sys_var_ulong Sys_slave_parallel_threads( static bool check_slave_domain_parallel_threads(sys_var *self, THD *thd, set_var *var) { - bool running; - - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) - return true; - - return false; + return give_error_if_slave_running(0); } static bool @@ -1827,13 +1802,10 @@ fix_slave_domain_parallel_threads(sys_var *self, THD *thd, enum_var_type type) bool running; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); + running= give_error_if_slave_running(0); mysql_mutex_lock(&LOCK_global_system_variables); - return running ? true : false; + return running; } @@ -1864,16 +1836,7 @@ static Sys_var_ulong Sys_slave_parallel_max_queued( static bool check_gtid_ignore_duplicates(sys_var *self, THD *thd, set_var *var) { - bool running; - - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) - return true; - - return false; + return give_error_if_slave_running(0); } static bool @@ -1882,13 +1845,10 @@ fix_gtid_ignore_duplicates(sys_var *self, THD *thd, enum_var_type type) bool running; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); + running= give_error_if_slave_running(0); mysql_mutex_lock(&LOCK_global_system_variables); - return running ? true : false; + return running; } @@ -2836,10 +2796,8 @@ Sys_var_replicate_events_marked_for_skip::global_update(THD *thd, set_var *var) DBUG_ENTER("Sys_var_replicate_events_marked_for_skip::global_update"); mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index && !master_info_index->give_error_if_slave_running()) + if (!give_error_if_slave_running(0)) result= Sys_var_enum::global_update(thd, var); - mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); DBUG_RETURN(result); } @@ -4111,19 +4069,16 @@ bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) Master_info *mi; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); if (!var->base.length) // no base name { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR); + mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR); } else // has base name { - mi= master_info_index-> - get_master_info(&var->base, - Sql_condition::WARN_LEVEL_WARN); + mi= get_master_info(&var->base, + Sql_condition::WARN_LEVEL_WARN); } if (mi) @@ -4131,17 +4086,17 @@ bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) if (mi->rli.slave_running) { my_error(ER_SLAVE_MUST_STOP, MYF(0), - mi->connection_name.length, - mi->connection_name.str); + mi->connection_name.length, + mi->connection_name.str); result= true; } else { result= set_filter_value(var->save_result.string_value.str, mi); } + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); return result; } @@ -4149,8 +4104,10 @@ bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) bool Sys_var_rpl_filter::set_filter_value(const char *value, Master_info *mi) { bool status= true; - Rpl_filter* rpl_filter= mi ? mi->rpl_filter : global_rpl_filter; + Rpl_filter* rpl_filter= mi->rpl_filter; + /* Proctect against other threads */ + mysql_mutex_lock(&LOCK_active_mi); switch (opt_id) { case OPT_REPLICATE_DO_DB: status= rpl_filter->set_do_db(value); @@ -4171,7 +4128,7 @@ bool Sys_var_rpl_filter::set_filter_value(const char *value, Master_info *mi) status= rpl_filter->set_wild_ignore_table(value); break; } - + mysql_mutex_unlock(&LOCK_active_mi); return status; } @@ -4184,29 +4141,24 @@ uchar *Sys_var_rpl_filter::global_value_ptr(THD *thd, LEX_STRING *base) Rpl_filter *rpl_filter; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); if (!base->length) // no base name { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR); + mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR); } else // has base name - { - mi= master_info_index-> - get_master_info(base, - Sql_condition::WARN_LEVEL_WARN); - } - mysql_mutex_lock(&LOCK_global_system_variables); + mi= get_master_info(base, Sql_condition::WARN_LEVEL_WARN); if (!mi) { - mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); return 0; } + rpl_filter= mi->rpl_filter; tmp.length(0); + mysql_mutex_lock(&LOCK_active_mi); switch (opt_id) { case OPT_REPLICATE_DO_DB: rpl_filter->get_do_db(&tmp); @@ -4227,9 +4179,12 @@ uchar *Sys_var_rpl_filter::global_value_ptr(THD *thd, LEX_STRING *base) rpl_filter->get_wild_ignore_table(&tmp); break; } + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); + + mi->release(); ret= (uchar *) thd->strmake(tmp.ptr(), tmp.length()); - mysql_mutex_unlock(&LOCK_active_mi); return ret; } @@ -4300,17 +4255,12 @@ get_master_info_ulonglong_value(THD *thd, ptrdiff_t offset) Master_info *mi; ulonglong res= 0; // Default value mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_WARN); - if (mi) + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_WARN))) { - mysql_mutex_lock(&mi->rli.data_lock); res= *((ulonglong*) (((uchar*) mi) + master_info_offset)); - mysql_mutex_unlock(&mi->rli.data_lock); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); return res; } @@ -4325,19 +4275,16 @@ bool update_multi_source_variable(sys_var *self_var, THD *thd, if (type == OPT_GLOBAL) mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR); - if (mi) + if ((mi= (get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR)))) { mysql_mutex_lock(&mi->rli.run_lock); mysql_mutex_lock(&mi->rli.data_lock); result= self->update_variable(thd, mi); mysql_mutex_unlock(&mi->rli.data_lock); mysql_mutex_unlock(&mi->rli.run_lock); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); if (type == OPT_GLOBAL) mysql_mutex_lock(&LOCK_global_system_variables); return result; diff --git a/sql/sys_vars.h b/sql/sys_vars.h index 79b67d3b45c..957c38749ba 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -1235,7 +1235,7 @@ public: if (var->value->result_type() == STRING_RESULT) { - if (!(res=var->value->val_str(&str))) + if (!(res=var->value->val_str_ascii(&str))) return true; else { diff --git a/sql/table.cc b/sql/table.cc index e357d508e9e..975d9d53882 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -569,7 +569,7 @@ enum open_frm_error open_table_def(THD *thd, TABLE_SHARE *share, uint flags) { DBUG_ASSERT(flags & GTS_TABLE); DBUG_ASSERT(flags & GTS_USE_DISCOVERY); - mysql_file_delete_with_symlink(key_file_frm, path, MYF(0)); + mysql_file_delete_with_symlink(key_file_frm, path, "", MYF(0)); file= -1; } else @@ -6686,11 +6686,9 @@ bool is_simple_order(ORDER *order) @details The function computes the values of the virtual columns of the table and stores them in the table record buffer. - If vcol_update_mode is set to VCOL_UPDATE_ALL then all virtual column are - computed. Otherwise, only fields from vcol_set are computed: all of them, - if vcol_update_mode is set to VCOL_UPDATE_FOR_WRITE, and, only those with - the stored_in_db flag set to false, if vcol_update_mode is equal to - VCOL_UPDATE_FOR_READ. + Only fields from vcol_set are computed: all of them, if vcol_update_mode is + set to VCOL_UPDATE_FOR_WRITE, and, only those with the stored_in_db flag + set to false, if vcol_update_mode is equal to VCOL_UPDATE_FOR_READ. @retval 0 Success @@ -6706,15 +6704,16 @@ int update_virtual_fields(THD *thd, TABLE *table, int error __attribute__ ((unused))= 0; DBUG_ASSERT(table && table->vfield); - thd->reset_arena_for_cached_items(table->expr_arena); + Query_arena backup_arena; + thd->set_n_backup_active_arena(table->expr_arena, &backup_arena); + /* Iterate over virtual fields in the table */ for (vfield_ptr= table->vfield; *vfield_ptr; vfield_ptr++) { vfield= (*vfield_ptr); DBUG_ASSERT(vfield->vcol_info && vfield->vcol_info->expr_item); - if ((bitmap_is_set(table->vcol_set, vfield->field_index) && - (vcol_update_mode == VCOL_UPDATE_FOR_WRITE || !vfield->stored_in_db)) || - vcol_update_mode == VCOL_UPDATE_ALL) + if (bitmap_is_set(table->vcol_set, vfield->field_index) && + (vcol_update_mode == VCOL_UPDATE_FOR_WRITE || !vfield->stored_in_db)) { /* Compute the actual value of the virtual fields */ error= vfield->vcol_info->expr_item->save_in_field(vfield, 0); @@ -6725,7 +6724,7 @@ int update_virtual_fields(THD *thd, TABLE *table, DBUG_PRINT("info", ("field '%s' - skipped", vfield->field_name)); } } - thd->reset_arena_for_cached_items(0); + thd->restore_active_arena(table->expr_arena, &backup_arena); DBUG_RETURN(0); } diff --git a/sql/table.h b/sql/table.h index 39faa8b9765..bf98f08842a 100644 --- a/sql/table.h +++ b/sql/table.h @@ -315,8 +315,7 @@ enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP }; enum enum_vcol_update_mode { VCOL_UPDATE_FOR_READ= 0, - VCOL_UPDATE_FOR_WRITE, - VCOL_UPDATE_ALL + VCOL_UPDATE_FOR_WRITE }; class Filesort_info diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index e4b0a372ee0..81441892215 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -270,8 +270,8 @@ IF(CONNECT_WITH_JDBC) # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) - install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + ${CMAKE_CURRENT_BINARY_DIR}/JdbcInterface.jar DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index 81511214bfe..eb06ee7ad1e 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -341,7 +341,7 @@ bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, PCSZ fn, bool append) bool ZIPUTIL::addEntry(PGLOBAL g, PCSZ entry) { //?? we dont need the stinking time - zip_fileinfo zi = { 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + zip_fileinfo zi = { {0, 0, 0, 0, 0, 0}, 0, 0, 0 }; getTime(zi.tmz_date); target = entry; @@ -455,7 +455,7 @@ loopStart: if (!*++pat) return TRUE; goto loopStart; default: - if (mapCaseTable[(uint)*s] != mapCaseTable[(uint)*p]) + if (mapCaseTable[(uchar)*s] != mapCaseTable[(uchar)*p]) goto starCheck; break; } /* endswitch */ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index a0dfbaceca0..9b8aa36b518 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1192,7 +1192,7 @@ char *ha_connect::GetRealString(PCSZ s) { char *sv; - if (IsPartitioned() && s && partname && *partname) { + if (IsPartitioned() && s && *partname) { sv= (char*)PlugSubAlloc(xp->g, NULL, 0); sprintf(sv, s, partname); PlugSubAlloc(xp->g, NULL, strlen(sv) + 1); diff --git a/storage/connect/tabext.cpp b/storage/connect/tabext.cpp index 8f6e610dfc2..ab79b1861de 100644 --- a/storage/connect/tabext.cpp +++ b/storage/connect/tabext.cpp @@ -298,7 +298,8 @@ bool TDBEXT::MakeSQL(PGLOBAL g, bool cnt) PCSZ ph = ((EXTDEF*)To_Def)->Phpos; if (!ph) - ph = (strstr(catp + 2, "%s")) ? "WH" : "W"; + ph = (strstr(catp + 2, "%s")) ? const_cast<char*>("WH") : + const_cast<char*>("W"); if (stricmp(ph, "H")) { fil1 = (To_CondFil && *To_CondFil->Body) diff --git a/storage/connect/tabext.h b/storage/connect/tabext.h index a7f5fb9d856..497b6074d48 100644 --- a/storage/connect/tabext.h +++ b/storage/connect/tabext.h @@ -7,7 +7,7 @@ /***********************************************************************/ #ifndef __TABEXT_H -#define __TABEXTF_H +#define __TABEXT_H #include "reldef.h" diff --git a/storage/csv/ha_tina.cc b/storage/csv/ha_tina.cc index 92ac20a8f82..35596a59c86 100644 --- a/storage/csv/ha_tina.cc +++ b/storage/csv/ha_tina.cc @@ -289,7 +289,7 @@ static int read_meta_file(File meta_file, ha_rows *rows) mysql_file_seek(meta_file, 0, MY_SEEK_SET, MYF(0)); if (mysql_file_read(meta_file, (uchar*)meta_buffer, META_BUFFER_SIZE, 0) != META_BUFFER_SIZE) - DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE); + DBUG_RETURN(my_errno= HA_ERR_CRASHED_ON_USAGE); /* Parse out the meta data, we ignore version at the moment @@ -418,10 +418,13 @@ static int free_share(TINA_SHARE *share) int result_code= 0; if (!--share->use_count){ /* Write the meta file. Mark it as crashed if needed. */ - (void)write_meta_file(share->meta_file, share->rows_recorded, - share->crashed ? TRUE :FALSE); - if (mysql_file_close(share->meta_file, MYF(0))) - result_code= 1; + if (share->meta_file != -1) + { + (void)write_meta_file(share->meta_file, share->rows_recorded, + share->crashed ? TRUE :FALSE); + if (mysql_file_close(share->meta_file, MYF(0))) + result_code= 1; + } if (share->tina_write_opened) { if (mysql_file_close(share->tina_write_filedes, MYF(0))) @@ -930,7 +933,7 @@ int ha_tina::open(const char *name, int mode, uint open_options) if (share->crashed && !(open_options & HA_OPEN_FOR_REPAIR)) { free_share(share); - DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE); + DBUG_RETURN(my_errno ? my_errno : HA_ERR_CRASHED_ON_USAGE); } local_data_file_version= share->data_file_version; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index fca433f9648..4f24ad75e26 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1880,7 +1880,6 @@ btr_cur_update_alloc_zip_func( const page_t* page = page_cur_get_page(cursor); ut_ad(page_zip == page_cur_get_page_zip(cursor)); - ut_ad(page_zip); ut_ad(!dict_index_is_ibuf(index)); ut_ad(rec_offs_validate(page_cur_get_rec(cursor), index, offsets)); @@ -2962,7 +2961,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && (thr != NULL)) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr)->id, index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -4278,7 +4277,6 @@ btr_cur_disown_inherited_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(mtr); for (i = 0; i < rec_offs_n_fields(offsets); i++) { if (rec_offs_nth_extern(offsets, i) @@ -4341,9 +4339,6 @@ btr_push_update_extern_fields( ulint n; const upd_field_t* uf; - ut_ad(tuple); - ut_ad(update); - uf = update->fields; n = upd_get_n_fields(update); @@ -4515,7 +4510,6 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(btr_mtr); ut_ad(mtr_memo_contains(btr_mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains(btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX)); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 2c7c578150d..0b305507271 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4415,7 +4415,8 @@ buf_all_freed_instance( block->page.space, block->page.offset); ib_logf(IB_LOG_LEVEL_ERROR, - "Page oldest_modification %lu fix_count %d io_fix %d.", + "Page oldest_modification " LSN_PF + " fix_count %d io_fix %d.", block->page.oldest_modification, block->page.buf_fix_count, buf_page_get_io_fix(&block->page)); @@ -5310,23 +5311,22 @@ buf_print_io_instance( pool_info->pages_written_rate); if (pool_info->n_page_get_delta) { - double hit_rate = ((1000 * pool_info->page_read_delta) - / pool_info->n_page_get_delta); + double hit_rate = double(pool_info->page_read_delta) + / pool_info->n_page_get_delta; - if (hit_rate > 1000) { - hit_rate = 1000; + if (hit_rate > 1) { + hit_rate = 1; } - hit_rate = 1000 - hit_rate; - fprintf(file, - "Buffer pool hit rate %lu / 1000," - " young-making rate %lu / 1000 not %lu / 1000\n", - (ulint) hit_rate, - (ulint) (1000 * pool_info->young_making_delta - / pool_info->n_page_get_delta), - (ulint) (1000 * pool_info->not_young_making_delta - / pool_info->n_page_get_delta)); + "Buffer pool hit rate " ULINTPF " / 1000," + " young-making rate " ULINTPF " / 1000 not " + ULINTPF " / 1000\n", + ulint(1000 * (1 - hit_rate)), + ulint(1000 * double(pool_info->young_making_delta) + / pool_info->n_page_get_delta), + ulint(1000 * double(pool_info->not_young_making_delta) + / pool_info->n_page_get_delta)); } else { fputs("No buffer pool page gets since the last printout\n", file); diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index d08bde763b3..d5efd87f9e1 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -52,8 +53,8 @@ enum status_severity { /* Flags that tell the buffer pool dump/load thread which action should it take after being waked up. */ -static ibool buf_dump_should_start = FALSE; -static ibool buf_load_should_start = FALSE; +static volatile bool buf_dump_should_start; +static volatile bool buf_load_should_start; static ibool buf_load_abort_flag = FALSE; @@ -78,7 +79,7 @@ void buf_dump_start() /*============*/ { - buf_dump_should_start = TRUE; + buf_dump_should_start = true; os_event_set(srv_buf_dump_event); } @@ -92,7 +93,7 @@ void buf_load_start() /*============*/ { - buf_load_should_start = TRUE; + buf_load_should_start = true; os_event_set(srv_buf_dump_event); } @@ -695,15 +696,18 @@ DECLARE_THREAD(buf_dump_thread)( os_event_wait(srv_buf_dump_event); if (buf_dump_should_start) { - buf_dump_should_start = FALSE; + buf_dump_should_start = false; buf_dump(TRUE /* quit on shutdown */); } if (buf_load_should_start) { - buf_load_should_start = FALSE; + buf_load_should_start = false; buf_load(); } + if (buf_dump_should_start || buf_load_should_start) { + continue; + } os_event_reset(srv_buf_dump_event); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index a139166d5d3..d923514123c 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -732,8 +733,7 @@ buf_flush_write_complete( flush_type = buf_page_get_flush_type(bpage); buf_pool->n_flush[flush_type]--; - /* fprintf(stderr, "n pending flush %lu\n", - buf_pool->n_flush[flush_type]); */ + ut_ad(buf_pool_mutex_own(buf_pool)); if (buf_pool->n_flush[flush_type] == 0 && buf_pool->init_flush[flush_type] == FALSE) { diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index f981eb31ba8..6ec443b4600 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1369,7 +1370,7 @@ loop: ++flush_failures; } - srv_stats.buf_pool_wait_free.add(n_iterations, 1); + srv_stats.buf_pool_wait_free.inc(); n_iterations++; diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 7c8369c0c09..27f45308813 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -881,15 +881,11 @@ buf_read_recv_pages( count++; if (count > 1000) { - fprintf(stderr, - "InnoDB: Error: InnoDB has waited for" - " 10 seconds for pending\n" - "InnoDB: reads to the buffer pool to" - " be finished.\n" - "InnoDB: Number of pending reads %lu," - " pending pread calls %lu\n", - (ulong) buf_pool->n_pend_reads, - (ulong) os_file_n_pending_preads); + ib_logf(IB_LOG_LEVEL_ERROR, + "waited for 10 seconds for " ULINTPF + " pending reads to the buffer pool to" + " be finished", + buf_pool->n_pend_reads); os_aio_print_debug = TRUE; } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 2e40946224e..b6ca02fa164 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -6156,7 +6156,6 @@ dict_set_corrupted( row_mysql_lock_data_dictionary(trx); } - ut_ad(index); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); @@ -6252,6 +6251,7 @@ dict_set_corrupted_index_cache_only( dict_table_t* table) /*!< in/out: table */ { ut_ad(index != NULL); + ut_ad(table != NULL); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); @@ -6261,9 +6261,7 @@ dict_set_corrupted_index_cache_only( if (dict_index_is_clust(index)) { dict_table_t* corrupt_table; - corrupt_table = (table != NULL) ? table : index->table; - ut_ad((index->table != NULL) || (table != NULL) - || index->table == table); + corrupt_table = table; if (corrupt_table) { corrupt_table->corrupted = TRUE; diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index a4aa43651f8..c0a83c951a5 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2009, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1109,7 +1109,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1295,8 +1296,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1572,6 +1577,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 3215fe9a456..05ee6f5886f 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -39,8 +40,9 @@ Created Apr 25, 2012 Vasil Dimov #define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) -/** Event to wake up the stats thread */ -UNIV_INTERN os_event_t dict_stats_event = NULL; +/** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() +or shutdown. Not protected by any mutex. */ +UNIV_INTERN os_event_t dict_stats_event; /** This mutex protects the "recalc_pool" variable. */ static ib_mutex_t recalc_pool_mutex; diff --git a/storage/innobase/dyn/dyn0dyn.cc b/storage/innobase/dyn/dyn0dyn.cc index 3ef5297a7c9..dd1f6863c14 100644 --- a/storage/innobase/dyn/dyn0dyn.cc +++ b/storage/innobase/dyn/dyn0dyn.cc @@ -40,7 +40,6 @@ dyn_array_add_block( mem_heap_t* heap; dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 9d471f9dbd3..72ee8d74a82 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -4953,15 +4954,12 @@ fil_extend_space_to_desired_size( byte* buf; ulint buf_size; ulint start_page_no; - ulint file_start_page_no; ulint page_size; - ulint pages_added; ibool success; ut_ad(!srv_read_only_mode); retry: - pages_added = 0; success = TRUE; fil_mutex_enter_and_prepare_for_io(space_id); @@ -5015,29 +5013,36 @@ retry: mutex_exit(&fil_system->mutex); start_page_no = space->size; - file_start_page_no = space->size - node->size; - + const ulint file_start_page_no = space->size - node->size; #ifdef HAVE_POSIX_FALLOCATE if (srv_use_posix_fallocate) { - os_offset_t start_offset = start_page_no * page_size; - os_offset_t n_pages = (size_after_extend - start_page_no); - os_offset_t len = n_pages * page_size; - - if (posix_fallocate(node->handle, start_offset, len) == -1) { - ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " - "space for file \'%s\' failed. Current size " - INT64PF ", desired size " INT64PF "\n", - node->name, start_offset, len+start_offset); - os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE); - success = FALSE; - } else { - success = TRUE; + const os_offset_t start_offset + = os_offset_t(start_page_no - file_start_page_no) + * page_size; + const ulint n_pages + = size_after_extend - start_page_no; + const os_offset_t len = os_offset_t(n_pages) * page_size; + + int err; + do { + err = posix_fallocate(node->handle, start_offset, len); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); + + success = !err; + if (!success) { + ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" + " from " INT64PF " to " INT64PF " bytes" + " failed with error %d", + node->name, start_offset, len + start_offset, + err); } DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28; os_has_said_disk_full = TRUE;); + success = FALSE; os_has_said_disk_full = TRUE;); mutex_enter(&fil_system->mutex); + if (success) { node->size += n_pages; space->size += n_pages; @@ -5054,14 +5059,24 @@ retry: } #endif +#ifdef _WIN32 + /* Write 1 page of zeroes at the desired end. */ + start_page_no = size_after_extend - 1; + buf_size = page_size; +#else /* Extend at most 64 pages at a time */ buf_size = ut_min(64, size_after_extend - start_page_no) * page_size; - buf2 = static_cast<byte*>(mem_alloc(buf_size + page_size)); +#endif + buf2 = static_cast<byte*>(calloc(1, buf_size + page_size)); + if (!buf2) { + ib_logf(IB_LOG_LEVEL_ERROR, "Cannot allocate " ULINTPF + " bytes to extend file", + buf_size + page_size); + success = FALSE; + } buf = static_cast<byte*>(ut_align(buf2, page_size)); - memset(buf, 0, buf_size); - - while (start_page_no < size_after_extend) { + while (success && start_page_no < size_after_extend) { ulint n_pages = ut_min(buf_size / page_size, size_after_extend - start_page_no); @@ -5070,56 +5085,47 @@ retry: = ((os_offset_t) (start_page_no - file_start_page_no)) * page_size; - const char* name = node->name == NULL ? space->name : node->name; - #ifdef UNIV_HOTBACKUP - success = os_file_write(name, node->handle, buf, + success = os_file_write(node->name, node->handle, buf, offset, page_size * n_pages); #else success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, - name, node->handle, buf, + node->name, node->handle, buf, offset, page_size * n_pages, NULL, NULL); #endif /* UNIV_HOTBACKUP */ - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28; os_has_said_disk_full = TRUE;); - - if (success) { - os_has_said_disk_full = FALSE; - } else { - /* Let us measure the size of the file to determine - how much we were able to extend it */ - os_offset_t size; + success = FALSE; os_has_said_disk_full = TRUE;); - size = os_file_get_size(node->handle); - ut_a(size != (os_offset_t) -1); + /* Let us measure the size of the file to determine + how much we were able to extend it */ + os_offset_t size = os_file_get_size(node->handle); + ut_a(size != (os_offset_t) -1); - n_pages = ((ulint) (size / page_size)) - - node->size - pages_added; - - pages_added += n_pages; - break; - } - - start_page_no += n_pages; - pages_added += n_pages; + start_page_no = (ulint) (size / page_size) + + file_start_page_no; } - mem_free(buf2); + free(buf2); mutex_enter(&fil_system->mutex); ut_a(node->being_extended); + ut_a(start_page_no - file_start_page_no >= node->size); - space->size += pages_added; - node->size += pages_added; + if (buf) { + ulint file_size = start_page_no - file_start_page_no; + space->size += file_size - node->size; + node->size = file_size; + } fil_node_complete_io(node, fil_system, OS_FILE_WRITE); /* At this point file has been extended */ +#ifdef HAVE_POSIX_FALLOCATE file_extended: +#endif /* HAVE_POSIX_FALLOCATE */ node->being_extended = FALSE; *actual_size = space->size; diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index afdd6667711..1732a02aeac 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1064,8 +1064,6 @@ fsp_fill_free_list( ulint i; mtr_t ibuf_mtr; - ut_ad(header != NULL); - ut_ad(mtr != NULL); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); /* Check if we can fill free list from above the free list limit */ @@ -1348,9 +1346,6 @@ fsp_alloc_free_page( ulint page_no; ulint space_size; - ut_ad(mtr); - ut_ad(init_mtr); - header = fsp_get_space_header(space, zip_size, mtr); /* Get the hinted descriptor */ @@ -2363,7 +2358,6 @@ fseg_alloc_free_page_low( ibool success; ulint n; - ut_ad(mtr); ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); @@ -2801,6 +2795,7 @@ try_again: } } else { ut_a(alloc_type == FSP_CLEANING); + reserve = 0; } success = fil_space_reserve_free_extents(space, n_free, n_ext); diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 6059c28eabc..18d09c8138f 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1934,6 +1935,8 @@ func_exit: /*************************************************************//** Wrapper function of fts_create_index_tables_low(), create auxiliary tables for an FTS index + +@see row_merge_create_fts_sort_index() @return: DB_SUCCESS or error code */ static dict_table_t* @@ -1965,13 +1968,12 @@ fts_create_one_index_table( (int)(field->col->prtype & DATA_MYSQL_TYPE_MASK), (uint) dtype_get_charset_coll(field->col->prtype)); - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARCHAR, - field->col->prtype, FTS_MAX_WORD_LEN); - } else { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARMYSQL, - field->col->prtype, FTS_MAX_WORD_LEN); - } + dict_mem_table_add_col(new_table, heap, "word", + charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL, + field->col->prtype, + FTS_MAX_WORD_LEN_IN_CHAR + * DATA_MBMAXLEN(field->col->mbminmaxlen)); dict_mem_table_add_col(new_table, heap, "first_doc_id", DATA_INT, DATA_NOT_NULL | DATA_UNSIGNED, diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ea937c20752..cb30122adcb 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -281,7 +282,7 @@ fts_zip_initialize( zip->last_big_block = 0; zip->word.f_len = 0; - memset(zip->word.f_str, 0, FTS_MAX_WORD_LEN); + *zip->word.f_str = 0; ib_vector_reset(zip->blocks); @@ -578,9 +579,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -655,10 +653,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -669,7 +666,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 59ebc411db0..1538471db4f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2000,12 +2000,11 @@ innobase_get_stmt( { const char* query = NULL; LEX_STRING *stmt = NULL; - if (thd) { - stmt = thd_query_string(thd); - if (stmt) { - *length = stmt->length; - query = stmt->str; - } + ut_ad(thd != NULL); + stmt = thd_query_string(thd); + if (stmt) { + *length = stmt->length; + query = stmt->str; } return (query); } @@ -3539,7 +3538,6 @@ innobase_change_buffering_inited_ok: and consequently we do not need to know the ordering internally in InnoDB. */ - ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); srv_latin1_ordering = my_charset_latin1.sort_order; innobase_commit_concurrency_init_default(); @@ -6149,18 +6147,16 @@ get_innobase_type_from_mysql_type( case MYSQL_TYPE_VARCHAR: /* new >= 5.0.3 true VARCHAR */ if (field->binary()) { return(DATA_BINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_VARCHAR); } else { return(DATA_VARMYSQL); } case MYSQL_TYPE_BIT: - case MYSQL_TYPE_STRING: if (field->binary()) { - + case MYSQL_TYPE_STRING: + if (field->binary()) { return(DATA_FIXBINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_CHAR); } else { return(DATA_MYSQL); @@ -13938,6 +13934,37 @@ ha_innobase::get_auto_increment( ulonglong col_max_value = innobase_get_int_col_max_value( table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { @@ -16462,6 +16489,12 @@ static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, "Preallocate files fast, using operating system functionality. On POSIX systems, posix_fallocate system call is used.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Scan delete marked records for persistent stat", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -16934,13 +16967,6 @@ static MYSQL_SYSVAR_ULONG(force_recovery, srv_force_recovery, "Helps to save your data in case the disk image of the database becomes corrupt.", NULL, NULL, 0, 0, 6, 0); -#ifndef DBUG_OFF -static MYSQL_SYSVAR_ULONG(force_recovery_crash, srv_force_recovery_crash, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Kills the server during crash recovery.", - NULL, NULL, 0, 0, 10, 0); -#endif /* !DBUG_OFF */ - static MYSQL_SYSVAR_ULONG(page_size, srv_page_size, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, "Page size to use for all InnoDB tablespaces.", @@ -17293,6 +17319,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(doublewrite), MYSQL_SYSVAR(use_atomic_writes), MYSQL_SYSVAR(use_fallocate), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), @@ -17308,9 +17335,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(flush_log_at_trx_commit), MYSQL_SYSVAR(flush_method), MYSQL_SYSVAR(force_recovery), -#ifndef DBUG_OFF - MYSQL_SYSVAR(force_recovery_crash), -#endif /* !DBUG_OFF */ MYSQL_SYSVAR(ft_cache_size), MYSQL_SYSVAR(ft_total_cache_size), MYSQL_SYSVAR(ft_result_cache_limit), diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 31648798776..dc084dc1b95 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -129,7 +129,6 @@ my_error_innodb( break; case DB_OUT_OF_FILE_SPACE: my_error(ER_RECORD_FILE_FULL, MYF(0), table); - ut_error; break; case DB_TEMP_FILE_WRITE_FAILURE: my_error(ER_GET_ERRMSG, MYF(0), @@ -1838,6 +1837,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -1906,26 +1906,11 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); - - if (maybe_null) { - new_primary = false; - break; - } - } + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id @@ -1944,8 +1929,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -3053,6 +3044,8 @@ prepare_inplace_alter_table_dict( ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info, old_table)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); if (!ctx->new_table->fts && innobase_fulltext_exist(altered_table)) { @@ -3890,7 +3883,7 @@ check_if_can_drop_indexes: index->name, TRUE); my_error(ER_INDEX_CORRUPT, MYF(0), index_name); - DBUG_RETURN(true); + goto err_exit; } } } @@ -4073,6 +4066,27 @@ found_col: add_fts_doc_id_idx)); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} + /** Alter the table structure in-place with operations specified using Alter_inplace_info. The level of concurrency allowed during this operation depends @@ -4190,17 +4204,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; default: my_error_innodb(error, @@ -5013,7 +5023,6 @@ innobase_update_foreign_cache( "Foreign key constraints for table '%s'" " are loaded with charset check off", user_table->name); - } } @@ -5113,14 +5122,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 50340e05860..2f5fe45656e 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -143,6 +144,7 @@ struct buf_page_info_t{ if ((expr) != 0) { \ DBUG_RETURN(1); \ } +#define BREAK_IF(expr) if ((expr)) break #define RETURN_IF_INNODB_NOT_STARTED(plugin_name) \ do { \ @@ -2994,14 +2996,16 @@ i_s_fts_deleted_generic_fill( fields = table->field; + int ret = 0; + for (ulint j = 0; j < ib_vector_size(deleted->doc_ids); ++j) { doc_id_t doc_id; doc_id = *(doc_id_t*) ib_vector_get_const(deleted->doc_ids, j); - OK(fields[I_S_FTS_DOC_ID]->store((longlong) doc_id, true)); + BREAK_IF(ret = fields[I_S_FTS_DOC_ID]->store(doc_id, true)); - OK(schema_table_store_record(thd, table)); + BREAK_IF(ret = schema_table_store_record(thd, table)); } trx_free_for_background(trx); @@ -3012,7 +3016,7 @@ i_s_fts_deleted_generic_fill( rw_lock_s_unlock(&dict_operation_lock); - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** @@ -3252,13 +3256,13 @@ i_s_fts_index_cache_fill_one_index( /*===============================*/ fts_index_cache_t* index_cache, /*!< in: FTS index cache */ THD* thd, /*!< in: thread */ + fts_string_t* conv_str, /*!< in/out: buffer */ TABLE_LIST* tables) /*!< in/out: tables to fill */ { TABLE* table = (TABLE*) tables->table; Field** fields; CHARSET_INFO* index_charset; const ib_rbt_node_t* rbt_node; - fts_string_t conv_str; uint dummy_errors; char* word_str; @@ -3267,10 +3271,9 @@ i_s_fts_index_cache_fill_one_index( fields = table->field; index_charset = index_cache->charset; - conv_str.f_len = system_charset_info->mbmaxlen - * FTS_MAX_WORD_LEN_IN_CHAR; - conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); - conv_str.f_n_char = 0; + conv_str->f_n_char = 0; + + int ret = 0; /* Go through each word in the index cache */ for (rbt_node = rbt_first(index_cache->words); @@ -3282,16 +3285,16 @@ i_s_fts_index_cache_fill_one_index( /* Convert word from index charset to system_charset_info */ if (index_charset->cset != system_charset_info->cset) { - conv_str.f_n_char = my_convert( - reinterpret_cast<char*>(conv_str.f_str), - static_cast<uint32>(conv_str.f_len), + conv_str->f_n_char = my_convert( + reinterpret_cast<char*>(conv_str->f_str), + static_cast<uint32>(conv_str->f_len), system_charset_info, reinterpret_cast<char*>(word->text.f_str), static_cast<uint32>(word->text.f_len), index_charset, &dummy_errors); - ut_ad(conv_str.f_n_char <= conv_str.f_len); - conv_str.f_str[conv_str.f_n_char] = 0; - word_str = reinterpret_cast<char*>(conv_str.f_str); + ut_ad(conv_str->f_n_char <= conv_str->f_len); + conv_str->f_str[conv_str->f_n_char] = 0; + word_str = reinterpret_cast<char*>(conv_str->f_str); } else { word_str = reinterpret_cast<char*>(word->text.f_str); } @@ -3349,9 +3352,7 @@ i_s_fts_index_cache_fill_one_index( } } - ut_free(conv_str.f_str); - - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** Fill the dynamic table INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHED @@ -3395,18 +3396,27 @@ i_s_fts_index_cache_fill( ut_a(cache); + int ret = 0; + fts_string_t conv_str; + conv_str.f_len = system_charset_info->mbmaxlen + * FTS_MAX_WORD_LEN_IN_CHAR; + conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); + for (ulint i = 0; i < ib_vector_size(cache->indexes); i++) { fts_index_cache_t* index_cache; index_cache = static_cast<fts_index_cache_t*> ( ib_vector_get(cache->indexes, i)); - i_s_fts_index_cache_fill_one_index(index_cache, thd, tables); + BREAK_IF(ret = i_s_fts_index_cache_fill_one_index( + index_cache, thd, &conv_str, tables)); } + ut_free(conv_str.f_str); + dict_table_close(user_table, FALSE, FALSE); - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** @@ -3710,8 +3720,6 @@ i_s_fts_index_table_fill_one_fetch( } } - i_s_fts_index_table_free_one_fetch(words); - DBUG_RETURN(ret); } @@ -3725,13 +3733,13 @@ i_s_fts_index_table_fill_one_index( /*===============================*/ dict_index_t* index, /*!< in: FTS index */ THD* thd, /*!< in: thread */ + fts_string_t* conv_str, /*!< in/out: buffer */ TABLE_LIST* tables) /*!< in/out: tables to fill */ { ib_vector_t* words; mem_heap_t* heap; fts_string_t word; CHARSET_INFO* index_charset; - fts_string_t conv_str; dberr_t error; int ret = 0; @@ -3748,10 +3756,6 @@ i_s_fts_index_table_fill_one_index( word.f_n_char = 0; index_charset = fts_index_get_charset(index); - conv_str.f_len = system_charset_info->mbmaxlen - * FTS_MAX_WORD_LEN_IN_CHAR; - conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); - conv_str.f_n_char = 0; /* Iterate through each auxiliary table as described in fts_index_selector */ @@ -3785,17 +3789,17 @@ i_s_fts_index_table_fill_one_index( /* Fill into tables */ ret = i_s_fts_index_table_fill_one_fetch( - index_charset, thd, tables, words, &conv_str, has_more); + index_charset, thd, tables, words, conv_str, + has_more); + i_s_fts_index_table_free_one_fetch(words); if (ret != 0) { - i_s_fts_index_table_free_one_fetch(words); goto func_exit; } } while (has_more); } func_exit: - ut_free(conv_str.f_str); mem_heap_free(heap); DBUG_RETURN(ret); @@ -3837,10 +3841,17 @@ i_s_fts_index_table_fill( DBUG_RETURN(0); } + int ret = 0; + fts_string_t conv_str; + conv_str.f_len = system_charset_info->mbmaxlen + * FTS_MAX_WORD_LEN_IN_CHAR; + conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); + for (index = dict_table_get_first_index(user_table); index; index = dict_table_get_next_index(index)) { if (index->type & DICT_FTS) { - i_s_fts_index_table_fill_one_index(index, thd, tables); + BREAK_IF(ret = i_s_fts_index_table_fill_one_index( + index, thd, &conv_str, tables)); } } @@ -3848,7 +3859,9 @@ i_s_fts_index_table_fill( rw_lock_s_unlock(&dict_operation_lock); - DBUG_RETURN(0); + ut_free(conv_str.f_str); + + DBUG_RETURN(ret); } /*******************************************************************//** @@ -4014,6 +4027,8 @@ i_s_fts_config_fill( DBUG_ASSERT(!dict_index_is_online_ddl(index)); } + int ret = 0; + while (fts_config_key[i]) { fts_string_t value; char* key_name; @@ -4038,13 +4053,14 @@ i_s_fts_config_fill( ut_free(key_name); } - OK(field_store_string( - fields[FTS_CONFIG_KEY], fts_config_key[i])); + BREAK_IF(ret = field_store_string( + fields[FTS_CONFIG_KEY], fts_config_key[i])); - OK(field_store_string( - fields[FTS_CONFIG_VALUE], (const char*) value.f_str)); + BREAK_IF(ret = field_store_string( + fields[FTS_CONFIG_VALUE], + reinterpret_cast<const char*>(value.f_str))); - OK(schema_table_store_record(thd, table)); + BREAK_IF(ret = schema_table_store_record(thd, table)); i++; } @@ -4057,7 +4073,7 @@ i_s_fts_config_fill( rw_lock_s_unlock(&dict_operation_lock); - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** @@ -4895,34 +4911,29 @@ i_s_innodb_buffer_page_fill( state_str = NULL; OK(fields[IDX_BUFFER_POOL_ID]->store( - static_cast<double>(page_info->pool_id))); + page_info->pool_id, true)); OK(fields[IDX_BUFFER_BLOCK_ID]->store( - static_cast<double>(page_info->block_id))); + page_info->block_id, true)); OK(fields[IDX_BUFFER_PAGE_SPACE]->store( - static_cast<double>(page_info->space_id))); + page_info->space_id, true)); OK(fields[IDX_BUFFER_PAGE_NUM]->store( - static_cast<double>(page_info->page_num))); + page_info->page_num, true)); OK(field_store_string( fields[IDX_BUFFER_PAGE_TYPE], i_s_page_type[page_info->page_type].type_str)); OK(fields[IDX_BUFFER_PAGE_FLUSH_TYPE]->store( - page_info->flush_type)); + page_info->flush_type, true)); OK(fields[IDX_BUFFER_PAGE_FIX_COUNT]->store( - page_info->fix_count)); + page_info->fix_count, true)); - if (page_info->hashed) { - OK(field_store_string( - fields[IDX_BUFFER_PAGE_HASHED], "YES")); - } else { - OK(field_store_string( - fields[IDX_BUFFER_PAGE_HASHED], "NO")); - } + OK(field_store_string(fields[IDX_BUFFER_PAGE_HASHED], + page_info->hashed ? "YES" : "NO")); OK(fields[IDX_BUFFER_PAGE_NEWEST_MOD]->store( (longlong) page_info->newest_mod, true)); @@ -4931,7 +4942,7 @@ i_s_innodb_buffer_page_fill( (longlong) page_info->oldest_mod, true)); OK(fields[IDX_BUFFER_PAGE_ACCESS_TIME]->store( - page_info->access_time)); + page_info->access_time, true)); fields[IDX_BUFFER_PAGE_TABLE_NAME]->set_null(); @@ -4940,44 +4951,48 @@ i_s_innodb_buffer_page_fill( /* If this is an index page, fetch the index name and table name */ if (page_info->page_type == I_S_PAGE_TYPE_INDEX) { - const dict_index_t* index; + bool ret = false; mutex_enter(&dict_sys->mutex); - index = dict_index_get_if_in_cache_low( - page_info->index_id); - - if (index) { + if (const dict_index_t* index = + dict_index_get_if_in_cache_low( + page_info->index_id)) { table_name_end = innobase_convert_name( table_name, sizeof(table_name), index->table_name, strlen(index->table_name), thd, TRUE); - OK(fields[IDX_BUFFER_PAGE_TABLE_NAME]->store( - table_name, - static_cast<uint>(table_name_end - table_name), - system_charset_info)); - fields[IDX_BUFFER_PAGE_TABLE_NAME]->set_notnull(); - - OK(field_store_index_name( - fields[IDX_BUFFER_PAGE_INDEX_NAME], - index->name)); + ret = fields[IDX_BUFFER_PAGE_TABLE_NAME] + ->store(table_name, + static_cast<uint>( + table_name_end + - table_name), + system_charset_info) + || field_store_index_name( + fields + [IDX_BUFFER_PAGE_INDEX_NAME], + index->name); } mutex_exit(&dict_sys->mutex); + + OK(ret); + + fields[IDX_BUFFER_PAGE_TABLE_NAME]->set_notnull(); } OK(fields[IDX_BUFFER_PAGE_NUM_RECS]->store( - page_info->num_recs)); + page_info->num_recs, true)); OK(fields[IDX_BUFFER_PAGE_DATA_SIZE]->store( - page_info->data_size)); + page_info->data_size, true)); OK(fields[IDX_BUFFER_PAGE_ZIP_SIZE]->store( - page_info->zip_ssize - ? (UNIV_ZIP_SIZE_MIN >> 1) << page_info->zip_ssize - : 0)); + page_info->zip_ssize + ? (UNIV_ZIP_SIZE_MIN >> 1) << page_info->zip_ssize + : 0, true)); #if BUF_PAGE_STATE_BITS > 3 # error "BUF_PAGE_STATE_BITS > 3, please ensure that all 1<<BUF_PAGE_STATE_BITS values are checked for" @@ -5015,32 +5030,29 @@ i_s_innodb_buffer_page_fill( switch (page_info->io_fix) { case BUF_IO_NONE: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_NONE")); + state_str = "IO_NONE"; break; case BUF_IO_READ: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_READ")); + state_str = "IO_READ"; break; case BUF_IO_WRITE: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_WRITE")); + state_str = "IO_WRITE"; break; case BUF_IO_PIN: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_PIN")); + state_str = "IO_PIN"; break; } + OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], + state_str)); + OK(field_store_string(fields[IDX_BUFFER_PAGE_IS_OLD], (page_info->is_old) ? "YES" : "NO")); OK(fields[IDX_BUFFER_PAGE_FREE_CLOCK]->store( page_info->freed_page_clock)); - if (schema_table_store_record(thd, table)) { - DBUG_RETURN(1); - } + OK(schema_table_store_record(thd, table)); } DBUG_RETURN(0); @@ -5584,17 +5596,10 @@ i_s_innodb_buf_page_lru_fill( ulint num_page) /*!< in: number of page info cached */ { - TABLE* table; - Field** fields; - mem_heap_t* heap; - DBUG_ENTER("i_s_innodb_buf_page_lru_fill"); - table = tables->table; - - fields = table->field; - - heap = mem_heap_create(1000); + TABLE* table = tables->table; + Field** fields = table->field; /* Iterate through the cached array and fill the I_S table rows */ for (ulint i = 0; i < num_page; i++) { @@ -5609,43 +5614,37 @@ i_s_innodb_buf_page_lru_fill( page_info = info_array + i; OK(fields[IDX_BUF_LRU_POOL_ID]->store( - static_cast<double>(page_info->pool_id))); - + page_info->pool_id, true)); OK(fields[IDX_BUF_LRU_POS]->store( - static_cast<double>(page_info->block_id))); + page_info->block_id, true)); OK(fields[IDX_BUF_LRU_PAGE_SPACE]->store( - static_cast<double>(page_info->space_id))); + page_info->space_id, true)); OK(fields[IDX_BUF_LRU_PAGE_NUM]->store( - static_cast<double>(page_info->page_num))); + page_info->page_num, true)); OK(field_store_string( - fields[IDX_BUF_LRU_PAGE_TYPE], - i_s_page_type[page_info->page_type].type_str)); + fields[IDX_BUF_LRU_PAGE_TYPE], + i_s_page_type[page_info->page_type].type_str)); OK(fields[IDX_BUF_LRU_PAGE_FLUSH_TYPE]->store( - static_cast<double>(page_info->flush_type))); + page_info->flush_type, true)); OK(fields[IDX_BUF_LRU_PAGE_FIX_COUNT]->store( - static_cast<double>(page_info->fix_count))); + page_info->fix_count, true)); - if (page_info->hashed) { - OK(field_store_string( - fields[IDX_BUF_LRU_PAGE_HASHED], "YES")); - } else { - OK(field_store_string( - fields[IDX_BUF_LRU_PAGE_HASHED], "NO")); - } + OK(field_store_string(fields[IDX_BUF_LRU_PAGE_HASHED], + page_info->hashed ? "YES" : "NO")); OK(fields[IDX_BUF_LRU_PAGE_NEWEST_MOD]->store( - page_info->newest_mod, true)); + page_info->newest_mod, true)); OK(fields[IDX_BUF_LRU_PAGE_OLDEST_MOD]->store( - page_info->oldest_mod, true)); + page_info->oldest_mod, true)); OK(fields[IDX_BUF_LRU_PAGE_ACCESS_TIME]->store( - page_info->access_time)); + page_info->access_time, true)); fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->set_null(); @@ -5654,43 +5653,47 @@ i_s_innodb_buf_page_lru_fill( /* If this is an index page, fetch the index name and table name */ if (page_info->page_type == I_S_PAGE_TYPE_INDEX) { - const dict_index_t* index; + bool ret = false; mutex_enter(&dict_sys->mutex); - index = dict_index_get_if_in_cache_low( - page_info->index_id); - - if (index) { + if (const dict_index_t* index = + dict_index_get_if_in_cache_low( + page_info->index_id)) { table_name_end = innobase_convert_name( table_name, sizeof(table_name), index->table_name, strlen(index->table_name), thd, TRUE); - OK(fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->store( - table_name, - static_cast<uint>(table_name_end - table_name), - system_charset_info)); - fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->set_notnull(); - - OK(field_store_index_name( - fields[IDX_BUF_LRU_PAGE_INDEX_NAME], - index->name)); + ret = fields[IDX_BUF_LRU_PAGE_TABLE_NAME] + ->store(table_name, + static_cast<uint>( + table_name_end + - table_name), + system_charset_info) + || field_store_index_name( + fields + [IDX_BUF_LRU_PAGE_INDEX_NAME], + index->name); } mutex_exit(&dict_sys->mutex); + + OK(ret); + + fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->set_notnull(); } OK(fields[IDX_BUF_LRU_PAGE_NUM_RECS]->store( - page_info->num_recs)); + page_info->num_recs, true)); OK(fields[IDX_BUF_LRU_PAGE_DATA_SIZE]->store( - page_info->data_size)); + page_info->data_size, true)); OK(fields[IDX_BUF_LRU_PAGE_ZIP_SIZE]->store( - page_info->zip_ssize ? - 512 << page_info->zip_ssize : 0)); + page_info->zip_ssize + ? 512 << page_info->zip_ssize : 0, true)); state = static_cast<enum buf_page_state>(page_info->page_state); @@ -5719,35 +5722,31 @@ i_s_innodb_buf_page_lru_fill( switch (page_info->io_fix) { case BUF_IO_NONE: - OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], - "IO_NONE")); + state_str = "IO_NONE"; break; case BUF_IO_READ: - OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], - "IO_READ")); + state_str = "IO_READ"; break; case BUF_IO_WRITE: - OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], - "IO_WRITE")); + state_str = "IO_WRITE"; + break; + case BUF_IO_PIN: + state_str = "IO_PIN"; break; } + OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], + state_str)); + OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IS_OLD], - (page_info->is_old) ? "YES" : "NO")); + page_info->is_old ? "YES" : "NO")); OK(fields[IDX_BUF_LRU_PAGE_FREE_CLOCK]->store( - page_info->freed_page_clock)); + page_info->freed_page_clock, true)); - if (schema_table_store_record(thd, table)) { - mem_heap_free(heap); - DBUG_RETURN(1); - } - - mem_heap_empty(heap); + OK(schema_table_store_record(thd, table)); } - mem_heap_free(heap); - DBUG_RETURN(0); } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index b80e5d505ac..78e961d91b7 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1905,7 +1906,9 @@ struct buf_pool_t{ os_event_t no_flush[BUF_FLUSH_N_TYPES]; /*!< this is in the set state when there is no flush batch - of the given type running */ + of the given type running; + os_event_set() and os_event_reset() + are protected by buf_pool_t::mutex */ ib_rbt_t* flush_rbt; /*!< a red-black tree is used exclusively during recovery to speed up insertions in the diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index a62a6400d97..5582778825c 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -134,11 +135,13 @@ struct buf_dblwr_t{ ulint b_reserved;/*!< number of slots currently reserved for batch flush. */ os_event_t b_event;/*!< event where threads wait for a - batch flush to end. */ + batch flush to end; + os_event_set() and os_event_reset() + are protected by buf_dblwr_t::mutex */ ulint s_reserved;/*!< number of slots currently reserved for single page flushes. */ os_event_t s_event;/*!< event where threads wait for a - single page flush slot. */ + single page flush slot. Protected by mutex. */ bool* in_use; /*!< flag used to indicate if a slot is in use. Only used for single page flushes. */ diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 6fc6098f3d9..1ed231aa248 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -266,7 +266,6 @@ dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(index->type & DICT_CLUSTERED); @@ -280,7 +279,6 @@ dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(index->type & DICT_UNIQUE); @@ -295,7 +293,6 @@ dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(index->type & DICT_IBUF); @@ -327,7 +324,6 @@ dict_index_is_sec_or_ibuf( { ulint type; - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); type = index->type; @@ -345,7 +341,6 @@ dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols - DATA_N_SYS_COLS); @@ -377,7 +372,6 @@ dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols); @@ -1373,7 +1367,6 @@ dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return((index->type & DICT_CORRUPT) diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 4b88a809520..10a57f620ad 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -133,7 +133,7 @@ allows InnoDB to update_create_info() accordingly. */ + DICT_TF_WIDTH_DATA_DIR) /** A mask of all the known/used bits in table flags */ -#define DICT_TF_BIT_MASK (~(~0 << DICT_TF_BITS)) +#define DICT_TF_BIT_MASK (~(~0U << DICT_TF_BITS)) /** Zero relative shift position of the COMPACT field */ #define DICT_TF_POS_COMPACT 0 diff --git a/storage/innobase/include/dict0stats_bg.h b/storage/innobase/include/dict0stats_bg.h index 82cd2b468b9..1973380d197 100644 --- a/storage/innobase/include/dict0stats_bg.h +++ b/storage/innobase/include/dict0stats_bg.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,7 +33,8 @@ Created Apr 26, 2012 Vasil Dimov #include "os0sync.h" /* os_event_t */ #include "os0thread.h" /* DECLARE_THREAD */ -/** Event to wake up the stats thread */ +/** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() +or shutdown. Not protected by any mutex. */ extern os_event_t dict_stats_event; /*****************************************************************//** diff --git a/storage/innobase/include/dyn0dyn.ic b/storage/innobase/include/dyn0dyn.ic index f18f2e6dff9..13003862638 100644 --- a/storage/innobase/include/dyn0dyn.ic +++ b/storage/innobase/include/dyn0dyn.ic @@ -47,8 +47,6 @@ dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((block->used) & ~DYN_BLOCK_FULL_FLAG); } @@ -76,7 +74,6 @@ dyn_array_create( dyn_array_t* arr) /*!< in/out: memory buffer of size sizeof(dyn_array_t) */ { - ut_ad(arr); #if DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG # error "DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG" #endif @@ -119,7 +116,6 @@ dyn_array_push( dyn_block_t* block; ulint used; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -159,7 +155,6 @@ dyn_array_open( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -195,7 +190,6 @@ dyn_array_close( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); block = dyn_array_get_last_block(arr); @@ -222,7 +216,6 @@ dyn_array_get_element( { const dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); /* Get the first array block */ @@ -260,7 +253,6 @@ dyn_array_get_data_size( const dyn_block_t* block; ulint sum = 0; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 35a4b2496d4..f29d1bf3408 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -206,7 +207,9 @@ struct fil_node_t { ibool open; /*!< TRUE if file open */ os_file_t handle; /*!< OS handle to the file, if file open */ os_event_t sync_event;/*!< Condition event to group and - serialize calls to fsync */ + serialize calls to fsync; + os_event_set() and os_event_reset() + are protected by fil_system_t::mutex */ ibool is_raw_disk;/*!< TRUE if the 'file' is actually a raw device or a raw disk partition */ ulint size; /*!< size of the file in database pages, 0 if diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 099cb8edc14..0097537e4d6 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -61,7 +61,7 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR) /** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 diff --git a/storage/innobase/include/fsp0types.h b/storage/innobase/include/fsp0types.h index 94fd908ab0c..9734f4c8abc 100644 --- a/storage/innobase/include/fsp0types.h +++ b/storage/innobase/include/fsp0types.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 3e2f359bbeb..7aa7055640c 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -366,8 +367,8 @@ extern ulong fts_min_token_size; need a sync to free some memory */ extern bool fts_need_sync; -/** Maximum possible Fulltext word length */ -#define FTS_MAX_WORD_LEN HA_FT_MAXBYTELEN +/** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */ +#define FTS_MAX_WORD_LEN (HA_FT_MAXCHARLEN * 4) /** Maximum possible Fulltext word length (in characters) */ #define FTS_MAX_WORD_LEN_IN_CHAR HA_FT_MAXCHARLEN diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index e495fe72a60..0dad75d8f1b 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -126,7 +126,9 @@ struct fts_sync_t { bool in_progress; /*!< flag whether sync is in progress.*/ bool unlock_cache; /*!< flag whether unlock cache when write fts node */ - os_event_t event; /*!< sync finish event */ + os_event_t event; /*!< sync finish event; + only os_event_set() and os_event_wait() + are used */ }; /** The cache for the FTS system. It is a memory-based inverted index diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 4bfc60690ce..7fbc19086d5 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -918,7 +918,12 @@ struct lock_sys_t{ srv_slot_t* waiting_threads; /*!< Array of user threads suspended while waiting for locks within InnoDB, protected - by the lock_sys->wait_mutex */ + by the lock_sys->wait_mutex; + os_event_set() and + os_event_reset() on + waiting_threads[]->event + are protected by + trx_t::mutex */ srv_slot_t* last_slot; /*!< highest slot ever used in the waiting_threads array, protected by @@ -931,10 +936,11 @@ struct lock_sys_t{ ulint n_lock_max_wait_time; /*!< Max wait time */ - os_event_t timeout_event; /*!< Set to the event that is - created in the lock wait monitor - thread. A value of 0 means the - thread is not active */ + os_event_t timeout_event; /*!< An event waited for by + lock_wait_timeout_thread. + Not protected by a mutex, + but the waits are timed. + Signaled on shutdown only. */ bool timeout_thread_active; /*!< True if the timeout thread is running */ diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index ad9710b1870..44074a61952 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -875,10 +876,8 @@ struct log_t{ be 'flush_or_write'! */ os_event_t no_flush_event; /*!< this event is in the reset state when a flush or a write is running; - a thread should wait for this without - owning the log mutex, but NOTE that - to set or reset this event, the - thread MUST own the log mutex! */ + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ ibool one_flushed; /*!< during a flush, this is first FALSE and becomes TRUE when one log group has been @@ -887,11 +886,9 @@ struct log_t{ flush or write has not yet completed for any log group; e.g., this means that a transaction has been committed - when this is set; a thread should wait - for this without owning the log mutex, - but NOTE that to set or reset this - event, the thread MUST own the log - mutex! */ + when this is set; + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ ulint n_log_ios; /*!< number of log i/os initiated thus far */ ulint n_log_ios_old; /*!< number of log i/o's at the @@ -976,9 +973,9 @@ struct log_t{ ulint archive_buf_size;/*!< size of archive_buf */ byte* archive_buf; /*!< log segment is written to the archive from this buffer */ - os_event_t archiving_on; /*!< if archiving has been stopped, - a thread can wait for this event to - become signaled */ + os_event_t archiving_on; /*!< if archiving has been stopped; + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ /* @} */ #endif /* UNIV_LOG_ARCHIVE */ }; diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index f4c7b4ed882..bf34b6a3e29 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -271,20 +272,12 @@ void recv_sys_var_init(void); /*===================*/ #endif /* !UNIV_HOTBACKUP */ -/*******************************************************************//** -Empties the hash table of stored log records, applying them to appropriate -pages. */ +/** Apply the hash table of stored log records to persistent data pages. +@param[in] last_batch whether the change buffer merge will be + performed as part of the operation */ UNIV_INTERN void -recv_apply_hashed_log_recs( -/*=======================*/ - ibool allow_ibuf); /*!< in: if TRUE, also ibuf operations are - allowed during the application; if FALSE, - no ibuf operations are allowed, and after - the application all file pages are flushed to - disk and invalidated in buffer pool: this - alternative means that no new log records - can be generated during the application */ +recv_apply_hashed_log_recs(bool last_batch); #ifdef UNIV_HOTBACKUP /*******************************************************************//** Applies log records in the hash table to a backup. */ @@ -434,6 +427,8 @@ struct recv_sys_t{ scan find a corrupt log block, or a corrupt log record, or there is a log parsing buffer overflow */ + /** the time when progress was last reported */ + ib_time_t progress_time; #ifdef UNIV_LOG_ARCHIVE log_group_t* archive_group; /*!< in archive recovery: the log group whose @@ -446,6 +441,20 @@ struct recv_sys_t{ addresses in the hash table */ recv_dblwr_t dblwr; + + /** Determine whether redo log recovery progress should be reported. + @param[in] time the current time + @return whether progress should be reported + (the last report was at least 15 seconds ago) */ + bool report(ib_time_t time) + { + if (time - progress_time < 15) { + return false; + } + + progress_time = time; + return true; + } }; /** The recovery system */ diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index c46fcec107e..6b303979b18 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -52,7 +52,6 @@ mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ { - ut_ad(b); return((ulint)(b[0])); } @@ -149,7 +148,6 @@ mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 16) | ((ulint)(b[1]) << 8) | (ulint)(b[2]) @@ -186,7 +184,6 @@ mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 24) | ((ulint)(b[1]) << 16) | ((ulint)(b[2]) << 8) @@ -265,8 +262,6 @@ mach_read_compressed( { ulint flag; - ut_ad(b); - flag = mach_read_from_1(b); if (flag < 0x80UL) { @@ -347,8 +342,6 @@ mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_3(b), mach_read_from_4(b + 3))); } @@ -378,8 +371,6 @@ mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_2(b), mach_read_from_4(b + 2))); } @@ -427,8 +418,6 @@ mach_ull_read_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - n = (ib_uint64_t) mach_read_compressed(b); size = mach_get_compressed_size((ulint) n); @@ -494,8 +483,6 @@ mach_ull_read_much_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - if (*b != (byte)0xFF) { n = 0; size = 0; diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index e7fa1926bc7..bdb4682eafb 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -50,16 +51,6 @@ extern ibool os_has_said_disk_full; /** Flag: enable debug printout for asynchronous i/o */ extern ibool os_aio_print_debug; -/** Number of pending os_file_pread() operations */ -extern ulint os_file_n_pending_preads; -/** Number of pending os_file_pwrite() operations */ -extern ulint os_file_n_pending_pwrites; - -/** Number of pending read operations */ -extern ulint os_n_pending_reads; -/** Number of pending write operations */ -extern ulint os_n_pending_writes; - #ifdef __WIN__ /** We define always WIN_ASYNC_IO, and check at run-time whether @@ -1143,6 +1134,7 @@ UNIV_INTERN void os_aio_simulated_wake_handler_threads(void); /*=======================================*/ +#ifdef _WIN32 /**********************************************************************//** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must @@ -1150,8 +1142,10 @@ call os_aio_simulated_wake_handler_threads later to ensure the threads are not left sleeping! */ UNIV_INTERN void -os_aio_simulated_put_read_threads_to_sleep(void); -/*============================================*/ +os_aio_simulated_put_read_threads_to_sleep(); +#else /* _WIN32 */ +# define os_aio_simulated_put_read_threads_to_sleep() +#endif /* _WIN32 */ #ifdef WIN_ASYNC_IO /**********************************************************************//** diff --git a/storage/innobase/include/os0sync.h b/storage/innobase/include/os0sync.h index 0754210c47a..bb225c53dfe 100644 --- a/storage/innobase/include/os0sync.h +++ b/storage/innobase/include/os0sync.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -36,14 +37,13 @@ Created 9/6/1995 Heikki Tuuri #include "univ.i" #include "ut0lst.h" -#include "sync0types.h" -#if defined __i386__ || defined __x86_64__ || defined _M_IX86 \ - || defined _M_X64 || defined __WIN__ - -#define IB_STRONG_MEMORY_MODEL - -#endif /* __i386__ || __x86_64__ || _M_IX86 || _M_X64 || __WIN__ */ +/** CPU cache line size */ +#ifdef __powerpc__ +#define CACHE_LINE_SIZE 128 +#else +#define CACHE_LINE_SIZE 64 +#endif #ifdef HAVE_WINDOWS_ATOMICS typedef LONG lock_word_t; /*!< On Windows, InterlockedExchange operates @@ -667,10 +667,7 @@ os_atomic_clear(volatile lock_word_t* ptr) # define HAVE_ATOMIC_BUILTINS # define HAVE_ATOMIC_BUILTINS_BYTE - -# ifndef _WIN32 -# define HAVE_ATOMIC_BUILTINS_64 -# endif +# define HAVE_ATOMIC_BUILTINS_64 /**********************************************************//** Atomic compare and exchange of signed integers (both 32 and 64 bit). @@ -892,6 +889,58 @@ for synchronization */ "Memory barrier is not used" #endif + +/** Simple counter aligned to CACHE_LINE_SIZE +@tparam Type the integer type of the counter +@tparam atomic whether to use atomic memory access */ +template <typename Type = ulint, bool atomic = false> +struct MY_ALIGNED(CACHE_LINE_SIZE) simple_counter +{ + /** Increment the counter */ + Type inc() { return add(1); } + /** Decrement the counter */ + Type dec() { return sub(1); } + + /** Add to the counter + @param[in] i amount to be added + @return the value of the counter after adding */ + Type add(Type i) + { + compile_time_assert(!atomic || sizeof(Type) == sizeof(ulint)); + if (atomic) { + /* GCC would perform a type check in this code + also in case the template is instantiated with + simple_counter<Type=not_ulint, atomic=false>. + On Solaris, os_atomic_increment_ulint() maps + to atomic_add_long_nv(), which expects the + parameter to be correctly typed. */ + return os_atomic_increment_ulint( + reinterpret_cast<ulint*>(&m_counter), i); + } else { + return m_counter += i; + } + } + /** Subtract from the counter + @param[in] i amount to be subtracted + @return the value of the counter after adding */ + Type sub(Type i) + { + compile_time_assert(!atomic || sizeof(Type) == sizeof(ulint)); + if (atomic) { + return os_atomic_decrement_ulint(&m_counter, i); + } else { + return m_counter -= i; + } + } + + /** @return the value of the counter (non-atomic access)! */ + operator Type() const { return m_counter; } + +private: + /** The counter */ + Type m_counter; +}; + #ifndef UNIV_NONINL #include "os0sync.ic" #endif diff --git a/storage/innobase/include/os0thread.h b/storage/innobase/include/os0thread.h index 9a1ada8fa0d..54f3d7554bf 100644 --- a/storage/innobase/include/os0thread.h +++ b/storage/innobase/include/os0thread.h @@ -117,14 +117,25 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread); + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach = true) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** Returns the thread identifier of current thread. diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 99e17001c0a..a4d3bd40426 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -160,7 +160,6 @@ page_header_get_offs( { ulint offs; - ut_ad(page); ut_ad((field == PAGE_FREE) || (field == PAGE_LAST_INSERT) || (field == PAGE_HEAP_TOP)); diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 2d90f47eefe..09af5d4159b 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -2,6 +2,7 @@ Copyright (c) 2010, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -541,22 +542,30 @@ on the counters */ /** Increment a monitor counter under mutex protection. Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be incremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_INC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_INC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ - if (MONITOR_IS_ON(monitor)) { \ + if (enabled) { \ mutex_enter(mutex); \ if (++MONITOR_VALUE(monitor) > MONITOR_MAX_VALUE(monitor)) { \ MONITOR_MAX_VALUE(monitor) = MONITOR_VALUE(monitor); \ } \ mutex_exit(mutex); \ } +/** Increment a monitor counter under mutex protection. +Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be incremented by 1 */ +#define MONITOR_MUTEX_INC(mutex, monitor) \ + MONITOR_MUTEX_INC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) /** Decrement a monitor counter under mutex protection. Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be decremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_DEC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_DEC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ if (MONITOR_IS_ON(monitor)) { \ mutex_enter(mutex); \ @@ -565,13 +574,20 @@ Use MONITOR_DEC if appropriate mutex protection already exists. } \ mutex_exit(mutex); \ } +/** Decrement a monitor counter under mutex protection. +Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be decremented by 1 */ +#define MONITOR_MUTEX_DEC(mutex, monitor) \ + MONITOR_MUTEX_DEC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) #if defined HAVE_ATOMIC_BUILTINS_64 /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_increment_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -584,9 +600,10 @@ Use MONITOR_INC if appropriate mutex protection exists. /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_decrement_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -617,14 +634,29 @@ srv_mon_free(void); /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) MONITOR_MUTEX_INC(&monitor_mutex, monitor) +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + MONITOR_MUTEX_INC_LOW(&monitor_mutex, monitor, enabled) /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) MONITOR_MUTEX_DEC(&monitor_mutex, monitor) +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + MONITOR_MUTEX_DEC_LOW(&monitor_mutex, monitor, enabled) #endif /* HAVE_ATOMIC_BUILTINS_64 */ +/** Atomically increment a monitor counter if it is enabled. +Use MONITOR_INC if appropriate mutex protection exists. +@param monitor monitor to be incremented by 1 */ +#define MONITOR_ATOMIC_INC(monitor) \ + MONITOR_ATOMIC_INC_LOW(monitor, MONITOR_IS_ON(monitor)) +/** Atomically decrement a monitor counter if it is enabled. +Use MONITOR_DEC if appropriate mutex protection exists. +@param monitor monitor to be decremented by 1 */ +#define MONITOR_ATOMIC_DEC(monitor) \ + MONITOR_ATOMIC_DEC_LOW(monitor, MONITOR_IS_ON(monitor)) + #define MONITOR_DEC(monitor) \ if (MONITOR_IS_ON(monitor)) { \ MONITOR_VALUE(monitor)--; \ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index b8e1fcbbaf6..11e7a7b9a0a 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -55,11 +55,10 @@ Created 10/10/1995 Heikki Tuuri /* Global counters used inside InnoDB. */ struct srv_stats_t { - typedef ib_counter_t<lsn_t, 1, single_indexer_t> lsn_ctr_1_t; - typedef ib_counter_t<ulint, 1, single_indexer_t> ulint_ctr_1_t; - typedef ib_counter_t<lint, 1, single_indexer_t> lint_ctr_1_t; typedef ib_counter_t<ulint, 64> ulint_ctr_64_t; - typedef ib_counter_t<ib_int64_t, 1, single_indexer_t> ib_int64_ctr_1_t; + typedef simple_counter<lsn_t> lsn_ctr_1_t; + typedef simple_counter<ulint> ulint_ctr_1_t; + typedef simple_counter<ib_int64_t> ib_int64_ctr_1_t; /** Count the amount of data written in total (in bytes) */ ulint_ctr_1_t data_written; @@ -73,8 +72,9 @@ struct srv_stats_t { /** Amount of data written to the log files in bytes */ lsn_ctr_1_t os_log_written; - /** Number of writes being done to the log files */ - lint_ctr_1_t os_log_pending_writes; + /** Number of writes being done to the log files. + Protected by log_sys->write_mutex. */ + ulint_ctr_1_t os_log_pending_writes; /** We increase this counter, when we don't have enough space in the log buffer and have to flush it */ @@ -113,7 +113,7 @@ struct srv_stats_t { ulint_ctr_1_t n_lock_wait_count; /** Number of threads currently waiting on database locks */ - lint_ctr_1_t n_lock_wait_current_count; + simple_counter<ulint, true> n_lock_wait_current_count; /** Number of rows read. */ ulint_ctr_64_t n_rows_read; @@ -145,13 +145,16 @@ extern const char* srv_main_thread_op_info; /** Prefix used by MySQL to indicate pre-5.1 table name encoding */ extern const char srv_mysql50_table_name_prefix[10]; -/* The monitor thread waits on this event. */ +/** Event to signal srv_monitor_thread. Not protected by a mutex. +Set after setting srv_print_innodb_monitor. */ extern os_event_t srv_monitor_event; -/* The error monitor thread waits on this event. */ +/** Event to signal the shutdown of srv_error_monitor_thread. +Not protected by a mutex. */ extern os_event_t srv_error_event; -/** The buffer pool dump/load thread waits on this event. */ +/** Event for waking up buf_dump_thread. Not protected by a mutex. +Set on shutdown or by buf_dump_start() or buf_load_start(). */ extern os_event_t srv_buf_dump_event; /** The buffer pool dump/load file name */ @@ -345,9 +348,6 @@ extern double srv_adaptive_flushing_lwm; extern ulong srv_flushing_avg_loops; extern ulong srv_force_recovery; -#ifndef DBUG_OFF -extern ulong srv_force_recovery_crash; -#endif /* !DBUG_OFF */ extern ulint srv_fast_shutdown; /*!< If this is 1, do not do a purge and index buffer merge. @@ -364,6 +364,7 @@ extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; extern unsigned long long srv_stats_modified_counter; extern my_bool srv_stats_sample_traditional; +extern my_bool srv_stats_include_delete_marked; extern ibool srv_use_doublewrite_buf; extern ulong srv_doublewrite_batch_size; @@ -780,17 +781,12 @@ ulint srv_get_task_queue_length(void); /*===========================*/ -/*********************************************************************//** -Releases threads of the type given from suspension in the thread table. -NOTE! The server mutex has to be reserved by the caller! -@return number of threads released: this may be less than n if not -enough threads were suspended at the moment */ -UNIV_INTERN -ulint -srv_release_threads( -/*================*/ - enum srv_thread_type type, /*!< in: thread type */ - ulint n); /*!< in: number of threads to release */ +/** Ensure that a given number of threads of the type given are running +(or are already terminated). +@param[in] type thread type +@param[in] n number of threads that have to run */ +void +srv_release_threads(enum srv_thread_type type, ulint n); /**********************************************************************//** Check whether any background thread are active. If so print which thread @@ -801,12 +797,10 @@ const char* srv_any_background_threads_are_active(void); /*=======================================*/ -/**********************************************************************//** -Wakeup the purge threads. */ +/** Wake up the purge threads. */ UNIV_INTERN void -srv_purge_wakeup(void); -/*==================*/ +srv_purge_wakeup(); /** Status variables to be passed to MySQL */ struct export_var_t{ diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index 1e13c883800..f72652963c9 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -145,7 +146,10 @@ struct trx_purge_t{ log operation can prevent this by obtaining an s-latch here. It also protects state and running */ - os_event_t event; /*!< State signal event */ + os_event_t event; /*!< State signal event; + os_event_set() and os_event_reset() + are protected by trx_purge_t::latch + X-lock */ ulint n_stop; /*!< Counter to track number stops */ volatile bool running; /*!< true, if purge is active, we check this without the latch too */ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index e6ddf45faba..eff834503ad 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -44,7 +45,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 33 +#define INNODB_VERSION_BUGFIX 35 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; @@ -171,7 +172,7 @@ command. Not tested on Windows. */ #define UNIV_COMPILE_TEST_FUNCS */ -#if defined(HAVE_valgrind)&& defined(HAVE_VALGRIND_MEMCHECK_H) +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif /* HAVE_VALGRIND */ #if 0 @@ -284,22 +285,12 @@ definitions: */ #endif /* !UNIV_MUST_NOT_INLINE */ -#ifdef _WIN32 -#define UNIV_WORD_SIZE 4 -#elif defined(_WIN64) -#define UNIV_WORD_SIZE 8 -#else -/** MySQL config.h generated by GNU autoconf will define SIZEOF_LONG in Posix */ -#define UNIV_WORD_SIZE SIZEOF_LONG -#endif +#define UNIV_WORD_SIZE SIZEOF_SIZE_T /** The following alignment is used in memory allocations in memory heap management to ensure correct alignment for doubles etc. */ #define UNIV_MEM_ALIGNMENT 8 -/** The following alignment is used in aligning lints etc. */ -#define UNIV_WORD_ALIGNMENT UNIV_WORD_SIZE - /* DATABASE VERSION CONTROL ======================== @@ -428,13 +419,12 @@ the word size of the machine, that is on a 32-bit platform 32 bits, and on a macro ULINTPF. */ -#ifdef __WIN__ +#ifdef _WIN32 /* Use the integer types and formatting strings defined in Visual Studio. */ -# define UINT32PF "%I32u" -# define INT64PF "%I64d" -# define UINT64PF "%I64u" -# define UINT64PFx "%016I64x" -# define DBUG_LSN_PF "%llu" +# define UINT32PF "%u" +# define INT64PF "%lld" +# define UINT64PF "%llu" +# define UINT64PFx "%016llx" typedef __int64 ib_int64_t; typedef unsigned __int64 ib_uint64_t; typedef unsigned __int32 ib_uint32_t; @@ -444,13 +434,12 @@ typedef unsigned __int32 ib_uint32_t; # define INT64PF "%" PRId64 # define UINT64PF "%" PRIu64 # define UINT64PFx "%016" PRIx64 -# define DBUG_LSN_PF UINT64PF typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; -# endif /* __WIN__ */ +#endif -# define IB_ID_FMT UINT64PF +#define IB_ID_FMT UINT64PF #ifdef _WIN64 typedef unsigned __int64 ulint; diff --git a/storage/innobase/include/ut0counter.h b/storage/innobase/include/ut0counter.h index 63a133a175d..edc0db3b03d 100644 --- a/storage/innobase/include/ut0counter.h +++ b/storage/innobase/include/ut0counter.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,13 +31,7 @@ Created 2012/04/12 by Sunny Bains #include "univ.i" #include <string.h> #include "os0thread.h" - -/** CPU cache line size */ -#ifdef __powerpc__ -#define CACHE_LINE_SIZE 128 -#else -#define CACHE_LINE_SIZE 64 -#endif +#include "os0sync.h" /** Default number of slots to use in ib_counter_t */ #define IB_N_SLOTS 64 @@ -44,8 +39,6 @@ Created 2012/04/12 by Sunny Bains /** Get the offset into the counter array. */ template <typename Type, int N> struct generic_indexer_t { - /** Default constructor/destructor should be OK. */ - /** @return offset within m_counter */ size_t offset(size_t index) const UNIV_NOTHROW { return(((index % N) + 1) * (CACHE_LINE_SIZE / sizeof(Type))); @@ -58,8 +51,6 @@ struct generic_indexer_t { use the thread id. */ template <typename Type, int N> struct get_sched_indexer_t : public generic_indexer_t<Type, N> { - /** Default constructor/destructor should be OK. */ - /* @return result from sched_getcpu(), the thread id if it fails. */ size_t get_rnd_index() const UNIV_NOTHROW { @@ -76,31 +67,17 @@ struct get_sched_indexer_t : public generic_indexer_t<Type, N> { /** Use the thread id to index into the counter array. */ template <typename Type, int N> struct thread_id_indexer_t : public generic_indexer_t<Type, N> { - /** Default constructor/destructor should are OK. */ - /* @return a random number, currently we use the thread id. Where thread id is represented as a pointer, it may not work as effectively. */ size_t get_rnd_index() const UNIV_NOTHROW { return((lint) os_thread_get_curr_id()); } -}; - -/** For counters wher N=1 */ -template <typename Type, int N=1> -struct single_indexer_t { - /** Default constructor/destructor should are OK. */ - - /** @return offset within m_counter */ - size_t offset(size_t index) const UNIV_NOTHROW { - ut_ad(N == 1); - return((CACHE_LINE_SIZE / sizeof(Type))); - } - /* @return 1 */ - size_t get_rnd_index() const UNIV_NOTHROW { - ut_ad(N == 1); - return(1); + /** @return a random offset to the array */ + size_t get_rnd_offset() const UNIV_NOTHROW + { + return(generic_indexer_t<Type, N>::offset(get_rnd_index())); } }; @@ -112,17 +89,11 @@ template < typename Type, int N = IB_N_SLOTS, template<typename, int> class Indexer = thread_id_indexer_t> -class ib_counter_t { -public: - ib_counter_t() { memset(m_counter, 0x0, sizeof(m_counter)); } - +struct MY_ALIGNED(CACHE_LINE_SIZE) ib_counter_t +{ +#ifdef UNIV_DEBUG ~ib_counter_t() { - ut_ad(validate()); - } - - bool validate() UNIV_NOTHROW { -#ifdef UNIV_DEBUG size_t n = (CACHE_LINE_SIZE / sizeof(Type)); /* Check that we aren't writing outside our defined bounds. */ @@ -131,27 +102,23 @@ public: ut_ad(m_counter[i + j] == 0); } } -#endif /* UNIV_DEBUG */ - return(true); } +#endif /* UNIV_DEBUG */ - /** If you can't use a good index id. Increment by 1. */ + /** Increment the counter by 1. */ void inc() UNIV_NOTHROW { add(1); } - /** If you can't use a good index id. - * @param n - is the amount to increment */ - void add(Type n) UNIV_NOTHROW { - size_t i = m_policy.offset(m_policy.get_rnd_index()); - - ut_ad(i < UT_ARR_SIZE(m_counter)); + /** Increment the counter by 1. + @param[in] index a reasonably thread-unique identifier */ + void inc(size_t index) UNIV_NOTHROW { add(index, 1); } - m_counter[i] += n; - } + /** Add to the counter. + @param[in] n amount to be added */ + void add(Type n) UNIV_NOTHROW { add(m_policy.get_rnd_offset(), n); } - /** Use this if you can use a unique indentifier, saves a - call to get_rnd_index(). - @param i - index into a slot - @param n - amount to increment */ + /** Add to the counter. + @param[in] index a reasonably thread-unique identifier + @param[in] n amount to be added */ void add(size_t index, Type n) UNIV_NOTHROW { size_t i = m_policy.offset(index); @@ -160,31 +127,6 @@ public: m_counter[i] += n; } - /** If you can't use a good index id. Decrement by 1. */ - void dec() UNIV_NOTHROW { sub(1); } - - /** If you can't use a good index id. - * @param - n is the amount to decrement */ - void sub(Type n) UNIV_NOTHROW { - size_t i = m_policy.offset(m_policy.get_rnd_index()); - - ut_ad(i < UT_ARR_SIZE(m_counter)); - - m_counter[i] -= n; - } - - /** Use this if you can use a unique indentifier, saves a - call to get_rnd_index(). - @param i - index into a slot - @param n - amount to decrement */ - void sub(size_t index, Type n) UNIV_NOTHROW { - size_t i = m_policy.offset(index); - - ut_ad(i < UT_ARR_SIZE(m_counter)); - - m_counter[i] -= n; - } - /* @return total value - not 100% accurate, since it is not atomic. */ operator Type() const UNIV_NOTHROW { Type total = 0; diff --git a/storage/innobase/include/ut0wqueue.h b/storage/innobase/include/ut0wqueue.h index 33385ddf2d4..09b2eb717a7 100644 --- a/storage/innobase/include/ut0wqueue.h +++ b/storage/innobase/include/ut0wqueue.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -99,7 +100,9 @@ ib_wqueue_timedwait( struct ib_wqueue_t { ib_mutex_t mutex; /*!< mutex protecting everything */ ib_list_t* items; /*!< work item list */ - os_event_t event; /*!< event we use to signal additions to list */ + os_event_t event; /*!< event we use to signal additions to list; + os_event_set() and os_event_reset() are + protected by ib_wqueue_t::mutex */ }; #endif diff --git a/storage/innobase/lock/lock0wait.cc b/storage/innobase/lock/lock0wait.cc index 4ca048a50d5..5b67fbe72b7 100644 --- a/storage/innobase/lock/lock0wait.cc +++ b/storage/innobase/lock/lock0wait.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -248,6 +249,9 @@ lock_wait_suspend_thread( slot = lock_wait_table_reserve_slot(thr, lock_wait_timeout); + lock_wait_mutex_exit(); + trx_mutex_exit(trx); + if (thr->lock_state == QUE_THR_LOCK_ROW) { srv_stats.n_lock_wait_count.inc(); srv_stats.n_lock_wait_current_count.inc(); @@ -259,19 +263,21 @@ lock_wait_suspend_thread( } } - lock_wait_mutex_exit(); - trx_mutex_exit(trx); - ulint lock_type = ULINT_UNDEFINED; - lock_mutex_enter(); - + /* The wait_lock can be cleared by another thread when the + lock is released. But the wait can only be initiated by the + current thread which owns the transaction. Only acquire the + mutex if the wait_lock is still active. */ if (const lock_t* wait_lock = trx->lock.wait_lock) { - lock_type = lock_get_type_low(wait_lock); + lock_mutex_enter(); + wait_lock = trx->lock.wait_lock; + if (wait_lock) { + lock_type = lock_get_type_low(wait_lock); + } + lock_mutex_exit(); } - lock_mutex_exit(); - had_dict_lock = trx->dict_operation_lock_mode; switch (had_dict_lock) { diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 8f2af1b0454..63aaaf2efa8 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1459,17 +1460,7 @@ log_write_up_to( } loop: -#ifdef UNIV_DEBUG - loop_count++; - - ut_ad(loop_count < 5); - -# if 0 - if (loop_count > 2) { - fprintf(stderr, "Log loop count %lu\n", loop_count); - } -# endif -#endif + ut_ad(++loop_count < 100); mutex_enter(&(log_sys->mutex)); ut_ad(!recv_no_log_write); @@ -1755,7 +1746,7 @@ log_preflush_pool_modified_pages( and we could not make a new checkpoint on the basis of the info on the buffer pool only. */ - recv_apply_hashed_log_recs(TRUE); + recv_apply_hashed_log_recs(true); } success = buf_flush_list(ULINT_MAX, new_oldest, &n_pages); @@ -2094,7 +2085,7 @@ log_checkpoint( ut_ad(!srv_read_only_mode); if (recv_recovery_is_on()) { - recv_apply_hashed_log_recs(TRUE); + recv_apply_hashed_log_recs(true); } if (srv_unix_file_flush_method != SRV_UNIX_NOSYNC) { @@ -2345,6 +2336,11 @@ loop: start_lsn += len; buf += len; + if (recv_sys->report(ut_time())) { + ib_logf(IB_LOG_LEVEL_INFO, "Read redo log up to LSN=" LSN_PF, + start_lsn); + } + if (start_lsn != end_lsn) { goto loop; @@ -3204,7 +3200,6 @@ logs_empty_and_mark_files_at_shutdown(void) lsn_t lsn; ulint arch_log_no; ulint count = 0; - ulint total_trx; ulint pending_io; enum srv_thread_type active_thd; const char* thread_name; @@ -3212,12 +3207,6 @@ logs_empty_and_mark_files_at_shutdown(void) ib_logf(IB_LOG_LEVEL_INFO, "Starting shutdown..."); - while (srv_fast_shutdown == 0 && trx_rollback_or_clean_is_active) { - /* we should wait until rollback after recovery end - for slow shutdown */ - os_thread_sleep(100000); - } - /* Wait until the master thread and all other operations are idle: our algorithm only works if the server is idle at shutdown */ @@ -3251,10 +3240,9 @@ loop: shutdown, because the InnoDB layer may have committed or prepared transactions and we don't want to lose them. */ - total_trx = trx_sys_any_active_transactions(); - - if (total_trx > 0) { - + if (ulint total_trx = srv_was_started && !srv_read_only_mode + && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + ? trx_sys_any_active_transactions() : 0) { if (srv_print_verbose_log && count > 600) { ib_logf(IB_LOG_LEVEL_INFO, "Waiting for %lu active transactions to finish", @@ -3270,7 +3258,8 @@ loop: active_thd = srv_get_active_thread_type(); - if (active_thd != SRV_NONE) { + if (active_thd != SRV_NONE + || (srv_fast_shutdown != 2 && trx_rollback_or_clean_is_active)) { if (active_thd == SRV_PURGE) { srv_purge_wakeup(); @@ -3286,11 +3275,9 @@ loop: switch (active_thd) { case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; + thread_type = "rollback of" + " recovered transactions"; + break; case SRV_WORKER: thread_type = "worker threads"; break; @@ -3715,7 +3702,6 @@ log_shutdown(void) #ifdef UNIV_LOG_ARCHIVE rw_lock_free(&log_sys->archive_lock); - os_event_create(); #endif /* UNIV_LOG_ARCHIVE */ #ifdef UNIV_LOG_DEBUG diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 85f4f6ea671..ec9c2795d0a 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +73,7 @@ this must be less than UNIV_PAGE_SIZE as it is stored in the buffer pool */ #define RECV_READ_AHEAD_AREA 32 /** The recovery system */ -UNIV_INTERN recv_sys_t* recv_sys = NULL; +UNIV_INTERN recv_sys_t* recv_sys; /** TRUE when applying redo log records during crash recovery; FALSE otherwise. Note that this is FALSE while a background thread is rolling back incomplete transactions. */ @@ -128,9 +129,6 @@ UNIV_INTERN ibool recv_is_making_a_backup = FALSE; UNIV_INTERN ibool recv_is_from_backup = FALSE; # define buf_pool_get_curr_size() (5 * 1024 * 1024) #endif /* !UNIV_HOTBACKUP */ -/** The following counter is used to decide when to print info on -log scan */ -static ulint recv_scan_print_counter; /** The type of the previous parsed redo log record */ static ulint recv_previous_parsed_rec_type; @@ -173,7 +171,7 @@ UNIV_INTERN mysql_pfs_key_t recv_writer_mutex_key; # endif /* UNIV_PFS_MUTEX */ /** Flag indicating if recv_writer thread is active. */ -UNIV_INTERN bool recv_writer_thread_active = false; +static volatile bool recv_writer_thread_active; UNIV_INTERN os_thread_t recv_writer_thread_handle = 0; #endif /* !UNIV_HOTBACKUP */ @@ -305,8 +303,6 @@ recv_sys_var_init(void) recv_no_ibuf_operations = FALSE; - recv_scan_print_counter = 0; - recv_previous_parsed_rec_type = 999999; recv_previous_parsed_rec_offset = 0; @@ -343,8 +339,6 @@ DECLARE_THREAD(recv_writer_thread)( os_thread_pf(os_thread_get_curr_id())); #endif /* UNIV_DEBUG_THREAD_CREATION */ - recv_writer_thread_active = true; - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { os_thread_sleep(100000); @@ -387,12 +381,6 @@ recv_sys_init( } #ifndef UNIV_HOTBACKUP - /* Initialize red-black tree for fast insertions into the - flush_list during recovery process. - As this initialization is done while holding the buffer pool - mutex we perform it before acquiring recv_sys->mutex. */ - buf_flush_init_flush_rbt(); - mutex_enter(&(recv_sys->mutex)); recv_sys->heap = mem_heap_create_typed(256, @@ -425,6 +413,7 @@ recv_sys_init( recv_sys->last_block_buf_start, OS_FILE_LOG_BLOCK_SIZE)); recv_sys->found_corrupt_log = FALSE; + recv_sys->progress_time = ut_time(); recv_max_page_lsn = 0; @@ -1594,6 +1583,7 @@ recv_recover_page_func( ibool success; #endif /* !UNIV_HOTBACKUP */ mtr_t mtr; + ib_time_t time; mutex_enter(&(recv_sys->mutex)); @@ -1712,7 +1702,7 @@ recv_recover_page_func( } DBUG_PRINT("ib_log", - ("apply " DBUG_LSN_PF ": %u len %u " + ("apply " LSN_PF ": %u len %u " "page %u:%u", recv->start_lsn, (unsigned) recv->type, (unsigned) recv->len, @@ -1769,6 +1759,8 @@ recv_recover_page_func( mtr_commit(&mtr); + time = ut_time(); + mutex_enter(&(recv_sys->mutex)); if (recv_max_page_lsn < page_lsn) { @@ -1777,11 +1769,16 @@ recv_recover_page_func( recv_addr->state = RECV_PROCESSED; - ut_a(recv_sys->n_addrs); - recv_sys->n_addrs--; - - mutex_exit(&(recv_sys->mutex)); + ut_a(recv_sys->n_addrs > 0); + if (--recv_sys->n_addrs && recv_sys->progress_time - time >= 15) { + recv_sys->progress_time = time; + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: To recover: " ULINTPF " pages from log\n", + recv_sys->n_addrs); + } + mutex_exit(&recv_sys->mutex); } #ifndef UNIV_HOTBACKUP @@ -1827,59 +1824,48 @@ recv_read_in_area( } buf_read_recv_pages(FALSE, space, zip_size, page_nos, n); - /* - fprintf(stderr, "Recv pages at %lu n %lu\n", page_nos[0], n); - */ return(n); } -/*******************************************************************//** -Empties the hash table of stored log records, applying them to appropriate -pages. */ +/** Apply the hash table of stored log records to persistent data pages. +@param[in] last_batch whether the change buffer merge will be + performed as part of the operation */ UNIV_INTERN void -recv_apply_hashed_log_recs( -/*=======================*/ - ibool allow_ibuf) /*!< in: if TRUE, also ibuf operations are - allowed during the application; if FALSE, - no ibuf operations are allowed, and after - the application all file pages are flushed to - disk and invalidated in buffer pool: this - alternative means that no new log records - can be generated during the application; - the caller must in this case own the log - mutex */ +recv_apply_hashed_log_recs(bool last_batch) { - recv_addr_t* recv_addr; - ulint i; - ibool has_printed = FALSE; - mtr_t mtr; -loop: - mutex_enter(&(recv_sys->mutex)); - - if (recv_sys->apply_batch_on) { + for (;;) { + mutex_enter(&recv_sys->mutex); - mutex_exit(&(recv_sys->mutex)); + if (!recv_sys->apply_batch_on) { + break; + } + mutex_exit(&recv_sys->mutex); os_thread_sleep(500000); - - goto loop; } - ut_ad((!allow_ibuf) == mutex_own(&log_sys->mutex)); + ut_ad(!last_batch == mutex_own(&log_sys->mutex)); - if (!allow_ibuf) { + if (!last_batch) { recv_no_ibuf_operations = TRUE; } + if (ulint n = recv_sys->n_addrs) { + const char* msg = last_batch + ? "Starting final batch to recover " + : "Starting a batch to recover "; + ib_logf(IB_LOG_LEVEL_INFO, + "%s" ULINTPF " pages from redo log", msg, n); + } + recv_sys->apply_log_recs = TRUE; recv_sys->apply_batch_on = TRUE; - for (i = 0; i < hash_get_n_cells(recv_sys->addr_hash); i++) { - - for (recv_addr = static_cast<recv_addr_t*>( - HASH_GET_FIRST(recv_sys->addr_hash, i)); - recv_addr != 0; + for (ulint i = 0; i < hash_get_n_cells(recv_sys->addr_hash); i++) { + for (recv_addr_t* recv_addr = static_cast<recv_addr_t*>( + HASH_GET_FIRST(recv_sys->addr_hash, i)); + recv_addr; recv_addr = static_cast<recv_addr_t*>( HASH_GET_NEXT(addr_hash, recv_addr))) { @@ -1888,24 +1874,12 @@ loop: ulint page_no = recv_addr->page_no; if (recv_addr->state == RECV_NOT_PROCESSED) { - if (!has_printed) { - ib_logf(IB_LOG_LEVEL_INFO, - "Starting an apply batch" - " of log records" - " to the database..."); - fputs("InnoDB: Progress in percent: ", - stderr); - has_printed = TRUE; - } - - mutex_exit(&(recv_sys->mutex)); + mutex_exit(&recv_sys->mutex); if (buf_page_peek(space, page_no)) { - buf_block_t* block; - + mtr_t mtr; mtr_start(&mtr); - - block = buf_page_get( + buf_block_t* block = buf_page_get( space, zip_size, page_no, RW_X_LATCH, &mtr); buf_block_dbg_add_level( @@ -1918,19 +1892,9 @@ loop: page_no); } - mutex_enter(&(recv_sys->mutex)); + mutex_enter(&recv_sys->mutex); } } - - if (has_printed - && (i * 100) / hash_get_n_cells(recv_sys->addr_hash) - != ((i + 1) * 100) - / hash_get_n_cells(recv_sys->addr_hash)) { - - fprintf(stderr, "%lu ", (ulong) - ((i * 100) - / hash_get_n_cells(recv_sys->addr_hash))); - } } /* Wait until all the pages have been processed */ @@ -1944,12 +1908,7 @@ loop: mutex_enter(&(recv_sys->mutex)); } - if (has_printed) { - - fprintf(stderr, "\n"); - } - - if (!allow_ibuf) { + if (!last_batch) { bool success; /* Flush all the file pages to disk and invalidate them in @@ -1989,11 +1948,7 @@ loop: recv_sys_empty_hash(); - if (has_printed) { - fprintf(stderr, "InnoDB: Apply batch completed\n"); - } - - mutex_exit(&(recv_sys->mutex)); + mutex_exit(&recv_sys->mutex); } #else /* !UNIV_HOTBACKUP */ /*******************************************************************//** @@ -2016,11 +1971,6 @@ recv_apply_log_recs_for_backup(void) block = back_block1; - ib_logf(IB_LOG_LEVEL_INFO, - "Starting an apply batch of log records to the database..."); - - fputs("InnoDB: Progress in percent: ", stderr); - n_hash_cells = hash_get_n_cells(recv_sys->addr_hash); for (i = 0; i < n_hash_cells; i++) { @@ -2132,13 +2082,6 @@ recv_apply_log_recs_for_backup(void) skip_this_recv_addr: recv_addr = HASH_GET_NEXT(addr_hash, recv_addr); } - - if ((100 * i) / n_hash_cells - != (100 * (i + 1)) / n_hash_cells) { - fprintf(stderr, "%lu ", - (ulong) ((100 * i) / n_hash_cells)); - fflush(stderr); - } } recv_sys_empty_hash(); @@ -2407,7 +2350,7 @@ loop: recv_sys->recovered_lsn = new_recovered_lsn; DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": log rec %u len %u " + ("scan " LSN_PF ": log rec %u len %u " "page %u:%u", old_lsn, (unsigned) type, (unsigned) len, (unsigned) space, (unsigned) page_no)); @@ -2498,7 +2441,7 @@ loop: #endif /* UNIV_LOG_DEBUG */ DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": multi-log rec %u " + ("scan " LSN_PF ": multi-log rec %u " "len %u page %u:%u", recv_sys->recovered_lsn, (unsigned) type, (unsigned) len, @@ -2802,20 +2745,18 @@ recv_scan_log_recs( #ifndef UNIV_HOTBACKUP if (recv_log_scan_is_startup_type && !recv_needed_recovery) { - if (!srv_read_only_mode) { ib_logf(IB_LOG_LEVEL_INFO, - "Log scan progressed past the " - "checkpoint lsn " LSN_PF "", + "Starting crash recovery from " + "checkpoint LSN=" LSN_PF, recv_sys->scanned_lsn); recv_init_crash_recovery(); } else { - - ib_logf(IB_LOG_LEVEL_WARN, - "Recovery skipped, " - "--innodb-read-only set!"); - + ib_logf(IB_LOG_LEVEL_ERROR, + "innodb_read_only prevents" + " crash recovery"); + recv_needed_recovery = TRUE; return(TRUE); } } @@ -2866,19 +2807,6 @@ recv_scan_log_recs( *group_scanned_lsn = scanned_lsn; - if (recv_needed_recovery - || (recv_is_from_backup && !recv_is_making_a_backup)) { - recv_scan_print_counter++; - - if (finished || (recv_scan_print_counter % 80 == 0)) { - - fprintf(stderr, - "InnoDB: Doing recovery: scanned up to" - " log sequence number " LSN_PF "\n", - *group_scanned_lsn); - } - } - if (more_data && !recv_sys->found_corrupt_log) { /* Try to parse more log records */ @@ -2894,7 +2822,7 @@ recv_scan_log_recs( log yet: they would be produced by ibuf operations */ - recv_apply_hashed_log_recs(FALSE); + recv_apply_hashed_log_recs(false); } #endif /* !UNIV_HOTBACKUP */ @@ -2970,11 +2898,6 @@ recv_init_crash_recovery(void) recv_needed_recovery = TRUE; - ib_logf(IB_LOG_LEVEL_INFO, "Database was not shutdown normally!"); - ib_logf(IB_LOG_LEVEL_INFO, "Starting crash recovery."); - ib_logf(IB_LOG_LEVEL_INFO, - "Reading tablespace information from the .ibd files..."); - fil_load_single_table_tablespaces(); /* If we are using the doublewrite method, we will @@ -2985,15 +2908,14 @@ recv_init_crash_recovery(void) if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { ib_logf(IB_LOG_LEVEL_INFO, - "Restoring possible half-written data pages "); - - ib_logf(IB_LOG_LEVEL_INFO, + "Restoring possible half-written data pages " "from the doublewrite buffer..."); buf_dblwr_process(); /* Spawn the background thread to flush dirty pages from the buffer pools. */ + recv_writer_thread_active = true; recv_writer_thread_handle = os_thread_create( recv_writer_thread, 0, 0); } @@ -3030,6 +2952,11 @@ recv_recovery_from_checkpoint_start_func( byte* buf; byte log_hdr_buf[LOG_FILE_HDR_SIZE]; dberr_t err; + + /* Initialize red-black tree for fast insertions into the + flush_list during recovery process. */ + buf_flush_init_flush_rbt(); + ut_when_dtor<recv_dblwr_t> tmp(recv_sys->dblwr); #ifdef UNIV_LOG_ARCHIVE @@ -3227,6 +3154,11 @@ recv_recovery_from_checkpoint_start_func( /* Done with startup scan. Clear the flag. */ recv_log_scan_is_startup_type = FALSE; + + if (srv_read_only_mode && recv_needed_recovery) { + return(DB_READ_ONLY); + } + if (TYPE_CHECKPOINT) { /* NOTE: we always do a 'recovery' at startup, but only if there is something wrong we will print a message to the @@ -3377,15 +3309,6 @@ void recv_recovery_from_checkpoint_finish(void) /*======================================*/ { - /* Apply the hashed log records to the respective file pages */ - - if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - - recv_apply_hashed_log_recs(TRUE); - } - - DBUG_PRINT("ib_log", ("apply completed")); - if (recv_needed_recovery) { trx_sys_print_mysql_master_log_pos(); trx_sys_print_mysql_binlog_offset(); @@ -3956,7 +3879,7 @@ recv_recovery_from_archive_start( if (limit_lsn != LSN_MAX) { - recv_apply_hashed_log_recs(FALSE); + recv_apply_hashed_log_recs(false); recv_reset_logs(0, FALSE, recv_sys->recovered_lsn); } diff --git a/storage/innobase/mach/mach0data.cc b/storage/innobase/mach/mach0data.cc index df68aab8a18..feeedb01609 100644 --- a/storage/innobase/mach/mach0data.cc +++ b/storage/innobase/mach/mach0data.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,8 +55,22 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xC0UL) { + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } @@ -64,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xE0UL) { + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -73,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -81,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index f60ec648ba1..6e87671817e 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -309,7 +309,6 @@ mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 2b4bbb79d24..d8cb44217bb 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -193,11 +194,15 @@ struct os_aio_array_t{ os_event_t not_full; /*!< The event which is set to the signaled state when there is space in - the aio outside the ibuf segment */ + the aio outside the ibuf segment; + os_event_set() and os_event_reset() + are protected by os_aio_array_t::mutex */ os_event_t is_empty; /*!< The event which is set to the signaled state when there are no - pending i/os in this array */ + pending i/os in this array; + os_event_set() and os_event_reset() + are protected by os_aio_array_t::mutex */ ulint n_slots;/*!< Total number of slots in the aio array. This must be divisible by n_threads. */ @@ -248,8 +253,8 @@ struct os_aio_array_t{ #define OS_AIO_IO_SETUP_RETRY_ATTEMPTS 5 #endif -/** Array of events used in simulated aio */ -static os_event_t* os_aio_segment_wait_events = NULL; +/** Array of events used in simulated aio. */ +static os_event_t* os_aio_segment_wait_events; /** The aio arrays for non-ibuf i/o and ibuf i/o, as well as sync aio. These are NULL when the module has not yet been initialized. @{ */ @@ -279,21 +284,6 @@ UNIV_INTERN time_t os_last_printout; UNIV_INTERN ibool os_has_said_disk_full = FALSE; -#if !defined(UNIV_HOTBACKUP) \ - && (!defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8) -/** The mutex protecting the following counts of pending I/O operations */ -static os_ib_mutex_t os_file_count_mutex; -#endif /* !UNIV_HOTBACKUP && (!HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8) */ - -/** Number of pending os_file_pread() operations */ -UNIV_INTERN ulint os_file_n_pending_preads = 0; -/** Number of pending os_file_pwrite() operations */ -UNIV_INTERN ulint os_file_n_pending_pwrites = 0; -/** Number of pending write operations */ -UNIV_INTERN ulint os_n_pending_writes = 0; -/** Number of pending read operations */ -UNIV_INTERN ulint os_n_pending_reads = 0; - #ifdef UNIV_DEBUG # ifndef UNIV_HOTBACKUP /**********************************************************************//** @@ -747,10 +737,6 @@ void os_io_init_simple(void) /*===================*/ { -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_file_count_mutex = os_mutex_create(); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_file_seek_mutexes[i] = os_mutex_create(); } @@ -2118,48 +2104,52 @@ os_file_set_size( os_file_t file, /*!< in: handle to a file */ os_offset_t size) /*!< in: file size */ { - os_offset_t current_size; ibool ret; byte* buf; byte* buf2; ulint buf_size; - current_size = 0; - #ifdef HAVE_POSIX_FALLOCATE if (srv_use_posix_fallocate) { + int err; + do { + err = posix_fallocate(file, 0, size); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); - if (posix_fallocate(file, current_size, size) == -1) { - - fprintf(stderr, "InnoDB: Error: preallocating file " - "space for file \'%s\' failed. Current size " - "%lu, desired size %lu\n", - name, (long unsigned) current_size, (long unsigned) size); - os_file_handle_error_no_exit(name, "posix_fallocate", FALSE); - return(FALSE); + if (err) { + ib_logf(IB_LOG_LEVEL_ERROR, + "preallocating " INT64PF " bytes for" + "file %s failed with error %d", + size, name, err); } - return(TRUE); + return(!err); } #endif - +#ifdef _WIN32 + /* Write 1 page of zeroes at the desired end. */ + buf_size = UNIV_PAGE_SIZE; + os_offset_t current_size = size - buf_size; +#else /* Write up to 1 megabyte at a time. */ buf_size = ut_min(64, (ulint) (size / UNIV_PAGE_SIZE)) * UNIV_PAGE_SIZE; - buf2 = static_cast<byte*>(ut_malloc(buf_size + UNIV_PAGE_SIZE)); + os_offset_t current_size = 0; +#endif + buf2 = static_cast<byte*>(calloc(1, buf_size + UNIV_PAGE_SIZE)); + + if (!buf2) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot allocate " ULINTPF " bytes to extend file\n", + buf_size + UNIV_PAGE_SIZE); + return(FALSE); + } /* Align the buffer for possible raw i/o */ buf = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); - /* Write buffer full of zeros */ - memset(buf, 0, buf_size); - - if (size >= (os_offset_t) 100 << 20) { - - fprintf(stderr, "InnoDB: Progress in MB:"); - } - - while (current_size < size) { + do { ulint n_bytes; if (size - current_size < (os_offset_t) buf_size) { @@ -2170,37 +2160,15 @@ os_file_set_size( ret = os_file_write(name, file, buf, current_size, n_bytes); if (!ret) { - ut_free(buf2); - goto error_handling; - } - - /* Print about progress for each 100 MB written */ - if ((current_size + n_bytes) / (100 << 20) - != current_size / (100 << 20)) { - - fprintf(stderr, " %lu00", - (ulong) ((current_size + n_bytes) - / (100 << 20))); + break; } current_size += n_bytes; - } - - if (size >= (os_offset_t) 100 << 20) { - - fprintf(stderr, "\n"); - } + } while (current_size < size); - ut_free(buf2); + free(buf2); - ret = os_file_flush(file); - - if (ret) { - return(TRUE); - } - -error_handling: - return(FALSE); + return(ret && os_file_flush(file)); } /***********************************************************************//** @@ -2377,9 +2345,6 @@ os_file_pread( os_offset_t offset) /*!< in: file offset from where to read */ { off_t offs; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) - ssize_t n_bytes; -#endif /* HAVE_PREAD && !HAVE_BROKEN_PREAD */ ut_ad(n); @@ -2396,33 +2361,12 @@ os_file_pread( os_n_file_reads++; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads++; - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ - - n_bytes = pread(file, buf, n, offs); - -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads--; - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD == 8 */ + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); +#ifdef HAVE_PREAD + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); + ssize_t n_bytes = pread(file, buf, n, offs); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); return(n_bytes); #else { @@ -2432,15 +2376,7 @@ os_file_pread( ulint i; #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; @@ -2460,16 +2396,7 @@ os_file_pread( os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - MONITOR_ATOIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD_SIZE == 8 */ - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); return(ret); } #endif @@ -2506,32 +2433,11 @@ os_file_pwrite( os_n_file_writes++; -#if defined(HAVE_PWRITE) && !defined(HAVE_BROKEN_PREAD) -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites++; - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_increment_ulint(&os_n_pending_writes, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ - + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); +#ifdef HAVE_PWRITE + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); ret = pwrite(file, buf, (ssize_t) n, offs); - -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites--; - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_decrement_ulint(&os_n_pending_writes, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); #else @@ -2541,10 +2447,7 @@ os_file_pwrite( ulint i; # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); # ifndef UNIV_HOTBACKUP /* Protect the seek / write operation with a mutex */ @@ -2568,14 +2471,10 @@ func_exit: os_mutex_exit(os_file_seek_mutexes[i]); # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); } -#endif /* !UNIV_HOTBACKUP */ +#endif /* HAVE_PWRITE */ } #endif @@ -2610,6 +2509,7 @@ os_file_read_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); @@ -2618,10 +2518,7 @@ try_again: low = (DWORD) offset & 0xFFFFFFFF; high = (DWORD) (offset >> 32); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ @@ -2639,11 +2536,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); goto error_handling; } @@ -2653,10 +2546,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -2741,6 +2631,7 @@ os_file_read_no_error_handling_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); @@ -2749,10 +2640,7 @@ try_again: low = (DWORD) offset & 0xFFFFFFFF; high = (DWORD) (offset >> 32); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ @@ -2770,11 +2658,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); goto error_handling; } @@ -2784,10 +2668,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -2867,7 +2748,6 @@ os_file_write_func( ulint n) /*!< in: number of bytes to write */ { ut_ad(!srv_read_only_mode); - #ifdef __WIN__ BOOL ret; DWORD len; @@ -2889,14 +2769,12 @@ os_file_write_func( ut_ad(buf); ut_ad(n > 0); + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); retry: low = (DWORD) offset & 0xFFFFFFFF; high = (DWORD) (offset >> 32); - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / write operation with a mutex */ @@ -2914,10 +2792,7 @@ retry: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); ut_print_timestamp(stderr); @@ -2941,10 +2816,7 @@ retry: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); if (ret && len == n) { @@ -4008,6 +3880,12 @@ os_aio_init( os_aio_validate(); + os_last_printout = ut_time(); + + if (srv_use_native_aio) { + return(TRUE); + } + os_aio_segment_wait_events = static_cast<os_event_t*>( ut_malloc(n_segments * sizeof *os_aio_segment_wait_events)); @@ -4015,10 +3893,7 @@ os_aio_init( os_aio_segment_wait_events[i] = os_event_create(); } - os_last_printout = ut_time(); - return(TRUE); - } /*********************************************************************** @@ -4046,8 +3921,10 @@ os_aio_free(void) os_aio_array_free(os_aio_read_array); - for (ulint i = 0; i < os_aio_n_segments; i++) { - os_event_free(os_aio_segment_wait_events[i]); + if (!srv_use_native_aio) { + for (ulint i = 0; i < os_aio_n_segments; i++) { + os_event_free(os_aio_segment_wait_events[i]); + } } ut_free(os_aio_segment_wait_events); @@ -4096,22 +3973,17 @@ os_aio_wake_all_threads_at_shutdown(void) if (os_aio_log_array != 0) { os_aio_array_wake_win_aio_at_shutdown(os_aio_log_array); } - #elif defined(LINUX_NATIVE_AIO) - /* When using native AIO interface the io helper threads wait on io_getevents with a timeout value of 500ms. At each wake up these threads check the server status. No need to do anything to wake them up. */ +#endif /* !WIN_ASYNC_AIO */ if (srv_use_native_aio) { return; } - /* Fall through to simulated AIO handler wakeup if we are - not using native AIO. */ -#endif /* !WIN_ASYNC_AIO */ - /* This loop wakes up all simulated ai/o threads */ for (ulint i = 0; i < os_aio_n_segments; i++) { @@ -4480,6 +4352,7 @@ os_aio_simulated_wake_handler_threads(void) } } +#ifdef _WIN32 /**********************************************************************//** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must @@ -4487,15 +4360,14 @@ call os_aio_simulated_wake_handler_threads later to ensure the threads are not left sleeping! */ UNIV_INTERN void -os_aio_simulated_put_read_threads_to_sleep(void) -/*============================================*/ +os_aio_simulated_put_read_threads_to_sleep() { /* The idea of putting background IO threads to sleep is only for Windows when using simulated AIO. Windows XP seems to schedule background threads too eagerly to allow for coalescing during readahead requests. */ -#ifdef __WIN__ + os_aio_array_t* array; if (srv_use_native_aio) { @@ -4514,8 +4386,8 @@ readahead requests. */ os_event_reset(os_aio_segment_wait_events[i]); } } -#endif /* __WIN__ */ } +#endif /* _WIN32 */ #if defined(LINUX_NATIVE_AIO) /*******************************************************************//** @@ -5725,11 +5597,12 @@ os_aio_print( srv_io_thread_op_info[i], srv_io_thread_function[i]); -#ifndef __WIN__ - if (os_aio_segment_wait_events[i]->is_set) { +#ifndef _WIN32 + if (!srv_use_native_aio + && os_aio_segment_wait_events[i]->is_set) { fprintf(file, " ev set"); } -#endif /* __WIN__ */ +#endif /* _WIN32 */ fprintf(file, "\n"); } @@ -5763,19 +5636,24 @@ os_aio_print( time_elapsed = 0.001 + difftime(current_time, os_last_printout); fprintf(file, - "Pending flushes (fsync) log: %lu; buffer pool: %lu\n" - "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, - (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { + "Pending flushes (fsync) log: " ULINTPF + "; buffer pool: " ULINTPF "\n" + ULINTPF " OS file reads, " + ULINTPF " OS file writes, " + ULINTPF " OS fsyncs\n", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + const ulint n_reads = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); + const ulint n_writes = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); + + if (n_reads != 0 || n_writes != 0) { fprintf(file, - "%lu pending preads, %lu pending pwrites\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + ULINTPF " pending reads, " ULINTPF " pending writes\n", + n_reads, n_writes); } if (os_n_file_reads == os_n_file_reads_old) { diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index a5b0f7de6ae..a6bef176ba4 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -191,14 +191,38 @@ os_thread_create_func( #endif } +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread) +{ +#ifdef __WIN__ + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed. */ + ut_ad(ret == 0 || ret == ESRCH); +#endif /* __WIN__ */ +} + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ { #ifdef UNIV_DEBUG_THREAD_CREATION fprintf(stderr, "Thread exits, id %lu\n", @@ -216,7 +240,9 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - pthread_detach(pthread_self()); + if (detach) { + pthread_detach(pthread_self()); + } pthread_exit(exit_value); #endif } diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 95b8db0ccc8..090848a4525 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -1442,7 +1442,6 @@ page_dir_split_slot( ulint i; ulint n_owned; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); @@ -1504,7 +1503,6 @@ page_dir_balance_slot( rec_t* old_rec; rec_t* new_rec; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index e7d545ddb52..e8a147aab47 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4807,8 +4807,6 @@ page_zip_parse_compress( ulint size; ulint trailer_size; - ut_ad(ptr != NULL); - ut_ad(end_ptr != NULL); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 1ce99acaef8..5b8ed009a2a 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -57,6 +57,8 @@ tokenized doc string. The index has three "fields": integer value) 3) Word's position in original doc. +@see fts_create_one_index_table() + @return dict_index_t structure for the fts sort index */ UNIV_INTERN dict_index_t* @@ -96,16 +98,12 @@ row_merge_create_fts_sort_index( field->prefix_len = 0; field->col = static_cast<dict_col_t*>( mem_heap_alloc(new_index->heap, sizeof(dict_col_t))); - field->col->len = FTS_MAX_WORD_LEN; - - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - field->col->mtype = DATA_VARCHAR; - } else { - field->col->mtype = DATA_VARMYSQL; - } - field->col->prtype = idx_field->col->prtype | DATA_NOT_NULL; + field->col->mtype = charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL; field->col->mbminmaxlen = idx_field->col->mbminmaxlen; + field->col->len = HA_FT_MAXCHARLEN * DATA_MBMAXLEN(field->col->mbminmaxlen); + field->fixed_len = 0; /* Doc ID */ @@ -359,6 +357,7 @@ row_fts_free_pll_merge_buf( /*********************************************************************//** Tokenize incoming text data and add to the sort buffer. +@see row_merge_buf_encode() @return TRUE if the record passed, FALSE if out of space */ static ibool @@ -367,8 +366,6 @@ row_merge_fts_doc_tokenize( row_merge_buf_t** sort_buf, /*!< in/out: sort buffer */ doc_id_t doc_id, /*!< in: Doc ID */ fts_doc_t* doc, /*!< in: Doc to be tokenized */ - dtype_t* word_dtype, /*!< in: data structure for - word col */ merge_file_t** merge_file, /*!< in/out: merge file */ ibool opt_doc_id_size,/*!< in: whether to use 4 bytes instead of 8 bytes integer to @@ -400,7 +397,7 @@ row_merge_fts_doc_tokenize( ulint idx = 0; ib_uint32_t position; ulint offset = 0; - ulint cur_len = 0; + ulint cur_len; doc_id_t write_doc_id; inc = innobase_mysql_fts_get_token( @@ -454,14 +451,34 @@ row_merge_fts_doc_tokenize( dfield_set_data(field, t_str.f_str, t_str.f_len); len = dfield_get_len(field); - field->type.mtype = word_dtype->mtype; - field->type.prtype = word_dtype->prtype | DATA_NOT_NULL; + dict_col_copy_type(dict_index_get_nth_col(buf->index, 0), &field->type); + field->type.prtype |= DATA_NOT_NULL; + ut_ad(len <= field->type.len); - /* Variable length field, set to max size. */ - field->type.len = FTS_MAX_WORD_LEN; - field->type.mbminmaxlen = word_dtype->mbminmaxlen; + /* For the temporary file, row_merge_buf_encode() uses + 1 byte for representing the number of extra_size bytes. + This number will always be 1, because for this 3-field index + consisting of one variable-size column, extra_size will always + be 1 or 2, which can be encoded in one byte. + + The extra_size is 1 byte if the length of the + variable-length column is less than 128 bytes or the + maximum length is less than 256 bytes. */ + + /* One variable length column, word with its lenght less than + fts_max_token_size, add one extra size and one extra byte. + + Since the max length for FTS token now is larger than 255, + so we will need to signify length byte itself, so only 1 to 128 + bytes can be used for 1 bytes, larger than that 2 bytes. */ + if (len < 128 || field->type.len < 256) { + /* Extra size is one byte. */ + cur_len = 2 + len; + } else { + /* Extra size is two bytes. */ + cur_len = 3 + len; + } - cur_len += len; dfield_dup(field, buf->heap); field++; @@ -511,20 +528,6 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* One variable length column, word with its lenght less than - fts_max_token_size, add one extra size and one extra byte. - - Since the max length for FTS token now is larger than 255, - so we will need to signify length byte itself, so only 1 to 128 - bytes can be used for 1 bytes, larger than that 2 bytes. */ - if (t_str.f_len < 128) { - /* Extra size is one byte. */ - cur_len += 2; - } else { - /* Extra size is two bytes. */ - cur_len += 3; - } - /* Reserve one byte for the end marker of row_merge_block_t. */ if (buf->total_size + data_size[idx] + cur_len >= srv_sort_buf_size - 1) { @@ -617,8 +620,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dtype_t word_dtype; - dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; @@ -640,13 +641,6 @@ fts_parallel_tokenization( doc.charset = fts_index_get_charset( psort_info->psort_common->dup->index); - idx_field = dict_index_get_nth_field( - psort_info->psort_common->dup->index, 0); - word_dtype.prtype = idx_field->col->prtype; - word_dtype.mbminmaxlen = idx_field->col->mbminmaxlen; - word_dtype.mtype = (strcmp(doc.charset->name, "latin1_swedish_ci") == 0) - ? DATA_VARCHAR : DATA_VARMYSQL; - block = psort_info->merge_block; zip_size = dict_table_zip_size(table); @@ -696,7 +690,6 @@ loop: processed = row_merge_fts_doc_tokenize( buf, doc_item->doc_id, &doc, - &word_dtype, merge_file, psort_info->psort_common->opt_doc_id_size, &t_ctx); @@ -958,7 +951,7 @@ fts_parallel_merge( CloseHandle(psort_info->thread_hdl); #endif /*__WIN__ */ - os_thread_exit(NULL); + os_thread_exit(NULL, false); OS_THREAD_DUMMY_RETURN; } @@ -1255,10 +1248,9 @@ row_fts_build_sel_tree_level( int child_left; int child_right; ulint i; - ulint num_item; + ulint num_item = ulint(1) << level; - start = static_cast<ulint>((1 << level) - 1); - num_item = static_cast<ulint>(1 << level); + start = num_item - 1; for (i = 0; i < num_item; i++) { child_left = sel_tree[(start + i) * 2 + 1]; @@ -1327,7 +1319,7 @@ row_fts_build_sel_tree( treelevel++; } - start = (1 << treelevel) - 1; + start = (ulint(1) << treelevel) - 1; for (i = 0; i < (int) fts_sort_pll_degree; i++) { sel_tree[i + start] = i; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 12609c3a65d..da8e1aab2c9 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -573,8 +574,8 @@ AbstractCallback::init( } else if (!is_compressed_table() && m_page_size != UNIV_PAGE_SIZE) { ib_logf(IB_LOG_LEVEL_ERROR, - "Page size %lu of ibd file is not the same " - "as the server page size %lu", + "Page size " ULINTPF " of ibd file is not the same " + "as the server page size " ULINTPF, m_page_size, UNIV_PAGE_SIZE); return(DB_CORRUPTION); @@ -583,8 +584,8 @@ AbstractCallback::init( ib_logf(IB_LOG_LEVEL_ERROR, "File size " UINT64PF " is not a multiple " - "of the page size %lu", - (ib_uint64_t) file_size, (ulong) m_page_size); + "of the page size " ULINTPF, + (ib_uint64_t) file_size, m_page_size); return(DB_CORRUPTION); } @@ -725,8 +726,8 @@ FetchIndexRootPages::operator() ( if (block->page.offset * m_page_size != offset) { ib_logf(IB_LOG_LEVEL_ERROR, "Page offset doesn't match file offset: " - "page offset: %lu, file offset: %lu", - (ulint) block->page.offset, + "page offset: %u, file offset: " ULINTPF, + block->page.offset, (ulint) (offset / m_page_size)); err = DB_CORRUPTION; @@ -1159,10 +1160,9 @@ row_import::match_index_columns( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Index field count %lu doesn't match" - " tablespace metadata file value %lu", - (ulong) index->n_fields, - (ulong) cfg_index->m_n_fields); + "Index field count %u doesn't match" + " tablespace metadata file value " ULINTPF, + index->n_fields, cfg_index->m_n_fields); return(DB_ERROR); } @@ -1179,34 +1179,31 @@ row_import::match_index_columns( ER_TABLE_SCHEMA_MISMATCH, "Index field name %s doesn't match" " tablespace metadata field name %s" - " for field position %lu", - field->name, cfg_field->name, (ulong) i); + " for field position " ULINTPF, + field->name, cfg_field->name, i); err = DB_ERROR; } if (cfg_field->prefix_len != field->prefix_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s prefix len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->prefix_len, - (ulong) cfg_field->prefix_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s prefix len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->prefix_len, cfg_field->prefix_len); err = DB_ERROR; } if (cfg_field->fixed_len != field->fixed_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s fixed len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->fixed_len, - (ulong) cfg_field->fixed_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s fixed len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->fixed_len, + cfg_field->fixed_len); err = DB_ERROR; } @@ -1248,12 +1245,11 @@ row_import::match_table_columns( } else if (cfg_col_index != col->ind) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Column %s ordinal value mismatch, it's at " - "%lu in the table and %lu in the tablespace " - "meta-data file", - col_name, - (ulong) col->ind, (ulong) cfg_col_index); + ER_TABLE_SCHEMA_MISMATCH, + "Column %s ordinal value mismatch, it's at %u" + " in the table and " ULINTPF + " in the tablespace meta-data file", + col_name, col->ind, cfg_col_index); err = DB_ERROR; } else { @@ -1336,19 +1332,19 @@ row_import::match_schema( { /* Do some simple checks. */ - if (m_flags != m_table->flags) { + if ((m_table->flags ^ m_flags) & ~DICT_TF_MASK_DATA_DIR) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Table flags don't match, server table has 0x%lx " - "and the meta-data file has 0x%lx", - (ulong) m_table->n_cols, (ulong) m_flags); + "Table flags don't match, server table has 0x%x" + " and the meta-data file has 0x%lx", + m_table->flags, ulong(m_flags)); return(DB_ERROR); } else if (m_table->n_cols != m_n_cols) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of columns don't match, table has %lu " - "columns but the tablespace meta-data file has " - "%lu columns", - (ulong) m_table->n_cols, (ulong) m_n_cols); + "Number of columns don't match, table has %u" + " columns but the tablespace meta-data file has " + ULINTPF " columns", + m_table->n_cols, m_n_cols); return(DB_ERROR); } else if (UT_LIST_GET_LEN(m_table->indexes) != m_n_indexes) { @@ -1358,11 +1354,10 @@ row_import::match_schema( table matching the IMPORT definition. */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of indexes don't match, table has %lu " - "indexes but the tablespace meta-data file has " - "%lu indexes", - (ulong) UT_LIST_GET_LEN(m_table->indexes), - (ulong) m_n_indexes); + "Number of indexes don't match, table has " ULINTPF + " indexes but the tablespace meta-data file has " + ULINTPF " indexes", + UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); return(DB_ERROR); } @@ -1438,8 +1433,8 @@ row_import::set_root_by_heuristic() UNIV_NOTHROW table_name, sizeof(table_name), m_table->name, FALSE); ib_logf(IB_LOG_LEVEL_WARN, - "Table %s should have %lu indexes but the tablespace " - "has %lu indexes", + "Table %s should have " ULINTPF + " indexes but the tablespace has " ULINTPF " indexes", table_name, UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); @@ -1677,9 +1672,10 @@ PageConverter::adjust_cluster_index_blob_column( ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Externally stored column(%lu) has a reference " - "length of %lu in the cluster index %s", - (ulong) i, (ulong) len, index_name); + "Externally stored column(" ULINTPF + ") has a reference length of " ULINTPF + " in the cluster index %s", + i, len, index_name); return(DB_CORRUPTION); } @@ -2040,7 +2036,8 @@ PageConverter::update_page( return(err); } - ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (%lu)", page_type); + ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (" ULINTPF ")", + page_type); return(DB_CORRUPTION); } @@ -2074,7 +2071,8 @@ PageConverter::validate( if (checksum != 0) { /* Checksum check passed in buf_page_is_corrupted(). */ ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu checksum %lu should be zero.", + "%s: Page %lu checksum " ULINTPF + " should be zero.", m_filepath, (ulong) (offset / m_page_size), checksum); } @@ -2388,11 +2386,10 @@ row_import_adjust_root_pages_of_secondary_indexes( ib_errf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_INNODB_INDEX_CORRUPT, - "Index '%s' contains %lu entries, " - "should be %lu, you should recreate " + "Index '%s' contains " ULINTPF " entries, " + "should be " ULINTPF ", you should recreate " "this index.", index_name, - (ulong) purge.get_n_rows(), - (ulong) n_rows_in_table); + purge.get_n_rows(), n_rows_in_table); index->type |= DICT_CORRUPT; @@ -2743,7 +2740,7 @@ row_import_read_index_data( if (len > OS_FILE_MAX_PATH) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Index name length (%lu) is too long, " + "Index name length (" ULINTPF ") is too long, " "the meta-data is corrupt", len); return(DB_CORRUPTION); @@ -2824,8 +2821,8 @@ row_import_read_indexes( } else if (cfg->m_n_indexes > 1024) { // FIXME: What is the upper limit? */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Number of indexes in meta-data file is too high: %lu", - (ulong) cfg->m_n_indexes); + "Number of indexes in meta-data file is too high: " + ULINTPF, cfg->m_n_indexes); cfg->m_n_indexes = 0; return(DB_CORRUPTION); @@ -2923,8 +2920,8 @@ row_import_read_columns( if (len == 0 || len > 128) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Column name length %lu, is invalid", - (ulong) len); + "Column name length " ULINTPF ", is invalid", + len); return(DB_CORRUPTION); } @@ -3095,8 +3092,9 @@ row_import_read_v1( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, "Tablespace to be imported has a different " "page size than this server. Server page size " - "is %lu, whereas tablespace page size is %lu", - UNIV_PAGE_SIZE, (ulong) cfg->m_page_size); + "is " ULINTPF ", whereas tablespace page size is " + ULINTPF, + UNIV_PAGE_SIZE, cfg->m_page_size); return(DB_ERROR); } @@ -3161,8 +3159,8 @@ row_import_read_meta_data( return(row_import_read_v1(file, thd, &cfg)); default: ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Unsupported meta-data version number (%lu), " - "file ignored", (ulong) cfg.m_version); + "Unsupported meta-data version number (" ULINTPF "), " + "file ignored", cfg.m_version); } return(DB_ERROR); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 9cd6e63e031..33345c2e6f2 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -95,7 +96,8 @@ row_merge_tuple_print( } ut_print_buf(f, dfield_get_data(field), len); if (len != field_len) { - fprintf(f, " (total %lu bytes)", field_len); + fprintf(f, " (total " ULINTPF " bytes)", + field_len); } } } @@ -781,9 +783,9 @@ row_merge_buf_write( ut_ad(b < &block[srv_sort_buf_size]); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu %lu", - (void*) b, of->fd, (ulong) of->offset, - (ulong) i); + fprintf(stderr, "row_merge_buf_write %p,%d," + ULINTPF " " ULINTPF, + (void*) b, of->fd, of->offset, i); row_merge_tuple_print(stderr, entry, n_fields); } #endif /* UNIV_DEBUG */ @@ -800,8 +802,8 @@ row_merge_buf_write( #endif /* UNIV_DEBUG_VALGRIND */ #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu EOF\n", - (void*) b, of->fd, (ulong) of->offset); + fprintf(stderr, "row_merge_buf_write %p,%d," ULINTPF " EOF\n", + (void*) b, of->fd, of->offset); } #endif /* UNIV_DEBUG */ } @@ -857,15 +859,8 @@ row_merge_read( #ifdef UNIV_DEBUG if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); - } -#endif /* UNIV_DEBUG */ - -#ifdef UNIV_DEBUG - if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_read fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -908,8 +903,8 @@ row_merge_write( #ifdef UNIV_DEBUG if (row_merge_print_block_write) { - fprintf(stderr, "row_merge_write fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_write fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -944,14 +939,8 @@ row_merge_read_rec( ulint data_size; ulint avail_size; - ut_ad(block); - ut_ad(buf); ut_ad(b >= &block[0]); ut_ad(b < &block[srv_sort_buf_size]); - ut_ad(index); - ut_ad(foffs); - ut_ad(mrec); - ut_ad(offsets); ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); @@ -963,9 +952,10 @@ row_merge_read_rec( *mrec = NULL; #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu EOF\n", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF + " EOF\n", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); } #endif /* UNIV_DEBUG */ return(NULL); @@ -1080,9 +1070,9 @@ err_exit: func_exit: #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu ", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF " ", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); rec_print_comp(stderr, *mrec, offsets); putc('\n', stderr); } @@ -1116,8 +1106,8 @@ row_merge_write_rec_low( ut_ad(e == rec_offs_extra_size(offsets) + 1); if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%d,%lu ", - (void*) b, fd, (ulong) foffs); + fprintf(stderr, "row_merge_write %p,%d," ULINTPF " ", + (void*) b, fd, foffs); rec_print_comp(stderr, mrec, offsets); putc('\n', stderr); } @@ -1219,8 +1209,8 @@ row_merge_write_eof( ut_ad(foffs); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%p,%d,%lu EOF\n", - (void*) b, (void*) block, fd, (ulong) *foffs); + fprintf(stderr, "row_merge_write %p,%p,%d," ULINTPF " EOF\n", + (void*) b, (void*) block, fd, *foffs); } #endif /* UNIV_DEBUG */ @@ -2059,11 +2049,12 @@ row_merge_blocks( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks fd=%d ofs=%lu + fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) *foffs0, - file->fd, (ulong) *foffs1, - of->fd, (ulong) of->offset); + "row_merge_blocks fd=%d ofs=" ULINTPF + " + fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + file->fd, *foffs1, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ @@ -2162,10 +2153,10 @@ row_merge_blocks_copy( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks_copy fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) foffs0, - of->fd, (ulong) of->offset); + "row_merge_blocks_copy fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ @@ -3824,6 +3815,13 @@ wait_again: " exited when creating FTS" " index '%s'", indexes[i]->name); + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 07b841de300..a3f4355ec6c 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1365,6 +1366,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { @@ -1449,9 +1452,9 @@ error_exit: que_thr_stop_for_mysql_no_error(thr, trx); if (table->is_system_db) { - srv_stats.n_system_rows_inserted.add((size_t)trx->id, 1); + srv_stats.n_system_rows_inserted.inc(size_t(trx->id)); } else { - srv_stats.n_rows_inserted.add((size_t)trx->id, 1); + srv_stats.n_rows_inserted.inc(size_t(trx->id)); } /* Not protected by dict_table_stats_lock() for performance @@ -1845,18 +1848,16 @@ run_again: dict_table_n_rows_dec(prebuilt->table); if (table->is_system_db) { - srv_stats.n_system_rows_deleted.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_deleted.inc(size_t(trx->id)); } else { - srv_stats.n_rows_deleted.add((size_t)trx->id, 1); + srv_stats.n_rows_deleted.inc(size_t(trx->id)); } } else { if (table->is_system_db) { - srv_stats.n_system_rows_updated.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_updated.inc(size_t(trx->id)); } else { - srv_stats.n_rows_updated.add((size_t)trx->id, 1); + srv_stats.n_rows_updated.inc(size_t(trx->id)); } } @@ -2089,17 +2090,15 @@ run_again: dict_table_n_rows_dec(table); if (table->is_system_db) { - srv_stats.n_system_rows_deleted.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_deleted.inc(size_t(trx->id)); } else { - srv_stats.n_rows_deleted.add((size_t)trx->id, 1); + srv_stats.n_rows_deleted.inc(size_t(trx->id)); } } else { if (table->is_system_db) { - srv_stats.n_system_rows_updated.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_updated.inc(size_t(trx->id)); } else { - srv_stats.n_rows_updated.add((size_t)trx->id, 1); + srv_stats.n_rows_updated.inc(size_t(trx->id)); } } diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 7f4cf1a39dd..7151801783c 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1114,8 +1114,6 @@ row_upd_index_replace_new_col_vals_index_pos( ulint n_fields; const ulint zip_size = dict_table_zip_size(index->table); - ut_ad(index); - dtuple_set_info_bits(entry, update->info_bits); if (order_only) { @@ -1300,8 +1298,6 @@ row_upd_changes_ord_field_binary_func( ulint i; const dict_index_t* clust_index; - ut_ad(index); - ut_ad(update); ut_ad(thr); ut_ad(thr->graph); ut_ad(thr->graph->trx); diff --git a/storage/innobase/srv/srv0conc.cc b/storage/innobase/srv/srv0conc.cc index dc3c0b1dd88..3eda0700a15 100644 --- a/storage/innobase/srv/srv0conc.cc +++ b/storage/innobase/srv/srv0conc.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -77,7 +78,9 @@ typedef UT_LIST_NODE_T(struct srv_conc_slot_t) srv_conc_node_t; /** Slot for a thread waiting in the concurrency control queue. */ struct srv_conc_slot_t{ - os_event_t event; /*!< event to wait */ + os_event_t event; /*!< event to wait for; + os_event_set() and os_event_reset() + are protected by srv_conc_mutex */ ibool reserved; /*!< TRUE if slot reserved */ ibool wait_ended; /*!< TRUE when another thread has @@ -339,11 +342,11 @@ srv_conc_exit_innodb_without_atomics( } } - os_fast_mutex_unlock(&srv_conc_mutex); - if (slot != NULL) { os_event_set(slot->event); } + + os_fast_mutex_unlock(&srv_conc_mutex); } /*********************************************************************//** diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 4a709160ea6..ea21a4c1454 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -2,6 +2,7 @@ Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -643,11 +644,11 @@ static monitor_info_t innodb_counter_info[] = MONITOR_DEFAULT_START, MONITOR_OVLD_OS_FSYNC}, {"os_pending_reads", "os", "Number of reads pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_READS}, {"os_pending_writes", "os", "Number of writes pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_WRITES}, {"os_log_bytes_written", "os", diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e9eeda9acb1..067e32a83aa 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -313,11 +313,6 @@ starting from SRV_FORCE_IGNORE_CORRUPT, so that data can be recovered by SELECT or mysqldump. When this is nonzero, we do not allow any user modifications to the data. */ UNIV_INTERN ulong srv_force_recovery; -#ifndef DBUG_OFF -/** Inject a crash at different steps of the recovery process. -This is for testing and debugging only. */ -UNIV_INTERN ulong srv_force_recovery_crash; -#endif /* !DBUG_OFF */ /** Print all user-level transactions deadlocks to mysqld stderr */ @@ -343,6 +338,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ UNIV_INTERN unsigned long long srv_stats_transient_sample_pages = 8; UNIV_INTERN my_bool srv_stats_persistent = TRUE; +UNIV_INTERN my_bool srv_stats_include_delete_marked = FALSE; UNIV_INTERN unsigned long long srv_stats_persistent_sample_pages = 20; UNIV_INTERN my_bool srv_stats_auto_recalc = TRUE; @@ -415,7 +411,7 @@ UNIV_INTERN const char* srv_io_thread_function[SRV_MAX_N_IO_THREADS]; UNIV_INTERN time_t srv_last_monitor_time; -UNIV_INTERN ib_mutex_t srv_innodb_monitor_mutex; +static ib_mutex_t srv_innodb_monitor_mutex; /* Mutex for locking srv_monitor_file. Not created if srv_read_only_mode */ UNIV_INTERN ib_mutex_t srv_monitor_file_mutex; @@ -595,7 +591,11 @@ struct srv_sys_t{ ulint n_sys_threads; /*!< size of the sys_threads array */ - srv_slot_t* sys_threads; /*!< server thread table */ + srv_slot_t* sys_threads; /*!< server thread table; + os_event_set() and + os_event_reset() on + sys_threads[]->event are + covered by srv_sys_t::mutex */ ulint n_threads_active[SRV_MASTER + 1]; /*!< number of threads active @@ -613,13 +613,16 @@ UNIV_INTERN ib_mutex_t server_mutex; static srv_sys_t* srv_sys = NULL; -/** Event to signal the monitor thread. */ +/** Event to signal srv_monitor_thread. Not protected by a mutex. +Set after setting srv_print_innodb_monitor. */ UNIV_INTERN os_event_t srv_monitor_event; -/** Event to signal the error thread */ +/** Event to signal the shutdown of srv_error_monitor_thread. +Not protected by a mutex. */ UNIV_INTERN os_event_t srv_error_event; -/** Event to signal the buffer pool dump/load thread */ +/** Event for waking up buf_dump_thread. Not protected by a mutex. +Set on shutdown or by buf_dump_start() or buf_load_start(). */ UNIV_INTERN os_event_t srv_buf_dump_event; /** The buffer pool dump/load file name */ @@ -780,7 +783,6 @@ srv_suspend_thread_low( /*===================*/ srv_slot_t* slot) /*!< in/out: thread slot */ { - ut_ad(!srv_read_only_mode); ut_ad(srv_sys_mutex_own()); @@ -838,34 +840,71 @@ srv_suspend_thread( return(sig_count); } -/*********************************************************************//** -Releases threads of the type given from suspension in the thread table. -NOTE! The server mutex has to be reserved by the caller! -@return number of threads released: this may be less than n if not - enough threads were suspended at the moment. */ -UNIV_INTERN -ulint -srv_release_threads( -/*================*/ - srv_thread_type type, /*!< in: thread type */ - ulint n) /*!< in: number of threads to release */ +/** Resume the calling thread. +@param[in,out] slot thread slot +@param[in] sig_count signal count (if wait) +@param[in] wait whether to wait for the event +@param[in] timeout_usec timeout in microseconds (0=infinite) +@return whether the wait timed out */ +static +bool +srv_resume_thread(srv_slot_t* slot, ib_int64_t sig_count = 0, bool wait = true, + ulint timeout_usec = 0) +{ + bool timeout; + + ut_ad(!srv_read_only_mode); + ut_ad(slot->in_use); + ut_ad(slot->suspended); + + if (!wait) { + timeout = false; + } else if (timeout_usec) { + timeout = OS_SYNC_TIME_EXCEEDED == os_event_wait_time_low( + slot->event, timeout_usec, sig_count); + } else { + timeout = false; + os_event_wait_low(slot->event, sig_count); + } + + srv_sys_mutex_enter(); + ut_ad(slot->in_use); + ut_ad(slot->suspended); + + slot->suspended = FALSE; + ++srv_sys->n_threads_active[slot->type]; + srv_sys_mutex_exit(); + return(timeout); +} + +/** Ensure that a given number of threads of the type given are running +(or are already terminated). +@param[in] type thread type +@param[in] n number of threads that have to run */ +void +srv_release_threads(enum srv_thread_type type, ulint n) { - ulint i; - ulint count = 0; + ulint running; ut_ad(srv_thread_type_validate(type)); ut_ad(n > 0); - srv_sys_mutex_enter(); + do { + running = 0; - for (i = 0; i < srv_sys->n_sys_threads; i++) { - srv_slot_t* slot; + srv_sys_mutex_enter(); - slot = &srv_sys->sys_threads[i]; + for (ulint i = 0; i < srv_sys->n_sys_threads; i++) { + srv_slot_t* slot = &srv_sys->sys_threads[i]; - if (slot->in_use - && srv_slot_get_type(slot) == type - && slot->suspended) { + if (!slot->in_use || srv_slot_get_type(slot) != type) { + continue; + } else if (!slot->suspended) { + if (++running >= n) { + break; + } + continue; + } switch (type) { case SRV_NONE: @@ -895,21 +934,11 @@ srv_release_threads( break; } - slot->suspended = FALSE; - - ++srv_sys->n_threads_active[type]; - os_event_set(slot->event); - - if (++count == n) { - break; - } } - } - - srv_sys_mutex_exit(); - return(count); + srv_sys_mutex_exit(); + } while (running && running < n); } /*********************************************************************//** @@ -922,11 +951,8 @@ srv_free_slot( { srv_sys_mutex_enter(); - if (!slot->suspended) { - /* Mark the thread as inactive. */ - srv_suspend_thread_low(slot); - } - + /* Mark the thread as inactive. */ + srv_suspend_thread_low(slot); /* Free the slot for reuse. */ ut_ad(slot->in_use); slot->in_use = FALSE; @@ -1383,10 +1409,10 @@ srv_export_innodb_status(void) mutex_enter(&srv_innodb_monitor_mutex); export_vars.innodb_data_pending_reads = - os_n_pending_reads; + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); export_vars.innodb_data_pending_writes = - os_n_pending_writes; + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); export_vars.innodb_data_pending_fsyncs = fil_n_pending_log_flushes @@ -1964,15 +1990,7 @@ srv_active_wake_master_thread(void) if (slot->in_use) { ut_a(srv_slot_get_type(slot) == SRV_MASTER); - - if (slot->suspended) { - - slot->suspended = FALSE; - - ++srv_sys->n_threads_active[SRV_MASTER]; - - os_event_set(slot->event); - } + os_event_set(slot->event); } srv_sys_mutex_exit(); @@ -2438,7 +2456,7 @@ suspend_thread: manual also mentions this string in several places. */ srv_main_thread_op_info = "waiting for server activity"; - os_event_wait(slot->event); + srv_resume_thread(slot); if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) { os_thread_exit(NULL); @@ -2550,8 +2568,7 @@ DECLARE_THREAD(srv_worker_thread)( do { srv_suspend_thread(slot); - - os_event_wait(slot->event); + srv_resume_thread(slot); if (srv_task_execute()) { @@ -2687,8 +2704,6 @@ srv_purge_coordinator_suspend( ib_int64_t sig_count = srv_suspend_thread(slot); do { - ulint ret; - rw_lock_x_lock(&purge_sys->latch); purge_sys->running = false; @@ -2697,32 +2712,11 @@ srv_purge_coordinator_suspend( /* We don't wait right away on the the non-timed wait because we want to signal the thread that wants to suspend purge. */ - - if (stop) { - os_event_wait_low(slot->event, sig_count); - ret = 0; - } else if (rseg_history_len <= trx_sys->rseg_history_len) { - ret = os_event_wait_time_low( - slot->event, SRV_PURGE_MAX_TIMEOUT, sig_count); - } else { - /* We don't want to waste time waiting, if the - history list increased by the time we got here, - unless purge has been stopped. */ - ret = 0; - } - - srv_sys_mutex_enter(); - - /* The thread can be in state !suspended after the timeout - but before this check if another thread sent a wakeup signal. */ - - if (slot->suspended) { - slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; - ut_a(srv_sys->n_threads_active[slot->type] == 1); - } - - srv_sys_mutex_exit(); + const bool wait = stop + || rseg_history_len <= trx_sys->rseg_history_len; + const bool timeout = srv_resume_thread( + slot, sig_count, wait, + stop ? 0 : SRV_PURGE_MAX_TIMEOUT); sig_count = srv_suspend_thread(slot); @@ -2734,6 +2728,19 @@ srv_purge_coordinator_suspend( if (!stop) { ut_a(purge_sys->n_stop == 0); purge_sys->running = true; + + if (timeout + && rseg_history_len == trx_sys->rseg_history_len + && trx_sys->rseg_history_len < 5000) { + /* No new records were added since the + wait started. Simply wait for new + records. The magic number 5000 is an + approximation for the case where we + have cached UNDO log records which + prevent truncate of the UNDO + segments. */ + stop = true; + } } else { ut_a(purge_sys->n_stop > 0); @@ -2742,33 +2749,9 @@ srv_purge_coordinator_suspend( } rw_lock_x_unlock(&purge_sys->latch); - - if (ret == OS_SYNC_TIME_EXCEEDED) { - - /* No new records added since wait started then simply - wait for new records. The magic number 5000 is an - approximation for the case where we have cached UNDO - log records which prevent truncate of the UNDO - segments. */ - - if (rseg_history_len == trx_sys->rseg_history_len - && trx_sys->rseg_history_len < 5000) { - - stop = true; - } - } - } while (stop); - srv_sys_mutex_enter(); - - if (slot->suspended) { - slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; - ut_a(srv_sys->n_threads_active[slot->type] == 1); - } - - srv_sys_mutex_exit(); + srv_resume_thread(slot, 0, false); } /*********************************************************************//** @@ -2821,8 +2804,9 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( srv_purge_coordinator_suspend(slot, rseg_history_len); } + ut_ad(!slot->suspended); + if (srv_purge_should_exit(n_total_purged)) { - ut_a(!slot->suspended); break; } @@ -2933,12 +2917,10 @@ srv_get_task_queue_length(void) return(n_tasks); } -/**********************************************************************//** -Wakeup the purge threads. */ +/** Wake up the purge threads. */ UNIV_INTERN void -srv_purge_wakeup(void) -/*==================*/ +srv_purge_wakeup() { ut_ad(!srv_read_only_mode); @@ -2953,4 +2935,3 @@ srv_purge_wakeup(void) } } } - diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 5c3bee5e0b7..6f606b25324 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,6 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -578,19 +579,6 @@ create_log_file( /** Initial number of the first redo log file */ #define INIT_LOG_FILE0 (SRV_N_LOG_FILES_MAX + 1) -#ifdef DBUG_OFF -# define RECOVERY_CRASH(x) do {} while(0) -#else -# define RECOVERY_CRASH(x) do { \ - if (srv_force_recovery_crash == x) { \ - fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ - srv_force_recovery_crash); \ - fflush(stderr); \ - exit(3); \ - } \ -} while (0) -#endif - /*********************************************************************//** Creates all log files. @return DB_SUCCESS or error code */ @@ -631,13 +619,14 @@ create_log_files( file should be recoverable. The buffer pool was clean, and we can simply create all log files from the scratch. */ - RECOVERY_CRASH(6); + DBUG_EXECUTE_IF("innodb_log_abort_6", + return(DB_ERROR);); } } ut_ad(!buf_pool_check_no_pending_io()); - RECOVERY_CRASH(7); + DBUG_EXECUTE_IF("innodb_log_abort_7", return(DB_ERROR);); for (unsigned i = 0; i < srv_n_log_files; i++) { sprintf(logfilename + dirnamelen, @@ -650,7 +639,7 @@ create_log_files( } } - RECOVERY_CRASH(8); + DBUG_EXECUTE_IF("innodb_log_abort_8", return(DB_ERROR);); /* We did not create the first log file initially as ib_logfile0, so that crash recovery cannot find it until it @@ -695,10 +684,16 @@ create_log_files( return(DB_SUCCESS); } -/*********************************************************************//** -Renames the first log file. */ +/** Rename the first redo log file. +@param[in,out] logfilename buffer for the log file name +@param[in] dirnamelen length of the directory path +@param[in] lsn FIL_PAGE_FILE_FLUSH_LSN value +@param[in,out] logfile0 name of the first log file +@return error code +@retval DB_SUCCESS on successful operation */ +MY_ATTRIBUTE((warn_unused_result, nonnull)) static -void +dberr_t create_log_files_rename( /*====================*/ char* logfilename, /*!< in/out: buffer for log file name */ @@ -709,6 +704,9 @@ create_log_files_rename( /* If innodb_flush_method=O_DSYNC, we need to explicitly flush the log buffers. */ fil_flush(SRV_LOG_SPACE_FIRST_ID); + + DBUG_EXECUTE_IF("innodb_log_abort_9", return(DB_ERROR);); + /* Close the log files, so that we can rename the first one. */ fil_close_log_files(false); @@ -717,26 +715,28 @@ create_log_files_rename( checkpoint has been created. */ sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); - RECOVERY_CRASH(9); - ib_logf(IB_LOG_LEVEL_INFO, "Renaming log file %s to %s", logfile0, logfilename); mutex_enter(&log_sys->mutex); ut_ad(strlen(logfile0) == 2 + strlen(logfilename)); - ibool success = os_file_rename( - innodb_file_log_key, logfile0, logfilename); - ut_a(success); - - RECOVERY_CRASH(10); + dberr_t err = os_file_rename( + innodb_file_log_key, logfile0, logfilename) + ? DB_SUCCESS : DB_ERROR; /* Replace the first file with ib_logfile0. */ strcpy(logfile0, logfilename); mutex_exit(&log_sys->mutex); - fil_open_log_and_system_tablespace_files(); + DBUG_EXECUTE_IF("innodb_log_abort_10", err = DB_ERROR;); - ib_logf(IB_LOG_LEVEL_WARN, "New log files created, LSN=" LSN_PF, lsn); + if (err == DB_SUCCESS) { + fil_open_log_and_system_tablespace_files(); + ib_logf(IB_LOG_LEVEL_WARN, + "New log files created, LSN=" LSN_PF, lsn); + } + + return(err); } /*********************************************************************//** @@ -1550,8 +1550,6 @@ innobase_start_or_create_for_mysql(void) ulint max_arch_log_no; #endif /* UNIV_LOG_ARCHIVE */ ulint sum_of_new_sizes; - ulint sum_of_data_file_sizes; - ulint tablespace_size_in_header; dberr_t err; unsigned i; ulint srv_n_log_files_found = srv_n_log_files; @@ -1564,6 +1562,19 @@ innobase_start_or_create_for_mysql(void) size_t dirnamelen; bool sys_datafiles_created = false; + /* Check that os_fast_mutexes work as expected */ + os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); + + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); + + os_fast_mutex_unlock(&srv_os_test_mutex); + + os_fast_mutex_lock(&srv_os_test_mutex); + + os_fast_mutex_unlock(&srv_os_test_mutex); + + os_fast_mutex_free(&srv_os_test_mutex); + high_level_read_only = srv_read_only_mode || srv_force_recovery > SRV_FORCE_NO_TRX_UNDO; @@ -1606,22 +1617,7 @@ innobase_start_or_create_for_mysql(void) #endif /* PAGE_ATOMIC_REF_COUNT */ ); - - if (sizeof(ulint) != sizeof(void*)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: size of InnoDB's ulint is %lu, " - "but size of void*\n", (ulong) sizeof(ulint)); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: is %lu. The sizes should be the same " - "so that on a 64-bit\n", - (ulong) sizeof(void*)); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: platforms you can allocate more than 4 GB " - "of memory.\n"); - } + compile_time_assert(sizeof(ulint) == sizeof(void*)); #ifdef UNIV_DEBUG ut_print_timestamp(stderr); @@ -2199,14 +2195,18 @@ innobase_start_or_create_for_mysql(void) dirnamelen, max_flushed_lsn, logfile0); + if (err == DB_SUCCESS) { + err = create_log_files_rename( + logfilename, + dirnamelen, + max_flushed_lsn, + logfile0); + } + if (err != DB_SUCCESS) { return(err); } - create_log_files_rename( - logfilename, dirnamelen, - max_flushed_lsn, logfile0); - /* Suppress the message about crash recovery. */ max_flushed_lsn = min_flushed_lsn @@ -2332,7 +2332,6 @@ files_checked: trx_sys_create(); if (create_new_db) { - ut_a(!srv_read_only_mode); mtr_start(&mtr); @@ -2375,8 +2374,12 @@ files_checked: fil_flush_file_spaces(FIL_TABLESPACE); - create_log_files_rename(logfilename, dirnamelen, - max_flushed_lsn, logfile0); + err = create_log_files_rename(logfilename, dirnamelen, + max_flushed_lsn, logfile0); + + if (err != DB_SUCCESS) { + return(err); + } #ifdef UNIV_LOG_ARCHIVE } else if (srv_archive_recovery) { @@ -2446,24 +2449,89 @@ files_checked: LOG_CHECKPOINT, LSN_MAX, min_flushed_lsn, max_flushed_lsn); + if (err == DB_SUCCESS) { + /* Initialize the change buffer. */ + err = dict_boot(); + } + if (err != DB_SUCCESS) { + return(err); + } - return(DB_ERROR); + if (!srv_read_only_mode) { + if (sum_of_new_sizes > 0) { + /* New data file(s) were added */ + mtr_start(&mtr); + fsp_header_inc_size(0, sum_of_new_sizes, &mtr); + mtr_commit(&mtr); + /* Immediately write the log record about + increased tablespace size to disk, so that it + is durable even if mysqld would crash + quickly */ + log_buffer_flush_to_disk(); + } } - /* Since the insert buffer init is in dict_boot, and the - insert buffer is needed in any disk i/o, first we call - dict_boot(). Note that trx_sys_init_at_db_start() only needs - to access space 0, and the insert buffer at this stage already - works for space 0. */ + const ulint tablespace_size_in_header + = fsp_header_get_tablespace_size(); - err = dict_boot(); +#ifdef UNIV_DEBUG + /* buf_debug_prints = TRUE; */ +#endif /* UNIV_DEBUG */ + ulint sum_of_data_file_sizes = 0; + + for (ulint d = 0; d < srv_n_data_files; d++) { + sum_of_data_file_sizes += srv_data_file_sizes[d]; + } + + /* Compare the system tablespace file size to what is + stored in FSP_SIZE. In open_or_create_data_files() + we already checked that the file sizes match the + innodb_data_file_path specification. */ + if (srv_read_only_mode + || sum_of_data_file_sizes == tablespace_size_in_header) { + /* Do not complain about the size. */ + } else if (!srv_auto_extend_last_data_file + || sum_of_data_file_sizes + < tablespace_size_in_header) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Tablespace size stored in header is " ULINTPF + " pages, but the sum of data file sizes is " + ULINTPF " pages", + tablespace_size_in_header, + sum_of_data_file_sizes); + + if (srv_force_recovery == 0 + && sum_of_data_file_sizes + < tablespace_size_in_header) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot start InnoDB. The tail of" + " the system tablespace is" + " missing. Have you edited" + " innodb_data_file_path in my.cnf" + " in an inappropriate way, removing" + " data files from there?" + " You can set innodb_force_recovery=1" + " in my.cnf to force" + " a startup if you are trying to" + " recover a badly corrupt database."); - if (err != DB_SUCCESS) { - return(err); + return(DB_ERROR); + } } + /* This must precede recv_apply_hashed_log_recs(true). */ ib_bh = trx_sys_init_at_db_start(); + + if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { + /* Apply the hashed log records to the + respective file pages, for the last batch of + recv_group_scan_log_recs(). */ + + recv_apply_hashed_log_recs(true); + DBUG_PRINT("ib_log", ("apply completed")); + } + n_recovered_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list); /* The purge system needs to create the purge view and @@ -2532,7 +2600,8 @@ files_checked: ULINT_MAX, LSN_MAX, NULL); ut_a(success); - RECOVERY_CRASH(1); + DBUG_EXECUTE_IF("innodb_log_abort_1", + return(DB_ERROR);); min_flushed_lsn = max_flushed_lsn = log_get_lsn(); @@ -2547,8 +2616,6 @@ files_checked: buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST); - RECOVERY_CRASH(2); - /* Flush the old log files. */ log_buffer_flush_to_disk(); /* If innodb_flush_method=O_DSYNC, @@ -2563,21 +2630,27 @@ files_checked: ut_d(recv_no_log_write = TRUE); ut_ad(!buf_pool_check_no_pending_io()); - RECOVERY_CRASH(3); + DBUG_EXECUTE_IF("innodb_log_abort_3", + return(DB_ERROR);); /* Stamp the LSN to the data files. */ fil_write_flushed_lsn_to_data_files( max_flushed_lsn, 0); - fil_flush_file_spaces(FIL_TABLESPACE); + DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;); - RECOVERY_CRASH(4); + if (err != DB_SUCCESS) { + return(err); + } + + fil_flush_file_spaces(FIL_TABLESPACE); /* Close and free the redo log files, so that we can replace them. */ fil_close_log_files(true); - RECOVERY_CRASH(5); + DBUG_EXECUTE_IF("innodb_log_abort_5", + return(DB_ERROR);); /* Free the old log file space. */ log_group_close_all(); @@ -2591,12 +2664,15 @@ files_checked: dirnamelen, max_flushed_lsn, logfile0); + if (err == DB_SUCCESS) { + err = create_log_files_rename( + logfilename, dirnamelen, + max_flushed_lsn, logfile0); + } + if (err != DB_SUCCESS) { return(err); } - - create_log_files_rename(logfilename, dirnamelen, - max_flushed_lsn, logfile0); } srv_startup_is_before_trx_rollback_phase = FALSE; @@ -2610,20 +2686,8 @@ files_checked: trx_sys_file_format_tag_init(); } - if (!create_new_db && sum_of_new_sizes > 0) { - /* New data file(s) were added */ - mtr_start(&mtr); - - fsp_header_inc_size(0, sum_of_new_sizes, &mtr); - - mtr_commit(&mtr); - - /* Immediately write the log record about increased tablespace - size to disk, so that it is durable even if mysqld would crash - quickly */ - - log_buffer_flush_to_disk(); - } + ut_ad(err == DB_SUCCESS); + ut_a(sum_of_new_sizes != ULINT_UNDEFINED); #ifdef UNIV_LOG_ARCHIVE /* Archiving is always off under MySQL */ @@ -2766,125 +2830,6 @@ files_checked: buf_flush_page_cleaner_thread_started = true; } -#ifdef UNIV_DEBUG - /* buf_debug_prints = TRUE; */ -#endif /* UNIV_DEBUG */ - sum_of_data_file_sizes = 0; - - for (i = 0; i < srv_n_data_files; i++) { - sum_of_data_file_sizes += srv_data_file_sizes[i]; - } - - tablespace_size_in_header = fsp_header_get_tablespace_size(); - - if (!srv_read_only_mode - && !srv_auto_extend_last_data_file - && sum_of_data_file_sizes != tablespace_size_in_header) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: tablespace size" - " stored in header is %lu pages, but\n", - (ulong) tablespace_size_in_header); - ut_print_timestamp(stderr); - fprintf(stderr, - "InnoDB: the sum of data file sizes is %lu pages\n", - (ulong) sum_of_data_file_sizes); - - if (srv_force_recovery == 0 - && sum_of_data_file_sizes < tablespace_size_in_header) { - /* This is a fatal error, the tail of a tablespace is - missing */ - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot start InnoDB." - " The tail of the system tablespace is\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: missing. Have you edited" - " innodb_data_file_path in my.cnf in an\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: inappropriate way, removing" - " ibdata files from there?\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: You can set innodb_force_recovery=1" - " in my.cnf to force\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: a startup if you are trying" - " to recover a badly corrupt database.\n"); - - return(DB_ERROR); - } - } - - if (!srv_read_only_mode - && srv_auto_extend_last_data_file - && sum_of_data_file_sizes < tablespace_size_in_header) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: tablespace size stored in header" - " is %lu pages, but\n", - (ulong) tablespace_size_in_header); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: the sum of data file sizes" - " is only %lu pages\n", - (ulong) sum_of_data_file_sizes); - - if (srv_force_recovery == 0) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot start InnoDB. The tail of" - " the system tablespace is\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: missing. Have you edited" - " innodb_data_file_path in my.cnf in an\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: inappropriate way, removing" - " ibdata files from there?\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: You can set innodb_force_recovery=1" - " in my.cnf to force\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: a startup if you are trying to" - " recover a badly corrupt database.\n"); - - return(DB_ERROR); - } - } - - /* Check that os_fast_mutexes work as expected */ - os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - - if (0 != os_fast_mutex_trylock(&srv_os_test_mutex)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: pthread_mutex_trylock returns" - " an unexpected value on\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: success! Cannot continue.\n"); - exit(1); - } - - os_fast_mutex_unlock(&srv_os_test_mutex); - - os_fast_mutex_lock(&srv_os_test_mutex); - - os_fast_mutex_unlock(&srv_os_test_mutex); - - os_fast_mutex_free(&srv_os_test_mutex); - if (srv_print_verbose_log) { ib_logf(IB_LOG_LEVEL_INFO, "%s started; log sequence number " LSN_PF "", @@ -3141,7 +3086,7 @@ innobase_shutdown_for_mysql(void) srv_free(); fil_close(); - /* 4. Free the os_conc_mutex and all os_events and os_mutexes */ + /* 4. Free all os_events and os_mutexes */ os_sync_free(); diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc index c7163695a3f..6ffe3cab8c8 100644 --- a/storage/innobase/sync/sync0arr.cc +++ b/storage/innobase/sync/sync0arr.cc @@ -1069,9 +1069,10 @@ sync_array_print_long_waits( now the values of pending calls of these. */ fprintf(stderr, - "InnoDB: Pending preads %lu, pwrites %lu\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + "InnoDB: Pending reads " UINT64PF + ", writes " UINT64PF "\n", + MONITOR_VALUE(MONITOR_OS_PENDING_READS), + MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); srv_print_innodb_monitor = TRUE; os_event_set(srv_monitor_event); diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 2fe40109511..a2165a917b8 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -222,9 +223,6 @@ UNIV_INTERN mysql_pfs_key_t mutex_list_mutex_key; /** Latching order checks start when this is set TRUE */ UNIV_INTERN ibool sync_order_checks_on = FALSE; -/** Number of slots reserved for each OS thread in the sync level array */ -static const ulint SYNC_THREAD_N_LEVELS = 10000; - /** Array for tracking sync levels per thread. */ typedef std::vector<sync_level_t> sync_arr_t; @@ -1235,7 +1233,7 @@ sync_thread_add_level( case SYNC_TRX_UNDO_PAGE: /* Purge is allowed to read in as many UNDO pages as it likes, there was a bogus rule here earlier that forced the caller to - acquire the purge_sys_t::mutex. The purge mutex did not really + acquire the trx_purge_t::mutex. The purge mutex did not really protect anything because it was only ever acquired by the single purge thread. The purge thread can read the UNDO pages without any covering mutex. */ diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index efc600d16b1..f24c8f22277 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -171,10 +172,6 @@ trx_purge_sys_close(void) sess_close(purge_sys->sess); - purge_sys->sess = NULL; - - purge_sys->view = NULL; - rw_lock_free(&purge_sys->latch); mutex_free(&purge_sys->bh_mutex); @@ -183,9 +180,6 @@ trx_purge_sys_close(void) ib_bh_free(purge_sys->ib_bh); os_event_free(purge_sys->event); - - purge_sys->event = NULL; - mem_free(purge_sys); purge_sys = NULL; @@ -1301,20 +1295,16 @@ void trx_purge_stop(void) /*================*/ { - purge_state_t state; - ib_int64_t sig_count = os_event_reset(purge_sys->event); - ut_a(srv_n_purge_threads > 0); rw_lock_x_lock(&purge_sys->latch); - ut_a(purge_sys->state != PURGE_STATE_INIT); - ut_a(purge_sys->state != PURGE_STATE_EXIT); - ut_a(purge_sys->state != PURGE_STATE_DISABLED); + const ib_int64_t sig_count = os_event_reset(purge_sys->event); + const purge_state_t state = purge_sys->state; - ++purge_sys->n_stop; + ut_a(state == PURGE_STATE_RUN || state == PURGE_STATE_STOP); - state = purge_sys->state; + ++purge_sys->n_stop; if (state == PURGE_STATE_RUN) { ib_logf(IB_LOG_LEVEL_INFO, "Stopping purge"); @@ -1327,33 +1317,29 @@ trx_purge_stop(void) purge_sys->state = PURGE_STATE_STOP; - rw_lock_x_unlock(&purge_sys->latch); - if (state != PURGE_STATE_STOP) { - + rw_lock_x_unlock(&purge_sys->latch); /* Wait for purge coordinator to signal that it is suspended. */ os_event_wait_low(purge_sys->event, sig_count); - } else { - bool once = true; - - rw_lock_x_lock(&purge_sys->latch); + } else { + bool once = true; - /* Wait for purge to signal that it has actually stopped. */ - while (purge_sys->running) { + /* Wait for purge to signal that it has actually stopped. */ + while (purge_sys->running) { - if (once) { + if (once) { ib_logf(IB_LOG_LEVEL_INFO, "Waiting for purge to stop"); - once = false; + once = false; } rw_lock_x_unlock(&purge_sys->latch); - os_thread_sleep(10000); + os_thread_sleep(10000); rw_lock_x_lock(&purge_sys->latch); - } + } rw_lock_x_unlock(&purge_sys->latch); } diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index c65d95a9817..8ab9511befc 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -739,9 +740,9 @@ trx_rollback_or_clean_recovered( } if (all) { - fprintf(stderr, - "InnoDB: Starting in background the rollback" - " of uncommitted transactions\n"); + ib_logf(IB_LOG_LEVEL_INFO, + "Starting in background the rollback" + " of recovered transactions"); } /* Note: For XA recovered transactions, we rely on MySQL to @@ -761,6 +762,12 @@ trx_rollback_or_clean_recovered( assert_trx_in_rw_list(trx); + if (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown != 0) { + all = FALSE; + break; + } + /* If this function does a cleanup or rollback then it will release the trx_sys->mutex, therefore we need to reacquire it before retrying the loop. */ @@ -778,10 +785,8 @@ trx_rollback_or_clean_recovered( } while (trx != NULL); if (all) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Rollback of non-prepared" - " transactions completed\n"); + ib_logf(IB_LOG_LEVEL_INFO, + "Rollback of non-prepared transactions completed"); } } diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index e5f03f4b96a..62c9ee3546a 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -1190,7 +1190,9 @@ trx_sys_close(void) ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); /* Only prepared transactions may be left in the system. Free them. */ - ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx); + ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx + || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); while ((trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list)) != NULL) { trx_free_prepared(trx); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 522908397ec..f06d8c33183 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -307,7 +307,11 @@ trx_free_prepared( /*==============*/ trx_t* trx) /*!< in, own: trx object */ { - ut_a(trx_state_eq(trx, TRX_STATE_PREPARED)); + ut_a(trx_state_eq(trx, TRX_STATE_PREPARED) + || (trx_state_eq(trx, TRX_STATE_ACTIVE) + && trx->is_recovered + && (srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO))); ut_a(trx->magic_n == TRX_MAGIC_N); lock_trx_release_locks(trx); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index cdd23726f2e..220589dd9ff 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2011,13 +2012,37 @@ trx_undo_free_prepared( ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS); if (trx->update_undo) { - ut_a(trx->update_undo->state == TRX_UNDO_PREPARED); + switch (trx->update_undo->state) { + case TRX_UNDO_PREPARED: + break; + case TRX_UNDO_ACTIVE: + /* lock_trx_release_locks() assigns + trx->is_recovered=false */ + ut_a(srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + break; + default: + ut_error; + } + UT_LIST_REMOVE(undo_list, trx->rseg->update_undo_list, trx->update_undo); trx_undo_mem_free(trx->update_undo); } if (trx->insert_undo) { - ut_a(trx->insert_undo->state == TRX_UNDO_PREPARED); + switch (trx->insert_undo->state) { + case TRX_UNDO_PREPARED: + break; + case TRX_UNDO_ACTIVE: + /* lock_trx_release_locks() assigns + trx->is_recovered=false */ + ut_a(srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + break; + default: + ut_error; + } + UT_LIST_REMOVE(undo_list, trx->rseg->insert_undo_list, trx->insert_undo); trx_undo_mem_free(trx->insert_undo); diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 14ab6ec7599..01197ddae6d 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -1,6 +1,6 @@ /* Copyright (C) 2004-2008 MySQL AB & MySQL Finland AB & TCX DataKonsult AB Copyright (C) 2008-2009 Sun Microsystems, Inc. - Copyright (c) 2009, 2014, SkySQL Ab. + Copyright (c) 2009, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1284,75 +1284,75 @@ int ha_maria::write_row(uchar * buf) int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) { int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); MARIA_SHARE *share= file->s; const char *old_proc_info; TRN *old_trn= file->trn; - if (!file || !¶m) return HA_ADMIN_INTERNAL_ERROR; + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "check"; - param.db_name= table->s->db.str; - param.table_name= table->alias.c_ptr(); - param.testflag= check_opt->flags | T_CHECK | T_SILENT; - param.stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); + maria_chk_init(param); + param->thd= thd; + param->op_name= "check"; + param->db_name= table->s->db.str; + param->table_name= table->alias.c_ptr(); + param->testflag= check_opt->flags | T_CHECK | T_SILENT; + param->stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); if (!(table->db_stat & HA_READ_ONLY)) - param.testflag |= T_STATISTICS; - param.using_global_keycache= 1; + param->testflag |= T_STATISTICS; + param->using_global_keycache= 1; if (!maria_is_crashed(file) && - (((param.testflag & T_CHECK_ONLY_CHANGED) && + (((param->testflag & T_CHECK_ONLY_CHANGED) && !(share->state.changed & (STATE_CHANGED | STATE_CRASHED_FLAGS | STATE_IN_REPAIR)) && share->state.open_count == 0) || - ((param.testflag & T_FAST) && (share->state.open_count == + ((param->testflag & T_FAST) && (share->state.open_count == (uint) (share->global_changed ? 1 : 0))))) return HA_ADMIN_ALREADY_DONE; - maria_chk_init_for_check(¶m, file); + maria_chk_init_for_check(param, file); if ((file->s->state.changed & (STATE_CRASHED_FLAGS | STATE_MOVED)) == STATE_MOVED) { - _ma_check_print_error(¶m, "%s", zerofill_error_msg); + _ma_check_print_error(param, "%s", zerofill_error_msg); return HA_ADMIN_CORRUPT; } old_proc_info= thd_proc_info(thd, "Checking status"); thd_progress_init(thd, 3); - error= maria_chk_status(¶m, file); // Not fatal - if (maria_chk_size(¶m, file)) + error= maria_chk_status(param, file); // Not fatal + if (maria_chk_size(param, file)) error= 1; if (!error) - error|= maria_chk_del(¶m, file, param.testflag); + error|= maria_chk_del(param, file, param->testflag); thd_proc_info(thd, "Checking keys"); thd_progress_next_stage(thd); if (!error) - error= maria_chk_key(¶m, file); + error= maria_chk_key(param, file); thd_proc_info(thd, "Checking data"); thd_progress_next_stage(thd); if (!error) { - if ((!(param.testflag & T_QUICK) && + if ((!(param->testflag & T_QUICK) && ((share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) || - (param.testflag & (T_EXTEND | T_MEDIUM)))) || maria_is_crashed(file)) + (param->testflag & (T_EXTEND | T_MEDIUM)))) || maria_is_crashed(file)) { - ulonglong old_testflag= param.testflag; - param.testflag |= T_MEDIUM; - if (!(error= init_io_cache(¶m.read_cache, file->dfile.file, + ulonglong old_testflag= param->testflag; + param->testflag |= T_MEDIUM; + if (!(error= init_io_cache(¶m->read_cache, file->dfile.file, my_default_record_cache_size, READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)))) { - error= maria_chk_data_link(¶m, file, - MY_TEST(param.testflag & T_EXTEND)); - end_io_cache(&(param.read_cache)); + error= maria_chk_data_link(param, file, + MY_TEST(param->testflag & T_EXTEND)); + end_io_cache(¶m->read_cache); } - param.testflag= old_testflag; + param->testflag= old_testflag; } } if (!error) @@ -1360,7 +1360,7 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) if ((share->state.changed & (STATE_CHANGED | STATE_CRASHED_FLAGS | STATE_IN_REPAIR | STATE_NOT_ANALYZED)) || - (param.testflag & T_STATISTICS) || maria_is_crashed(file)) + (param->testflag & T_STATISTICS) || maria_is_crashed(file)) { file->update |= HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; mysql_mutex_lock(&share->intern_lock); @@ -1368,7 +1368,7 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED_FLAGS | STATE_IN_REPAIR); if (!(table->db_stat & HA_READ_ONLY)) - error= maria_update_state_info(¶m, file, + error= maria_update_state_info(param, file, UPDATE_TIME | UPDATE_OPEN_COUNT | UPDATE_STAT); mysql_mutex_unlock(&share->intern_lock); @@ -1399,33 +1399,33 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) int ha_maria::analyze(THD *thd, HA_CHECK_OPT * check_opt) { int error= 0; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); MARIA_SHARE *share= file->s; const char *old_proc_info; - if (!¶m) + if (!param) return HA_ADMIN_INTERNAL_ERROR; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "analyze"; - param.db_name= table->s->db.str; - param.table_name= table->alias.c_ptr(); - param.testflag= (T_FAST | T_CHECK | T_SILENT | T_STATISTICS | + maria_chk_init(param); + param->thd= thd; + param->op_name= "analyze"; + param->db_name= table->s->db.str; + param->table_name= table->alias.c_ptr(); + param->testflag= (T_FAST | T_CHECK | T_SILENT | T_STATISTICS | T_DONT_CHECK_CHECKSUM); - param.using_global_keycache= 1; - param.stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); + param->using_global_keycache= 1; + param->stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); if (!(share->state.changed & STATE_NOT_ANALYZED)) return HA_ADMIN_ALREADY_DONE; old_proc_info= thd_proc_info(thd, "Scanning"); thd_progress_init(thd, 1); - error= maria_chk_key(¶m, file); + error= maria_chk_key(param, file); if (!error) { mysql_mutex_lock(&share->intern_lock); - error= maria_update_state_info(¶m, file, UPDATE_STAT); + error= maria_update_state_info(param, file, UPDATE_STAT); mysql_mutex_unlock(&share->intern_lock); } else if (!maria_is_crashed(file) && !thd->killed) @@ -1438,46 +1438,46 @@ int ha_maria::analyze(THD *thd, HA_CHECK_OPT * check_opt) int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) { int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); ha_rows start_records; const char *old_proc_info; - if (!file || !¶m) + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "repair"; - param.testflag= ((check_opt->flags & ~(T_EXTEND)) | + maria_chk_init(param); + param->thd= thd; + param->op_name= "repair"; + param->testflag= ((check_opt->flags & ~(T_EXTEND)) | T_SILENT | T_FORCE_CREATE | T_CALC_CHECKSUM | (check_opt->flags & T_EXTEND ? T_REP : T_REP_BY_SORT)); - param.sort_buffer_length= THDVAR(thd, sort_buffer_size); - param.backup_time= check_opt->start_time; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->backup_time= check_opt->start_time; start_records= file->state->records; old_proc_info= thd_proc_info(thd, "Checking table"); thd_progress_init(thd, 1); - while ((error= repair(thd, ¶m, 0)) && param.retry_repair) + while ((error= repair(thd, param, 0)) && param->retry_repair) { - param.retry_repair= 0; - if (test_all_bits(param.testflag, + param->retry_repair= 0; + if (test_all_bits(param->testflag, (uint) (T_RETRY_WITHOUT_QUICK | T_QUICK))) { - param.testflag&= ~(T_RETRY_WITHOUT_QUICK | T_QUICK); + param->testflag&= ~(T_RETRY_WITHOUT_QUICK | T_QUICK); /* Ensure we don't loose any rows when retrying without quick */ - param.testflag|= T_SAFE_REPAIR; + param->testflag|= T_SAFE_REPAIR; if (thd->vio_ok()) - _ma_check_print_info(¶m, "Retrying repair without quick"); + _ma_check_print_info(param, "Retrying repair without quick"); else sql_print_information("Retrying repair of: '%s' without quick", table->s->path.str); continue; } - param.testflag &= ~T_QUICK; - if ((param.testflag & T_REP_BY_SORT)) + param->testflag &= ~T_QUICK; + if (param->testflag & T_REP_BY_SORT) { - param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP; + param->testflag= (param->testflag & ~T_REP_BY_SORT) | T_REP; if (thd->vio_ok()) - _ma_check_print_info(¶m, "Retrying repair with keycache"); + _ma_check_print_info(param, "Retrying repair with keycache"); sql_print_information("Retrying repair of: '%s' with keycache", table->s->path.str); continue; @@ -1501,20 +1501,20 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) int ha_maria::zerofill(THD * thd, HA_CHECK_OPT *check_opt) { int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); TRN *old_trn; MARIA_SHARE *share= file->s; - if (!file || !¶m) + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; old_trn= file->trn; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "zerofill"; - param.testflag= check_opt->flags | T_SILENT | T_ZEROFILL; - param.sort_buffer_length= THDVAR(thd, sort_buffer_size); - error=maria_zerofill(¶m, file, share->open_file_name.str); + maria_chk_init(param); + param->thd= thd; + param->op_name= "zerofill"; + param->testflag= check_opt->flags | T_SILENT | T_ZEROFILL; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + error=maria_zerofill(param, file, share->open_file_name.str); /* Reset trn, that may have been set by repair */ _ma_set_trn_for_table(file, old_trn); @@ -1524,7 +1524,7 @@ int ha_maria::zerofill(THD * thd, HA_CHECK_OPT *check_opt) TrID create_trid= trnman_get_min_safe_trid(); mysql_mutex_lock(&share->intern_lock); share->state.changed|= STATE_NOT_MOVABLE; - maria_update_state_info(¶m, file, UPDATE_TIME | UPDATE_OPEN_COUNT); + maria_update_state_info(param, file, UPDATE_TIME | UPDATE_OPEN_COUNT); _ma_update_state_lsns_sub(share, LSN_IMPOSSIBLE, create_trid, TRUE, TRUE); mysql_mutex_unlock(&share->intern_lock); @@ -1535,24 +1535,24 @@ int ha_maria::zerofill(THD * thd, HA_CHECK_OPT *check_opt) int ha_maria::optimize(THD * thd, HA_CHECK_OPT *check_opt) { int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); - if (!file || !¶m) + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "optimize"; - param.testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | + maria_chk_init(param); + param->thd= thd; + param->op_name= "optimize"; + param->testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | T_REP_BY_SORT | T_STATISTICS | T_SORT_INDEX); - param.sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); thd_progress_init(thd, 1); - if ((error= repair(thd, ¶m, 1)) && param.retry_repair) + if ((error= repair(thd, param, 1)) && param->retry_repair) { sql_print_warning("Warning: Optimize table got errno %d on %s.%s, retrying", - my_errno, param.db_name, param.table_name); - param.testflag &= ~T_REP_BY_SORT; - error= repair(thd, ¶m, 0); + my_errno, param->db_name, param->table_name); + param->testflag &= ~T_REP_BY_SORT; + error= repair(thd, param, 0); } thd_progress_end(thd); return error; @@ -1802,17 +1802,17 @@ int ha_maria::assign_to_keycache(THD * thd, HA_CHECK_OPT *check_opt) if (error != HA_ADMIN_OK) { /* Send error to user */ - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); - if (!¶m) + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) return HA_ADMIN_INTERNAL_ERROR; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "assign_to_keycache"; - param.db_name= table->s->db.str; - param.table_name= table->s->table_name.str; - param.testflag= 0; - _ma_check_print_error(¶m, errmsg); + maria_chk_init(param); + param->thd= thd; + param->op_name= "assign_to_keycache"; + param->db_name= table->s->db.str; + param->table_name= table->s->table_name.str; + param->testflag= 0; + _ma_check_print_error(param, errmsg); } DBUG_RETURN(error); #else @@ -1866,17 +1866,17 @@ int ha_maria::preload_keys(THD * thd, HA_CHECK_OPT *check_opt) errmsg= buf; } - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); - if (!¶m) + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) return HA_ADMIN_INTERNAL_ERROR; - maria_chk_init(¶m); - param.thd= thd; - param.op_name= "preload_keys"; - param.db_name= table->s->db.str; - param.table_name= table->s->table_name.str; - param.testflag= 0; - _ma_check_print_error(¶m, "%s", errmsg); + maria_chk_init(param); + param->thd= thd; + param->op_name= "preload_keys"; + param->db_name= table->s->db.str; + param->table_name= table->s->table_name.str; + param->testflag= 0; + _ma_check_print_error(param, "%s", errmsg); DBUG_RETURN(HA_ADMIN_FAILED); } DBUG_RETURN(HA_ADMIN_OK); @@ -1977,25 +1977,25 @@ int ha_maria::enable_indexes(uint mode) else if (mode == HA_KEY_SWITCH_NONUNIQ_SAVE) { THD *thd= table->in_use; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); - if (!¶m) + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) return HA_ADMIN_INTERNAL_ERROR; const char *save_proc_info= thd_proc_info(thd, "Creating index"); - maria_chk_init(¶m); - param.op_name= "recreating_index"; - param.testflag= (T_SILENT | T_REP_BY_SORT | T_QUICK | + maria_chk_init(param); + param->op_name= "recreating_index"; + param->testflag= (T_SILENT | T_REP_BY_SORT | T_QUICK | T_CREATE_MISSING_KEYS | T_SAFE_REPAIR); /* Don't lock and unlock table if it's locked. Normally table should be locked. This test is mostly for safety. */ if (likely(file->lock_type != F_UNLCK)) - param.testflag|= T_NO_LOCKS; + param->testflag|= T_NO_LOCKS; if (file->create_unique_index_by_sort) - param.testflag|= T_CREATE_UNIQUE_BY_SORT; + param->testflag|= T_CREATE_UNIQUE_BY_SORT; if (bulk_insert_single_undo == BULK_INSERT_SINGLE_UNDO_AND_NO_REPAIR) { @@ -2004,23 +2004,23 @@ int ha_maria::enable_indexes(uint mode) Don't bump create_rename_lsn, because UNDO_BULK_INSERT should not be skipped in case of crash during repair. */ - param.testflag|= T_NO_CREATE_RENAME_LSN; + param->testflag|= T_NO_CREATE_RENAME_LSN; } - param.myf_rw &= ~MY_WAIT_IF_FULL; - param.sort_buffer_length= THDVAR(thd,sort_buffer_size); - param.stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); - param.tmpdir= &mysql_tmpdir_list; - if ((error= (repair(thd, ¶m, 0) != HA_ADMIN_OK)) && param.retry_repair) + param->myf_rw &= ~MY_WAIT_IF_FULL; + param->sort_buffer_length= THDVAR(thd,sort_buffer_size); + param->stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method); + param->tmpdir= &mysql_tmpdir_list; + if ((error= (repair(thd, param, 0) != HA_ADMIN_OK)) && param->retry_repair) { sql_print_warning("Warning: Enabling keys got errno %d on %s.%s, " "retrying", - my_errno, param.db_name, param.table_name); + my_errno, param->db_name, param->table_name); /* This should never fail normally */ DBUG_ASSERT(thd->killed != 0); /* Repairing by sort failed. Now try standard repair method. */ - param.testflag &= ~T_REP_BY_SORT; - error= (repair(thd, ¶m, 0) != HA_ADMIN_OK); + param->testflag &= ~T_REP_BY_SORT; + error= (repair(thd, param, 0) != HA_ADMIN_OK); /* If the standard repair succeeded, clear all error messages which might have been set by the first repair. They can still be seen @@ -3609,10 +3609,6 @@ static int ha_maria_init(void *p) maria_pagecache->extra_debug= 1; maria_assert_if_crashed_table= debug_assert_if_crashed_table; -#if defined(HAVE_REALPATH) && !defined(HAVE_valgrind) && !defined(HAVE_BROKEN_REALPATH) - /* We can only test for sub paths if my_symlink.c is using realpath */ - maria_test_invalid_symlink= test_if_data_home_dir; -#endif if (res) maria_hton= 0; diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index a189af40362..bd5c67c0409 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -2839,7 +2839,7 @@ int maria_repair(HA_CHECK *param, register MARIA_HA *info, (param->testflag & T_BACKUP_DATA ? MYF(MY_REDEL_MAKE_BACKUP): MYF(0)) | sync_dir) || - _ma_open_datafile(info, share, NullS, -1)) + _ma_open_datafile(info, share)) { goto err; } @@ -3994,7 +3994,7 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, (param->testflag & T_BACKUP_DATA ? MYF(MY_REDEL_MAKE_BACKUP): MYF(0)) | sync_dir) || - _ma_open_datafile(info, share, NullS, -1)) + _ma_open_datafile(info, share)) { _ma_check_print_error(param, "Couldn't change to new data file"); goto err; @@ -4627,7 +4627,7 @@ err: MYF((param->testflag & T_BACKUP_DATA ? MY_REDEL_MAKE_BACKUP : 0) | sync_dir)) || - _ma_open_datafile(info,share, NullS, -1)) + _ma_open_datafile(info,share)) got_error=1; } } diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index 1176b2037b5..14809636616 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -53,7 +53,8 @@ int maria_create(const char *name, enum data_file_type datafile_type, uint max_field_lengths, extra_header_size, column_nr; uint internal_table= flags & HA_CREATE_INTERNAL_TABLE; ulong reclength, real_reclength,min_pack_length; - char filename[FN_REFLEN], linkname[FN_REFLEN], *linkname_ptr; + char kfilename[FN_REFLEN], klinkname[FN_REFLEN], *klinkname_ptr; + char dfilename[FN_REFLEN], dlinkname[FN_REFLEN], *dlinkname_ptr; ulong pack_reclength; ulonglong tot_length,max_rows, tmp; enum en_fieldtype type; @@ -808,19 +809,19 @@ int maria_create(const char *name, enum data_file_type datafile_type, /* chop off the table name, tempory tables use generated name */ if ((path= strrchr(ci->index_file_name, FN_LIBCHAR))) *path= '\0'; - fn_format(filename, name, ci->index_file_name, MARIA_NAME_IEXT, + fn_format(kfilename, name, ci->index_file_name, MARIA_NAME_IEXT, MY_REPLACE_DIR | MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | MY_APPEND_EXT); } else { - fn_format(filename, ci->index_file_name, "", MARIA_NAME_IEXT, + fn_format(kfilename, ci->index_file_name, "", MARIA_NAME_IEXT, MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | (have_iext ? MY_REPLACE_EXT : MY_APPEND_EXT)); } - fn_format(linkname, name, "", MARIA_NAME_IEXT, + fn_format(klinkname, name, "", MARIA_NAME_IEXT, MY_UNPACK_FILENAME|MY_APPEND_EXT); - linkname_ptr= linkname; + klinkname_ptr= klinkname; /* Don't create the table if the link or file exists to ensure that one doesn't accidently destroy another table. @@ -834,10 +835,10 @@ int maria_create(const char *name, enum data_file_type datafile_type, { char *iext= strrchr(name, '.'); int have_iext= iext && !strcmp(iext, MARIA_NAME_IEXT); - fn_format(filename, name, "", MARIA_NAME_IEXT, + fn_format(kfilename, name, "", MARIA_NAME_IEXT, MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | (have_iext ? MY_REPLACE_EXT : MY_APPEND_EXT)); - linkname_ptr= NullS; + klinkname_ptr= NullS; /* Replace the current file. Don't sync dir now if the data file has the same path. @@ -857,7 +858,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, NOTE: The filename is compared against unique_file_name of every open table. Hence we need a real path here. */ - if (!internal_table && _ma_test_if_reopen(filename)) + if (!internal_table && _ma_test_if_reopen(kfilename)) { my_printf_error(HA_ERR_TABLE_EXIST, "Aria table '%s' is in use " "(most likely by a MERGE table). Try FLUSH TABLES.", @@ -866,8 +867,8 @@ int maria_create(const char *name, enum data_file_type datafile_type, goto err; } - if ((file= mysql_file_create_with_symlink(key_file_kfile, linkname_ptr, - filename, 0, create_mode, + if ((file= mysql_file_create_with_symlink(key_file_kfile, klinkname_ptr, + kfilename, 0, create_mode, MYF(MY_WME|create_flag))) < 0) goto err; errpos=1; @@ -1121,30 +1122,30 @@ int maria_create(const char *name, enum data_file_type datafile_type, /* chop off the table name, tempory tables use generated name */ if ((path= strrchr(ci->data_file_name, FN_LIBCHAR))) *path= '\0'; - fn_format(filename, name, ci->data_file_name, MARIA_NAME_DEXT, + fn_format(dfilename, name, ci->data_file_name, MARIA_NAME_DEXT, MY_REPLACE_DIR | MY_UNPACK_FILENAME | MY_APPEND_EXT); } else { - fn_format(filename, ci->data_file_name, "", MARIA_NAME_DEXT, + fn_format(dfilename, ci->data_file_name, "", MARIA_NAME_DEXT, MY_UNPACK_FILENAME | (have_dext ? MY_REPLACE_EXT : MY_APPEND_EXT)); } - fn_format(linkname, name, "",MARIA_NAME_DEXT, + fn_format(dlinkname, name, "",MARIA_NAME_DEXT, MY_UNPACK_FILENAME | MY_APPEND_EXT); - linkname_ptr= linkname; + dlinkname_ptr= dlinkname; create_flag=0; } else { - fn_format(filename,name,"", MARIA_NAME_DEXT, + fn_format(dfilename,name,"", MARIA_NAME_DEXT, MY_UNPACK_FILENAME | MY_APPEND_EXT); - linkname_ptr= NullS; + dlinkname_ptr= NullS; create_flag= (flags & HA_CREATE_KEEP_FILES) ? 0 : MY_DELETE_OLD; } if ((dfile= - mysql_file_create_with_symlink(key_file_dfile, linkname_ptr, - filename, 0, create_mode, + mysql_file_create_with_symlink(key_file_dfile, dlinkname_ptr, + dfilename, 0, create_mode, MYF(MY_WME | create_flag | sync_dir))) < 0) goto err; errpos=3; @@ -1194,19 +1195,21 @@ err_no_lock: mysql_file_close(dfile, MYF(0)); /* fall through */ case 2: - if (! (flags & HA_DONT_TOUCH_DATA)) - mysql_file_delete_with_symlink(key_file_dfile, - fn_format(filename,name,"",MARIA_NAME_DEXT, - MY_UNPACK_FILENAME | MY_APPEND_EXT), - sync_dir); + if (! (flags & HA_DONT_TOUCH_DATA)) + { + mysql_file_delete(key_file_dfile, dfilename, MYF(sync_dir)); + if (dlinkname_ptr) + mysql_file_delete(key_file_dfile, dlinkname_ptr, MYF(sync_dir)); + } /* fall through */ case 1: mysql_file_close(file, MYF(0)); if (! (flags & HA_DONT_TOUCH_DATA)) - mysql_file_delete_with_symlink(key_file_kfile, - fn_format(filename,name,"",MARIA_NAME_IEXT, - MY_UNPACK_FILENAME | MY_APPEND_EXT), - sync_dir); + { + mysql_file_delete(key_file_kfile, kfilename, MYF(sync_dir)); + if (klinkname_ptr) + mysql_file_delete(key_file_kfile, klinkname_ptr, MYF(sync_dir)); + } } my_free(log_data); my_free(rec_per_key_part); diff --git a/storage/maria/ma_delete_table.c b/storage/maria/ma_delete_table.c index 56a6dfc8e5f..186075dac68 100644 --- a/storage/maria/ma_delete_table.c +++ b/storage/maria/ma_delete_table.c @@ -84,22 +84,13 @@ int maria_delete_table(const char *name) int maria_delete_table_files(const char *name, myf sync_dir) { - char from[FN_REFLEN]; DBUG_ENTER("maria_delete_table_files"); - fn_format(from,name,"",MARIA_NAME_IEXT,MY_UNPACK_FILENAME|MY_APPEND_EXT); - if (mysql_file_delete_with_symlink(key_file_kfile, from, - MYF(MY_WME | sync_dir))) - DBUG_RETURN(my_errno); - fn_format(from,name,"",MARIA_NAME_DEXT,MY_UNPACK_FILENAME|MY_APPEND_EXT); - if (mysql_file_delete_with_symlink(key_file_dfile, from, - MYF(MY_WME | sync_dir))) + if (mysql_file_delete_with_symlink(key_file_kfile, name, MARIA_NAME_IEXT, MYF(MY_WME | sync_dir)) || + mysql_file_delete_with_symlink(key_file_dfile, name, MARIA_NAME_DEXT, MYF(MY_WME | sync_dir))) DBUG_RETURN(my_errno); - // optional files from maria_pack: - fn_format(from,name,"",".TMD",MY_UNPACK_FILENAME|MY_APPEND_EXT); - mysql_file_delete_with_symlink(key_file_dfile, from, MYF(0)); - fn_format(from,name,"",".OLD",MY_UNPACK_FILENAME|MY_APPEND_EXT); - mysql_file_delete_with_symlink(key_file_dfile, from, MYF(0)); + mysql_file_delete_with_symlink(key_file_dfile, name, ".TMD", MYF(0)); + mysql_file_delete_with_symlink(key_file_dfile, name, ".OLD", MYF(0)); DBUG_RETURN(0); } diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index 1d274d796be..1db85180fcf 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -86,7 +86,7 @@ MARIA_HA *_ma_test_if_reopen(const char *filename) */ -static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, const char *name, +static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, int mode, File data_file, uint internal_table) { @@ -106,7 +106,7 @@ static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, const char *name, } if (data_file >= 0) info.dfile.file= data_file; - else if (_ma_open_datafile(&info, share, name, -1)) + else if (_ma_open_datafile(&info, share)) goto err; errpos= 5; @@ -252,7 +252,7 @@ MARIA_HA *maria_clone(MARIA_SHARE *share, int mode) { MARIA_HA *new_info; mysql_mutex_lock(&THR_LOCK_maria); - new_info= maria_clone_internal(share, NullS, mode, + new_info= maria_clone_internal(share, mode, share->data_file_type == BLOCK_RECORD ? share->bitmap.file.file : -1, 0); mysql_mutex_unlock(&THR_LOCK_maria); @@ -298,8 +298,13 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) realpath_err= my_realpath(name_buff, fn_format(org_name, name, "", MARIA_NAME_IEXT, MY_UNPACK_FILENAME),MYF(0)); + if (realpath_err > 0) /* File not found, no point in looking further. */ + { + DBUG_RETURN(NULL); + } + if (my_is_symlink(org_name) && - (realpath_err || (*maria_test_invalid_symlink)(name_buff))) + (realpath_err || mysys_test_invalid_symlink(name_buff))) { my_errno= HA_WRONG_CREATE_OPTION; DBUG_RETURN(0); @@ -324,13 +329,16 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) my_errno= HA_ERR_CRASHED; goto err; }); + DEBUG_SYNC_C("mi_open_kfile"); if ((kfile=mysql_file_open(key_file_kfile, name_buff, - (open_mode=O_RDWR) | O_SHARE,MYF(0))) < 0) + (open_mode=O_RDWR) | O_SHARE | O_NOFOLLOW, + MYF(MY_NOSYMLINKS))) < 0) { if ((errno != EROFS && errno != EACCES) || mode != O_RDONLY || (kfile=mysql_file_open(key_file_kfile, name_buff, - (open_mode=O_RDONLY) | O_SHARE,MYF(0))) < 0) + (open_mode=O_RDONLY) | O_SHARE | O_NOFOLLOW, + MYF(MY_NOSYMLINKS))) < 0) goto err; } share->mode=open_mode; @@ -375,7 +383,18 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) (void) strmov(index_name, org_name); *strrchr(org_name, FN_EXTCHAR)= '\0'; (void) fn_format(data_name,org_name,"",MARIA_NAME_DEXT, - MY_APPEND_EXT|MY_UNPACK_FILENAME|MY_RESOLVE_SYMLINKS); + MY_APPEND_EXT|MY_UNPACK_FILENAME); + if (my_is_symlink(data_name)) + { + if (my_realpath(data_name, data_name, MYF(0))) + goto err; + if (mysys_test_invalid_symlink(data_name)) + { + my_errno= HA_WRONG_CREATE_OPTION; + goto err; + } + share->mode|= O_NOFOLLOW; /* all symlinks are resolved by realpath() */ + } info_length=mi_uint2korr(share->state.header.header_length); base_pos= mi_uint2korr(share->state.header.base_pos); @@ -832,7 +851,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) if ((share->data_file_type == BLOCK_RECORD || share->data_file_type == COMPRESSED_RECORD)) { - if (_ma_open_datafile(&info, share, name, -1)) + if (_ma_open_datafile(&info, share)) goto err; data_file= info.dfile.file; } @@ -1004,7 +1023,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) data_file= share->bitmap.file.file; /* Only opened once */ } - if (!(m_info= maria_clone_internal(share, name, mode, data_file, + if (!(m_info= maria_clone_internal(share, mode, data_file, internal_table))) goto err; @@ -1878,35 +1897,15 @@ void _ma_set_index_pagecache_callbacks(PAGECACHE_FILE *file, Open data file We can't use dup() here as the data file descriptors need to have different active seek-positions. - - The argument file_to_dup is here for the future if there would on some OS - exist a dup()-like call that would give us two different file descriptors. *************************************************************************/ -int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share, const char *org_name, - File file_to_dup __attribute__((unused))) +int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share) { - char *data_name= share->data_file_name.str; - char real_data_name[FN_REFLEN]; - - if (org_name) - { - fn_format(real_data_name, org_name, "", MARIA_NAME_DEXT, 4); - if (my_is_symlink(real_data_name)) - { - if (my_realpath(real_data_name, real_data_name, MYF(0)) || - (*maria_test_invalid_symlink)(real_data_name)) - { - my_errno= HA_WRONG_CREATE_OPTION; - return 1; - } - data_name= real_data_name; - } - } - + myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS : 0); + DEBUG_SYNC_C("mi_open_datafile"); info->dfile.file= share->bitmap.file.file= - mysql_file_open(key_file_dfile, data_name, - share->mode | O_SHARE, MYF(MY_WME)); + mysql_file_open(key_file_dfile, share->data_file_name.str, + share->mode | O_SHARE, MYF(flags)); return info->dfile.file >= 0 ? 0 : 1; } @@ -1920,8 +1919,8 @@ int _ma_open_keyfile(MARIA_SHARE *share) mysql_mutex_lock(&share->intern_lock); share->kfile.file= mysql_file_open(key_file_kfile, share->unique_file_name.str, - share->mode | O_SHARE, - MYF(MY_WME)); + share->mode | O_SHARE | O_NOFOLLOW, + MYF(MY_WME | MY_NOSYMLINKS)); mysql_mutex_unlock(&share->intern_lock); return (share->kfile.file < 0); } diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index d3eb687d064..34daf1dcf83 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -493,7 +493,8 @@ error: #define FLUSH_CACHE 2000 /* sort this many blocks at once */ -static void free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block); +static my_bool free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block, + my_bool abort_if_pinned); static void unlink_hash(PAGECACHE *pagecache, PAGECACHE_HASH_LINK *hash_link); #ifndef DBUG_OFF static void test_key_cache(PAGECACHE *pagecache, @@ -883,8 +884,8 @@ size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem, pagecache->waiting_for_hash_link.last_thread= NULL; pagecache->waiting_for_block.last_thread= NULL; DBUG_PRINT("exit", - ("disk_blocks: %ld block_root: 0x%lx hash_entries: %ld\ - hash_root: 0x%lx hash_links: %ld hash_link_root: 0x%lx", + ("disk_blocks: %zu block_root: 0x%lx hash_entries: %zu\ + hash_root: 0x%lx hash_links: %zu hash_link_root: 0x%lx", pagecache->disk_blocks, (long) pagecache->block_root, pagecache->hash_entries, (long) pagecache->hash_root, pagecache->hash_links, (long) pagecache->hash_link_root)); @@ -1182,7 +1183,7 @@ void end_pagecache(PAGECACHE *pagecache, my_bool cleanup) pagecache->blocks_changed= 0; } - DBUG_PRINT("status", ("used: %lu changed: %lu w_requests: %lu " + DBUG_PRINT("status", ("used: %zu changed: %zu w_requests: %lu " "writes: %lu r_requests: %lu reads: %lu", pagecache->blocks_used, pagecache->global_blocks_changed, @@ -1517,7 +1518,7 @@ static void unreg_request(PAGECACHE *pagecache, if (block->temperature == PCBLOCK_WARM) pagecache->warm_blocks--; block->temperature= PCBLOCK_HOT; - KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks: %lu", + KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks: %zu", pagecache->warm_blocks)); } link_block(pagecache, block, hot, (my_bool)at_end); @@ -1536,7 +1537,7 @@ static void unreg_request(PAGECACHE *pagecache, pagecache->warm_blocks++; block->temperature= PCBLOCK_WARM; } - KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks: %lu", + KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks: %zu", pagecache->warm_blocks)); } } @@ -1939,7 +1940,7 @@ restart: removed from the cache as we set the PCBLOCK_REASSIGNED flag (see the code below that handles reading requests). */ - free_block(pagecache, block); + free_block(pagecache, block, 0); return 0; } /* Wait until the page is flushed on disk */ @@ -1950,7 +1951,7 @@ restart: /* Invalidate page in the block if it has not been done yet */ DBUG_ASSERT(block->status); /* Should always be true */ if (block->status) - free_block(pagecache, block); + free_block(pagecache, block, 0); return 0; } @@ -1975,8 +1976,13 @@ restart: } else { - DBUG_ASSERT(hash_link->requests > 0); - hash_link->requests--; + /* + When we come here either PCBLOCK_REASSIGNED or PCBLOCK_IN_SWITCH are + active. In both cases wqueue_release_queue() is called when the + state changes. + */ + DBUG_ASSERT(block->hash_link == hash_link); + remove_reader(block); KEYCACHE_DBUG_PRINT("find_block", ("request waiting for old page to be saved")); { @@ -3635,7 +3641,7 @@ static my_bool pagecache_delete_internal(PAGECACHE *pagecache, DBUG_ASSERT(block->hash_link->requests > 0); page_link->requests--; /* See NOTE for pagecache_unlock() about registering requests. */ - free_block(pagecache, block); + free_block(pagecache, block, 0); dec_counter_for_resize_op(pagecache); return 0; @@ -4227,7 +4233,8 @@ end: and add it to the free list. */ -static void free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block) +static my_bool free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block, + my_bool abort_if_pinned) { uint status= block->status; KEYCACHE_THREAD_TRACE("free block"); @@ -4241,11 +4248,27 @@ static void free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block) /* While waiting for readers to finish, new readers might request the block. But since we set block->status|= PCBLOCK_REASSIGNED, they - will wait on block->wqueue[COND_FOR_SAVED]. They must be signalled + will wait on block->wqueue[COND_FOR_SAVED]. They must be signaled later. */ block->status|= PCBLOCK_REASSIGNED; wait_for_readers(pagecache, block); + if (unlikely(abort_if_pinned) && unlikely(block->pins)) + { + /* + Block got pinned while waiting for readers. + This can only happens when called from flush_pagecache_blocks_int() + when flushing blocks as part of prepare for maria_close() or from + flush_cached_blocks() + */ + block->status&= ~PCBLOCK_REASSIGNED; + unreg_request(pagecache, block, 0); + + /* All pending requests for this page must be resubmitted. */ + if (block->wqueue[COND_FOR_SAVED].last_thread) + wqueue_release_queue(&block->wqueue[COND_FOR_SAVED]); + return 1; + } unlink_hash(pagecache, block->hash_link); } @@ -4296,6 +4319,8 @@ static void free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block) /* All pending requests for this page must be resubmitted. */ if (block->wqueue[COND_FOR_SAVED].last_thread) wqueue_release_queue(&block->wqueue[COND_FOR_SAVED]); + + return 0; } @@ -4431,9 +4456,16 @@ static int flush_cached_blocks(PAGECACHE *pagecache, if (! (type == FLUSH_KEEP || type == FLUSH_KEEP_LAZY || type == FLUSH_FORCE_WRITE)) { - pagecache->blocks_changed--; - pagecache->global_blocks_changed--; - free_block(pagecache, block); + if (!free_block(pagecache, block, 1)) + { + pagecache->blocks_changed--; + pagecache->global_blocks_changed--; + } + else + { + block->status&= ~PCBLOCK_IN_FLUSH; + link_to_file_list(pagecache, block, file, 1); + } } else { @@ -4491,7 +4523,7 @@ static int flush_pagecache_blocks_int(PAGECACHE *pagecache, int rc= PCFLUSH_OK; DBUG_ENTER("flush_pagecache_blocks_int"); DBUG_PRINT("enter", - ("fd: %d blocks_used: %lu blocks_changed: %lu type: %d", + ("fd: %d blocks_used: %zu blocks_changed: %zu type: %d", file->file, pagecache->blocks_used, pagecache->blocks_changed, type)); @@ -4671,7 +4703,7 @@ restart: /* It's a temporary file */ pagecache->blocks_changed--; pagecache->global_blocks_changed--; - free_block(pagecache, block); + free_block(pagecache, block, 0); } } else if (type != FLUSH_KEEP_LAZY) @@ -4741,11 +4773,12 @@ restart: #endif next= block->next_changed; if (block->hash_link->file.file == file->file && + !block->pins && (! (block->status & PCBLOCK_CHANGED) || type == FLUSH_IGNORE_CHANGED)) { reg_requests(pagecache, block, 1); - free_block(pagecache, block); + free_block(pagecache, block, 1); } } } @@ -4956,7 +4989,7 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache, ptr= str->str; int8store(ptr, (ulonglong)stored_list_size); ptr+= 8; - DBUG_PRINT("info", ("found %lu dirty pages", stored_list_size)); + DBUG_PRINT("info", ("found %zu dirty pages", stored_list_size)); if (stored_list_size == 0) goto end; for (file_hash= 0; file_hash < pagecache->changed_blocks_hash_size; file_hash++) diff --git a/storage/maria/ma_static.c b/storage/maria/ma_static.c index 35ad7d5a96a..3e4cf3249a8 100644 --- a/storage/maria/ma_static.c +++ b/storage/maria/ma_static.c @@ -106,12 +106,6 @@ uint32 maria_readnext_vec[]= SEARCH_BIGGER, SEARCH_SMALLER, SEARCH_SMALLER }; -static int always_valid(const char *filename __attribute__((unused))) -{ - return 0; -} - -int (*maria_test_invalid_symlink)(const char *filename)= always_valid; my_bool (*ma_killed)(MARIA_HA *)= ma_killed_standalone; #ifdef HAVE_PSI_INTERFACE diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c index c9d38400bc4..0cdbe28f86b 100644 --- a/storage/maria/maria_chk.c +++ b/storage/maria/maria_chk.c @@ -1275,7 +1275,7 @@ static int maria_chk(HA_CHECK *param, char *filename) mysql_file_close(info->dfile.file, MYF(MY_WME)); /* Close new file */ error|=maria_change_to_newfile(filename,MARIA_NAME_DEXT,DATA_TMP_EXT, 0, MYF(0)); - if (_ma_open_datafile(info,info->s, NullS, -1)) + if (_ma_open_datafile(info, info->s)) error=1; param->out_flag&= ~O_NEW_DATA; /* We are using new datafile */ param->read_cache.file= info->dfile.file; diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index a1fff2e0e43..52d1033ab66 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -1312,8 +1312,7 @@ int _ma_def_scan_restore_pos(MARIA_HA *info, MARIA_RECORD_POS lastpos); extern MARIA_HA *_ma_test_if_reopen(const char *filename); my_bool _ma_check_table_is_closed(const char *name, const char *where); -int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share, const char *org_name, - File file_to_dup); +int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share); int _ma_open_keyfile(MARIA_SHARE *share); void _ma_setup_functions(register MARIA_SHARE *share); my_bool _ma_dynmap_file(MARIA_HA *info, my_off_t size); diff --git a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt index 8959f883ca3..ef3e13e236d 100644 --- a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt @@ -22,8 +22,14 @@ include_directories( ${ONIGMO_INCLUDE_DIRS} ${MRUBY_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS}) -link_directories( - ${LIBLZ4_LIBRARY_DIRS}) +if (LIBLZ4_LIBRARY_DIRS) + find_library(LZ4_LIBS + NAMES ${LIBLZ4_LIBRARIES} + PATHS ${LIBLZ4_LIBRARY_DIRS} + NO_DEFAULT_PATH) +else() + set(LZ4_LIBS ${LIBLZ4_LIBRARIES}) +endif() read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/sources.am LIBGROONGA_SOURCES) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/dat/sources.am LIBGRNDAT_SOURCES) @@ -60,7 +66,7 @@ set(GRN_ALL_LIBRARIES ${RT_LIBS} ${PTHREAD_LIBS} ${Z_LIBS} - ${LIBLZ4_LIBRARIES} + ${LZ4_LIBS} ${DL_LIBS} ${M_LIBS} ${WS2_32_LIBS} diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index dac01b8af89..2cfaa5ebdcc 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2012, Oracle and/or its affiliates. - Copyright (c) 2009, 2014, SkySQL Ab. + Copyright (c) 2009, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -872,59 +872,59 @@ int ha_myisam::check(THD* thd, HA_CHECK_OPT* check_opt) { if (!file) return HA_ADMIN_INTERNAL_ERROR; int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); MYISAM_SHARE* share = file->s; const char *old_proc_info=thd->proc_info; - if (!¶m) + if (!param) return HA_ADMIN_INTERNAL_ERROR; thd_proc_info(thd, "Checking table"); - myisamchk_init(¶m); - param.thd = thd; - param.op_name = "check"; - param.db_name= table->s->db.str; - param.table_name= table->alias.c_ptr(); - param.testflag = check_opt->flags | T_CHECK | T_SILENT; - param.stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); + myisamchk_init(param); + param->thd = thd; + param->op_name = "check"; + param->db_name= table->s->db.str; + param->table_name= table->alias.c_ptr(); + param->testflag = check_opt->flags | T_CHECK | T_SILENT; + param->stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); if (!(table->db_stat & HA_READ_ONLY)) - param.testflag|= T_STATISTICS; - param.using_global_keycache = 1; + param->testflag|= T_STATISTICS; + param->using_global_keycache = 1; if (!mi_is_crashed(file) && - (((param.testflag & T_CHECK_ONLY_CHANGED) && + (((param->testflag & T_CHECK_ONLY_CHANGED) && !(share->state.changed & (STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR)) && share->state.open_count == 0) || - ((param.testflag & T_FAST) && (share->state.open_count == + ((param->testflag & T_FAST) && (share->state.open_count == (uint) (share->global_changed ? 1 : 0))))) return HA_ADMIN_ALREADY_DONE; - error = chk_status(¶m, file); // Not fatal - error = chk_size(¶m, file); + error = chk_status(param, file); // Not fatal + error = chk_size(param, file); if (!error) - error |= chk_del(¶m, file, param.testflag); + error |= chk_del(param, file, param->testflag); if (!error) - error = chk_key(¶m, file); + error = chk_key(param, file); if (!error) { - if ((!(param.testflag & T_QUICK) && + if ((!(param->testflag & T_QUICK) && ((share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) || - (param.testflag & (T_EXTEND | T_MEDIUM)))) || + (param->testflag & (T_EXTEND | T_MEDIUM)))) || mi_is_crashed(file)) { - ulonglong old_testflag= param.testflag; - param.testflag|=T_MEDIUM; - if (!(error= init_io_cache(¶m.read_cache, file->dfile, + ulonglong old_testflag= param->testflag; + param->testflag|=T_MEDIUM; + if (!(error= init_io_cache(¶m->read_cache, file->dfile, my_default_record_cache_size, READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)))) { - error= chk_data_link(¶m, file, MY_TEST(param.testflag & T_EXTEND)); - end_io_cache(&(param.read_cache)); + error= chk_data_link(param, file, MY_TEST(param->testflag & T_EXTEND)); + end_io_cache(¶m->read_cache); } - param.testflag= old_testflag; + param->testflag= old_testflag; } } if (!error) @@ -932,7 +932,7 @@ int ha_myisam::check(THD* thd, HA_CHECK_OPT* check_opt) if ((share->state.changed & (STATE_CHANGED | STATE_CRASHED_ON_REPAIR | STATE_CRASHED | STATE_NOT_ANALYZED)) || - (param.testflag & T_STATISTICS) || + (param->testflag & T_STATISTICS) || mi_is_crashed(file)) { file->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED; @@ -940,7 +940,7 @@ int ha_myisam::check(THD* thd, HA_CHECK_OPT* check_opt) share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR); if (!(table->db_stat & HA_READ_ONLY)) - error=update_state_info(¶m,file,UPDATE_TIME | UPDATE_OPEN_COUNT | + error=update_state_info(param,file,UPDATE_TIME | UPDATE_OPEN_COUNT | UPDATE_STAT); mysql_mutex_unlock(&share->intern_lock); info(HA_STATUS_NO_LOCK | HA_STATUS_TIME | HA_STATUS_VARIABLE | @@ -967,30 +967,30 @@ int ha_myisam::check(THD* thd, HA_CHECK_OPT* check_opt) int ha_myisam::analyze(THD *thd, HA_CHECK_OPT* check_opt) { int error=0; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); MYISAM_SHARE* share = file->s; - if (!¶m) + if (!param) return HA_ADMIN_INTERNAL_ERROR; - myisamchk_init(¶m); - param.thd = thd; - param.op_name= "analyze"; - param.db_name= table->s->db.str; - param.table_name= table->alias.c_ptr(); - param.testflag= (T_FAST | T_CHECK | T_SILENT | T_STATISTICS | + myisamchk_init(param); + param->thd = thd; + param->op_name= "analyze"; + param->db_name= table->s->db.str; + param->table_name= table->alias.c_ptr(); + param->testflag= (T_FAST | T_CHECK | T_SILENT | T_STATISTICS | T_DONT_CHECK_CHECKSUM); - param.using_global_keycache = 1; - param.stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); + param->using_global_keycache = 1; + param->stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); if (!(share->state.changed & STATE_NOT_ANALYZED)) return HA_ADMIN_ALREADY_DONE; - error = chk_key(¶m, file); + error = chk_key(param, file); if (!error) { mysql_mutex_lock(&share->intern_lock); - error=update_state_info(¶m,file,UPDATE_STAT); + error=update_state_info(param,file,UPDATE_STAT); mysql_mutex_unlock(&share->intern_lock); } else if (!mi_is_crashed(file) && !thd->killed) @@ -1002,37 +1002,37 @@ int ha_myisam::analyze(THD *thd, HA_CHECK_OPT* check_opt) int ha_myisam::repair(THD* thd, HA_CHECK_OPT *check_opt) { int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); ha_rows start_records; - if (!file || !¶m) return HA_ADMIN_INTERNAL_ERROR; + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; - myisamchk_init(¶m); - param.thd = thd; - param.op_name= "repair"; - param.testflag= ((check_opt->flags & ~(T_EXTEND)) | + myisamchk_init(param); + param->thd = thd; + param->op_name= "repair"; + param->testflag= ((check_opt->flags & ~(T_EXTEND)) | T_SILENT | T_FORCE_CREATE | T_CALC_CHECKSUM | (check_opt->flags & T_EXTEND ? T_REP : T_REP_BY_SORT)); - param.sort_buffer_length= THDVAR(thd, sort_buffer_size); - param.backup_time= check_opt->start_time; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->backup_time= check_opt->start_time; start_records=file->state->records; - while ((error=repair(thd,param,0)) && param.retry_repair) + while ((error=repair(thd,*param,0)) && param->retry_repair) { - param.retry_repair=0; - if (test_all_bits(param.testflag, + param->retry_repair=0; + if (test_all_bits(param->testflag, (uint) (T_RETRY_WITHOUT_QUICK | T_QUICK))) { - param.testflag&= ~(T_RETRY_WITHOUT_QUICK | T_QUICK); + param->testflag&= ~(T_RETRY_WITHOUT_QUICK | T_QUICK); /* Ensure we don't loose any rows when retrying without quick */ - param.testflag|= T_SAFE_REPAIR; + param->testflag|= T_SAFE_REPAIR; sql_print_information("Retrying repair of: '%s' including modifying data file", table->s->path.str); continue; } - param.testflag&= ~T_QUICK; - if ((param.testflag & T_REP_BY_SORT)) + param->testflag&= ~T_QUICK; + if ((param->testflag & T_REP_BY_SORT)) { - param.testflag= (param.testflag & ~T_REP_BY_SORT) | T_REP; + param->testflag= (param->testflag & ~T_REP_BY_SORT) | T_REP; sql_print_information("Retrying repair of: '%s' with keycache", table->s->path.str); continue; @@ -1054,22 +1054,22 @@ int ha_myisam::repair(THD* thd, HA_CHECK_OPT *check_opt) int ha_myisam::optimize(THD* thd, HA_CHECK_OPT *check_opt) { int error; - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); - if (!file || !¶m) return HA_ADMIN_INTERNAL_ERROR; + if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; - myisamchk_init(¶m); - param.thd = thd; - param.op_name= "optimize"; - param.testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | + myisamchk_init(param); + param->thd = thd; + param->op_name= "optimize"; + param->testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | T_REP_BY_SORT | T_STATISTICS | T_SORT_INDEX); - param.sort_buffer_length= THDVAR(thd, sort_buffer_size); - if ((error= repair(thd,param,1)) && param.retry_repair) + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + if ((error= repair(thd,*param,1)) && param->retry_repair) { sql_print_warning("Warning: Optimize table got errno %d on %s.%s, retrying", - my_errno, param.db_name, param.table_name); - param.testflag&= ~T_REP_BY_SORT; - error= repair(thd,param,1); + my_errno, param->db_name, param->table_name); + param->testflag&= ~T_REP_BY_SORT; + error= repair(thd,*param,1); } return error; } @@ -1274,17 +1274,17 @@ int ha_myisam::assign_to_keycache(THD* thd, HA_CHECK_OPT *check_opt) if (error != HA_ADMIN_OK) { /* Send error to user */ - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); - if (!¶m) + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) return HA_ADMIN_INTERNAL_ERROR; - myisamchk_init(¶m); - param.thd= thd; - param.op_name= "assign_to_keycache"; - param.db_name= table->s->db.str; - param.table_name= table->s->table_name.str; - param.testflag= 0; - mi_check_print_error(¶m, errmsg); + myisamchk_init(param); + param->thd= thd; + param->op_name= "assign_to_keycache"; + param->db_name= table->s->db.str; + param->table_name= table->s->table_name.str; + param->testflag= 0; + mi_check_print_error(param, errmsg); } DBUG_RETURN(error); } @@ -1341,16 +1341,16 @@ int ha_myisam::preload_keys(THD* thd, HA_CHECK_OPT *check_opt) err: { - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); - if (!¶m) + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); + if (!param) return HA_ADMIN_INTERNAL_ERROR; - myisamchk_init(¶m); - param.thd= thd; - param.op_name= "preload_keys"; - param.db_name= table->s->db.str; - param.table_name= table->s->table_name.str; - param.testflag= 0; - mi_check_print_error(¶m, errmsg); + myisamchk_init(param); + param->thd= thd; + param->op_name= "preload_keys"; + param->db_name= table->s->db.str; + param->table_name= table->s->table_name.str; + param->testflag= 0; + mi_check_print_error(param, errmsg); DBUG_RETURN(error); } } @@ -1455,45 +1455,45 @@ int ha_myisam::enable_indexes(uint mode) { THD *thd= table->in_use; int was_error= thd->is_error(); - HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); + HA_CHECK *param= (HA_CHECK*) thd->alloc(sizeof *param); const char *save_proc_info=thd->proc_info; - if (!¶m) + if (!param) DBUG_RETURN(HA_ADMIN_INTERNAL_ERROR); thd_proc_info(thd, "Creating index"); - myisamchk_init(¶m); - param.op_name= "recreating_index"; - param.testflag= (T_SILENT | T_REP_BY_SORT | T_QUICK | + myisamchk_init(param); + param->op_name= "recreating_index"; + param->testflag= (T_SILENT | T_REP_BY_SORT | T_QUICK | T_CREATE_MISSING_KEYS); /* Don't lock and unlock table if it's locked. Normally table should be locked. This test is mostly for safety. */ if (likely(file->lock_type != F_UNLCK)) - param.testflag|= T_NO_LOCKS; - + param->testflag|= T_NO_LOCKS; + if (file->create_unique_index_by_sort) - param.testflag|= T_CREATE_UNIQUE_BY_SORT; + param->testflag|= T_CREATE_UNIQUE_BY_SORT; - param.myf_rw&= ~MY_WAIT_IF_FULL; - param.sort_buffer_length= THDVAR(thd, sort_buffer_size); - param.stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); - param.tmpdir=&mysql_tmpdir_list; - if ((error= (repair(thd,param,0) != HA_ADMIN_OK)) && param.retry_repair) + param->myf_rw&= ~MY_WAIT_IF_FULL; + param->sort_buffer_length= THDVAR(thd, sort_buffer_size); + param->stats_method= (enum_handler_stats_method)THDVAR(thd, stats_method); + param->tmpdir=&mysql_tmpdir_list; + if ((error= (repair(thd,*param,0) != HA_ADMIN_OK)) && param->retry_repair) { sql_print_warning("Warning: Enabling keys got errno %d on %s.%s, retrying", - my_errno, param.db_name, param.table_name); + my_errno, param->db_name, param->table_name); /* Repairing by sort failed. Now try standard repair method. Still we want to fix only index file. If data file corruption was detected (T_RETRY_WITHOUT_QUICK), we shouldn't do much here. Let implicit repair do this job. */ - if (!(param.testflag & T_RETRY_WITHOUT_QUICK)) + if (!(param->testflag & T_RETRY_WITHOUT_QUICK)) { - param.testflag&= ~T_REP_BY_SORT; - error= (repair(thd,param,0) != HA_ADMIN_OK); + param->testflag&= ~T_REP_BY_SORT; + error= (repair(thd,*param,0) != HA_ADMIN_OK); } /* If the standard repair succeeded, clear all error messages which @@ -1896,15 +1896,22 @@ int ha_myisam::info(uint flag) Set data_file_name and index_file_name to point at the symlink value if table is symlinked (Ie; Real name is not same as generated name) */ + char buf[FN_REFLEN]; data_file_name= index_file_name= 0; fn_format(name_buff, file->filename, "", MI_NAME_DEXT, MY_APPEND_EXT | MY_UNPACK_FILENAME); - if (strcmp(name_buff, misam_info.data_file_name)) - data_file_name=misam_info.data_file_name; + if (my_is_symlink(name_buff)) + { + my_readlink(buf, name_buff, MYF(0)); + data_file_name= ha_thd()->strdup(buf); + } fn_format(name_buff, file->filename, "", MI_NAME_IEXT, MY_APPEND_EXT | MY_UNPACK_FILENAME); - if (strcmp(name_buff, misam_info.index_file_name)) - index_file_name=misam_info.index_file_name; + if (my_is_symlink(name_buff)) + { + my_readlink(buf, name_buff, MYF(0)); + index_file_name= ha_thd()->strdup(buf); + } } if (flag & HA_STATUS_ERRKEY) { @@ -2179,6 +2186,74 @@ uint ha_myisam::checksum() const } +enum_alter_inplace_result +ha_myisam::check_if_supported_inplace_alter(TABLE *new_table, + Alter_inplace_info *alter_info) +{ + DBUG_ENTER("ha_myisam::check_if_supported_inplace_alter"); + + const uint readd_index= Alter_inplace_info::ADD_INDEX | + Alter_inplace_info::DROP_INDEX; + const uint readd_unique= Alter_inplace_info::ADD_UNIQUE_INDEX | + Alter_inplace_info::DROP_UNIQUE_INDEX; + const uint readd_pk= Alter_inplace_info::ADD_PK_INDEX | + Alter_inplace_info::DROP_PK_INDEX; + + const uint op= alter_info->handler_flags; + + /* + ha_myisam::open() updates table->key_info->block_size to be the actual + MYI index block size, overwriting user-specified value (if any). + So, the server can not reliably detect whether ALTER TABLE changes + key_block_size or not, it might think the block size was changed, + when it wasn't, and in this case the server will recreate (drop+add) + the index unnecessary. Fix it. + */ + + if (table->s->keys == new_table->s->keys && + ((op & readd_pk) == readd_pk || + (op & readd_unique) == readd_unique || + (op & readd_index) == readd_index)) + { + for (uint i=0; i < table->s->keys; i++) + { + KEY *old_key= table->key_info + i; + KEY *new_key= new_table->key_info + i; + + if (old_key->block_size == new_key->block_size) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); // must differ somewhere else + + if (new_key->block_size && new_key->block_size != old_key->block_size) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); // really changed + + /* any difference besides the block_size, and we give up */ + if (old_key->key_length != new_key->key_length || + old_key->flags != new_key->flags || + old_key->user_defined_key_parts != new_key->user_defined_key_parts || + old_key->algorithm != new_key->algorithm || + strcmp(old_key->name, new_key->name)) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + + for (uint j= 0; j < old_key->user_defined_key_parts; j++) + { + KEY_PART_INFO *old_kp= old_key->key_part + j; + KEY_PART_INFO *new_kp= new_key->key_part + j; + if (old_kp->offset != new_kp->offset || + old_kp->null_offset != new_kp->null_offset || + old_kp->length != new_kp->length || + old_kp->fieldnr != new_kp->fieldnr || + old_kp->key_part_flag != new_kp->key_part_flag || + old_kp->type != new_kp->type || + old_kp->null_bit != new_kp->null_bit) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + } + } + alter_info->handler_flags &= ~(readd_pk | readd_unique | readd_index); + } + DBUG_RETURN(handler::check_if_supported_inplace_alter(new_table, alter_info)); +} + + bool ha_myisam::check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes) { diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h index 63fb0ea5a2a..b132c955795 100644 --- a/storage/myisam/ha_myisam.h +++ b/storage/myisam/ha_myisam.h @@ -138,6 +138,8 @@ class ha_myisam: public handler int optimize(THD* thd, HA_CHECK_OPT* check_opt); int assign_to_keycache(THD* thd, HA_CHECK_OPT* check_opt); int preload_keys(THD* thd, HA_CHECK_OPT* check_opt); + enum_alter_inplace_result check_if_supported_inplace_alter(TABLE *new_table, + Alter_inplace_info *alter_info); bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); #ifdef HAVE_QUERY_CACHE my_bool register_query_cache_table(THD *thd, char *table_key, diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index ec72e0b430a..bbe45aa3c26 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -80,8 +80,7 @@ static SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *param, uint blocks, uint buffer_length); static ha_checksum mi_byte_checksum(const uchar *buf, uint length); static void set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share); -static int replace_data_file(HA_CHECK *param, MI_INFO *info, - const char *name, File new_file); +static int replace_data_file(HA_CHECK *param, MI_INFO *info, File new_file); void myisamchk_init(HA_CHECK *param) { @@ -1715,7 +1714,7 @@ err: /* Replace the actual file with the temporary file */ if (new_file >= 0) { - got_error= replace_data_file(param, info, name, new_file); + got_error= replace_data_file(param, info, new_file); new_file= -1; param->retry_repair= 0; } @@ -2528,7 +2527,7 @@ err: /* Replace the actual file with the temporary file */ if (new_file >= 0) { - got_error= replace_data_file(param, info, name, new_file); + got_error= replace_data_file(param, info, new_file); new_file= -1; } } @@ -2542,7 +2541,7 @@ err: (void) mysql_file_delete(mi_key_file_datatmp, param->temp_filename, MYF(MY_WME)); if (info->dfile == new_file) /* Retry with key cache */ - if (unlikely(mi_open_datafile(info, share, name, -1))) + if (unlikely(mi_open_datafile(info, share))) param->retry_repair= 0; /* Safety */ } mi_mark_crashed_on_repair(info); @@ -3063,7 +3062,7 @@ err: /* Replace the actual file with the temporary file */ if (new_file >= 0) { - got_error= replace_data_file(param, info, name, new_file); + got_error= replace_data_file(param, info, new_file); new_file= -1; } } @@ -3077,7 +3076,7 @@ err: (void) mysql_file_delete(mi_key_file_datatmp, param->temp_filename, MYF(MY_WME)); if (info->dfile == new_file) /* Retry with key cache */ - if (unlikely(mi_open_datafile(info, share, name, -1))) + if (unlikely(mi_open_datafile(info, share))) param->retry_repair= 0; /* Safety */ } mi_mark_crashed_on_repair(info); @@ -4760,8 +4759,7 @@ int mi_make_backup_of_index(MI_INFO *info, time_t backup_time, myf flags) } -static int replace_data_file(HA_CHECK *param, MI_INFO *info, - const char *name, File new_file) +static int replace_data_file(HA_CHECK *param, MI_INFO *info, File new_file) { MYISAM_SHARE *share=info->s; @@ -4796,7 +4794,7 @@ static int replace_data_file(HA_CHECK *param, MI_INFO *info, DATA_TMP_EXT, param->backup_time, (param->testflag & T_BACKUP_DATA ? MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) || - mi_open_datafile(info, share, name, -1)) + mi_open_datafile(info, share)) return 1; return 0; } diff --git a/storage/myisam/mi_create.c b/storage/myisam/mi_create.c index 88b9da6f8a9..aa85a3718de 100644 --- a/storage/myisam/mi_create.c +++ b/storage/myisam/mi_create.c @@ -46,7 +46,8 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, uint aligned_key_start, block_length, res; uint internal_table= flags & HA_CREATE_INTERNAL_TABLE; ulong reclength, real_reclength,min_pack_length; - char filename[FN_REFLEN],linkname[FN_REFLEN], *linkname_ptr; + char kfilename[FN_REFLEN],klinkname[FN_REFLEN], *klinkname_ptr; + char dfilename[FN_REFLEN],dlinkname[FN_REFLEN], *dlinkname_ptr; ulong pack_reclength; ulonglong tot_length,max_rows, tmp; enum en_fieldtype type; @@ -594,19 +595,19 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, /* chop off the table name, tempory tables use generated name */ if ((path= strrchr(ci->index_file_name, FN_LIBCHAR))) *path= '\0'; - fn_format(filename, name, ci->index_file_name, MI_NAME_IEXT, + fn_format(kfilename, name, ci->index_file_name, MI_NAME_IEXT, MY_REPLACE_DIR | MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | MY_APPEND_EXT); } else { - fn_format(filename, ci->index_file_name, "", MI_NAME_IEXT, + fn_format(kfilename, ci->index_file_name, "", MI_NAME_IEXT, MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | (have_iext ? MY_REPLACE_EXT : MY_APPEND_EXT)); } - fn_format(linkname, name, "", MI_NAME_IEXT, + fn_format(klinkname, name, "", MI_NAME_IEXT, MY_UNPACK_FILENAME|MY_APPEND_EXT); - linkname_ptr=linkname; + klinkname_ptr= klinkname; /* Don't create the table if the link or file exists to ensure that one doesn't accidently destroy another table. @@ -617,10 +618,10 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, { char *iext= strrchr(name, '.'); int have_iext= iext && !strcmp(iext, MI_NAME_IEXT); - fn_format(filename, name, "", MI_NAME_IEXT, + fn_format(kfilename, name, "", MI_NAME_IEXT, MY_UNPACK_FILENAME | MY_RETURN_REAL_PATH | (have_iext ? MY_REPLACE_EXT : MY_APPEND_EXT)); - linkname_ptr=0; + klinkname_ptr= 0; /* Replace the current file */ create_flag=(flags & HA_CREATE_KEEP_FILES) ? 0 : MY_DELETE_OLD; } @@ -635,7 +636,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, NOTE: The filename is compared against unique_file_name of every open table. Hence we need a real path here. */ - if (!internal_table && test_if_reopen(filename)) + if (!internal_table && test_if_reopen(kfilename)) { my_printf_error(HA_ERR_TABLE_EXIST, "MyISAM table '%s' is in use " "(most likely by a MERGE table). Try FLUSH TABLES.", @@ -645,7 +646,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, } if ((file= mysql_file_create_with_symlink(mi_key_file_kfile, - linkname_ptr, filename, 0, + klinkname_ptr, kfilename, 0, create_mode, MYF(MY_WME | create_flag))) < 0) goto err; @@ -665,31 +666,31 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs, /* chop off the table name, tempory tables use generated name */ if ((path= strrchr(ci->data_file_name, FN_LIBCHAR))) *path= '\0'; - fn_format(filename, name, ci->data_file_name, MI_NAME_DEXT, + fn_format(dfilename, name, ci->data_file_name, MI_NAME_DEXT, MY_REPLACE_DIR | MY_UNPACK_FILENAME | MY_APPEND_EXT); } else { - fn_format(filename, ci->data_file_name, "", MI_NAME_DEXT, + fn_format(dfilename, ci->data_file_name, "", MI_NAME_DEXT, MY_UNPACK_FILENAME | (have_dext ? MY_REPLACE_EXT : MY_APPEND_EXT)); } - fn_format(linkname, name, "",MI_NAME_DEXT, + fn_format(dlinkname, name, "",MI_NAME_DEXT, MY_UNPACK_FILENAME | MY_APPEND_EXT); - linkname_ptr=linkname; + dlinkname_ptr= dlinkname; create_flag=0; } else { - fn_format(filename,name,"", MI_NAME_DEXT, + fn_format(dfilename,name,"", MI_NAME_DEXT, MY_UNPACK_FILENAME | MY_APPEND_EXT); - linkname_ptr=0; + dlinkname_ptr= 0; create_flag=(flags & HA_CREATE_KEEP_FILES) ? 0 : MY_DELETE_OLD; } if ((dfile= mysql_file_create_with_symlink(mi_key_file_dfile, - linkname_ptr, filename, 0, + dlinkname_ptr, dfilename, 0, create_mode, MYF(MY_WME | create_flag))) < 0) goto err; @@ -843,19 +844,21 @@ err_no_lock: (void) mysql_file_close(dfile, MYF(0)); /* fall through */ case 2: - if (! (flags & HA_DONT_TOUCH_DATA)) - mysql_file_delete_with_symlink(mi_key_file_dfile, - fn_format(filename, name, "", MI_NAME_DEXT, - MY_UNPACK_FILENAME | MY_APPEND_EXT), - MYF(0)); + if (! (flags & HA_DONT_TOUCH_DATA)) + { + mysql_file_delete(mi_key_file_dfile, dfilename, MYF(0)); + if (dlinkname_ptr) + mysql_file_delete(mi_key_file_dfile, dlinkname_ptr, MYF(0)); + } /* fall through */ case 1: (void) mysql_file_close(file, MYF(0)); if (! (flags & HA_DONT_TOUCH_DATA)) - mysql_file_delete_with_symlink(mi_key_file_kfile, - fn_format(filename, name, "", MI_NAME_IEXT, - MY_UNPACK_FILENAME | MY_APPEND_EXT), - MYF(0)); + { + mysql_file_delete(mi_key_file_kfile, kfilename, MYF(0)); + if (klinkname_ptr) + mysql_file_delete(mi_key_file_kfile, klinkname_ptr, MYF(0)); + } } my_free(rec_per_key_part); DBUG_RETURN(my_errno=save_errno); /* return the fatal errno */ diff --git a/storage/myisam/mi_delete_table.c b/storage/myisam/mi_delete_table.c index 7da960011ca..5ad63a5cc87 100644 --- a/storage/myisam/mi_delete_table.c +++ b/storage/myisam/mi_delete_table.c @@ -26,48 +26,21 @@ #define mi_key_file_dfile 0 #endif -static int delete_one_file(const char *name, const char *ext, - PSI_file_key pskey __attribute__((unused)), - myf flags) -{ - char from[FN_REFLEN]; - DBUG_ENTER("delete_one_file"); - fn_format(from,name, "", ext, MY_UNPACK_FILENAME | MY_APPEND_EXT); - if (my_is_symlink(from) && (*myisam_test_invalid_symlink)(from)) - { - /* - Symlink is pointing to file in data directory. - Remove symlink, keep file. - */ - if (mysql_file_delete(pskey, from, flags)) - DBUG_RETURN(my_errno); - } - else - { - if (mysql_file_delete_with_symlink(pskey, from, flags)) - DBUG_RETURN(my_errno); - } - DBUG_RETURN(0); -} - int mi_delete_table(const char *name) { - int res; DBUG_ENTER("mi_delete_table"); #ifdef EXTRA_DEBUG check_table_is_closed(name,"delete"); #endif - if ((res= delete_one_file(name, MI_NAME_IEXT, mi_key_file_kfile, MYF(MY_WME)))) - DBUG_RETURN(res); - if ((res= delete_one_file(name, MI_NAME_DEXT, mi_key_file_dfile, MYF(MY_WME)))) - DBUG_RETURN(res); + if (mysql_file_delete_with_symlink(mi_key_file_kfile, name, MI_NAME_IEXT, MYF(MY_WME)) || + mysql_file_delete_with_symlink(mi_key_file_dfile, name, MI_NAME_DEXT, MYF(MY_WME))) + DBUG_RETURN(my_errno); // optionally present: - delete_one_file(name, ".OLD", mi_key_file_dfile, MYF(0)); - delete_one_file(name, ".TMD", mi_key_file_dfile, MYF(0)); + mysql_file_delete_with_symlink(mi_key_file_dfile, name, ".OLD", MYF(0)); + mysql_file_delete_with_symlink(mi_key_file_dfile, name, ".TMD", MYF(0)); DBUG_RETURN(0); } - diff --git a/storage/myisam/mi_locking.c b/storage/myisam/mi_locking.c index 531b800c63e..1921926463e 100644 --- a/storage/myisam/mi_locking.c +++ b/storage/myisam/mi_locking.c @@ -29,7 +29,7 @@ static void mi_update_status_with_lock(MI_INFO *info); int mi_lock_database(MI_INFO *info, int lock_type) { - int error; + int error, mark_crashed= 0; uint count; MYISAM_SHARE *share=info->s; DBUG_ENTER("mi_lock_database"); @@ -52,6 +52,7 @@ int mi_lock_database(MI_INFO *info, int lock_type) } error= 0; + DBUG_EXECUTE_IF ("mi_lock_database_failure", error= EINVAL;); mysql_mutex_lock(&share->intern_lock); if (share->kfile >= 0) /* May only be false on windows */ { @@ -75,17 +76,15 @@ int mi_lock_database(MI_INFO *info, int lock_type) &share->dirty_part_map, FLUSH_KEEP)) { - error=my_errno; + mark_crashed= error=my_errno; mi_print_error(info->s, HA_ERR_CRASHED); - mi_mark_crashed(info); /* Mark that table must be checked */ } if (info->opt_flag & (READ_CACHE_USED | WRITE_CACHE_USED)) { if (end_io_cache(&info->rec_cache)) { - error=my_errno; + mark_crashed= error=my_errno; mi_print_error(info->s, HA_ERR_CRASHED); - mi_mark_crashed(info); } } if (!count) @@ -110,22 +109,19 @@ int mi_lock_database(MI_INFO *info, int lock_type) share->state.unique= info->last_unique= info->this_unique; share->state.update_count= info->last_loop= ++info->this_loop; if (mi_state_info_write(share->kfile, &share->state, 1)) - error=my_errno; + mark_crashed= error=my_errno; share->changed=0; if (myisam_flush) { if (mysql_file_sync(share->kfile, MYF(0))) - error= my_errno; + mark_crashed= error= my_errno; if (mysql_file_sync(info->dfile, MYF(0))) - error= my_errno; + mark_crashed= error= my_errno; } else share->not_flushed=1; if (error) - { mi_print_error(info->s, HA_ERR_CRASHED); - mi_mark_crashed(info); - } } if (info->lock_type != F_EXTRA_LCK) { @@ -260,6 +256,8 @@ int mi_lock_database(MI_INFO *info, int lock_type) } #endif mysql_mutex_unlock(&share->intern_lock); + if (mark_crashed) + mi_mark_crashed(info); DBUG_RETURN(error); } /* mi_lock_database */ diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 060017f10ad..bdb2fdf8447 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -104,8 +104,13 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) realpath_err= my_realpath(name_buff, fn_format(org_name,name,"",MI_NAME_IEXT,4),MYF(0)); + if (realpath_err > 0) /* File not found, no point in looking further. */ + { + DBUG_RETURN(NULL); + } + if (my_is_symlink(org_name) && - (realpath_err || (*myisam_test_invalid_symlink)(name_buff))) + (realpath_err || mysys_test_invalid_symlink(name_buff))) { my_errno= HA_WRONG_CREATE_OPTION; DBUG_RETURN (NULL); @@ -131,15 +136,17 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) my_errno= HA_ERR_CRASHED; goto err; }); - if ((kfile= mysql_file_open(mi_key_file_kfile, - name_buff, - (open_mode= O_RDWR) | O_SHARE, MYF(0))) < 0) + + DEBUG_SYNC_C("mi_open_kfile"); + if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff, + (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW, + MYF(MY_NOSYMLINKS))) < 0) { if ((errno != EROFS && errno != EACCES) || mode != O_RDONLY || - (kfile= mysql_file_open(mi_key_file_kfile, - name_buff, - (open_mode= O_RDONLY) | O_SHARE, MYF(0))) < 0) + (kfile= mysql_file_open(mi_key_file_kfile, name_buff, + (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW, + MYF(MY_NOSYMLINKS))) < 0) goto err; } share->mode=open_mode; @@ -183,7 +190,18 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) (void) strmov(index_name, org_name); *strrchr(org_name, '.')= '\0'; (void) fn_format(data_name,org_name,"",MI_NAME_DEXT, - MY_APPEND_EXT|MY_UNPACK_FILENAME|MY_RESOLVE_SYMLINKS); + MY_APPEND_EXT|MY_UNPACK_FILENAME); + if (my_is_symlink(data_name)) + { + if (my_realpath(data_name, data_name, MYF(0))) + goto err; + if (mysys_test_invalid_symlink(data_name)) + { + my_errno= HA_WRONG_CREATE_OPTION; + goto err; + } + share->mode|= O_NOFOLLOW; /* all symlinks are resolved by realpath() */ + } info_length=mi_uint2korr(share->state.header.header_length); base_pos=mi_uint2korr(share->state.header.base_pos); @@ -497,7 +515,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) lock_error=1; /* Database unlocked */ } - if (mi_open_datafile(&info, share, name, -1)) + if (mi_open_datafile(&info, share)) goto err; errpos=5; @@ -578,7 +596,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) my_errno=EACCES; /* Can't open in write mode */ goto err; } - if (mi_open_datafile(&info, share, name, old_info->dfile)) + if (mi_open_datafile(&info, share)) goto err; errpos=5; have_rtree= old_info->rtree_recursion_state != NULL; @@ -1250,33 +1268,14 @@ uchar *mi_recinfo_read(uchar *ptr, MI_COLUMNDEF *recinfo) Open data file. We can't use dup() here as the data file descriptors need to have different active seek-positions. - -The argument file_to_dup is here for the future if there would on some OS -exist a dup()-like call that would give us two different file descriptors. *************************************************************************/ -int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share, const char *org_name, - File file_to_dup __attribute__((unused))) +int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share) { - char *data_name= share->data_file_name; - char real_data_name[FN_REFLEN]; - - if (org_name) - { - fn_format(real_data_name,org_name,"",MI_NAME_DEXT,4); - if (my_is_symlink(real_data_name)) - { - if (my_realpath(real_data_name, real_data_name, MYF(0)) || - (*myisam_test_invalid_symlink)(real_data_name)) - { - my_errno= HA_WRONG_CREATE_OPTION; - return 1; - } - data_name= real_data_name; - } - } - info->dfile= mysql_file_open(mi_key_file_dfile, - data_name, share->mode | O_SHARE, MYF(MY_WME)); + myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0); + DEBUG_SYNC_C("mi_open_datafile"); + info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name, + share->mode | O_SHARE, MYF(flags)); return info->dfile >= 0 ? 0 : 1; } @@ -1285,8 +1284,8 @@ int mi_open_keyfile(MYISAM_SHARE *share) { if ((share->kfile= mysql_file_open(mi_key_file_kfile, share->unique_file_name, - share->mode | O_SHARE, - MYF(MY_WME))) < 0) + share->mode | O_SHARE | O_NOFOLLOW, + MYF(MY_NOSYMLINKS | MY_WME))) < 0) return 1; return 0; } diff --git a/storage/myisam/mi_static.c b/storage/myisam/mi_static.c index d77f4f6b8e2..49019fb861c 100644 --- a/storage/myisam/mi_static.c +++ b/storage/myisam/mi_static.c @@ -42,14 +42,6 @@ ulong myisam_data_pointer_size=4; ulonglong myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0; my_bool (*mi_killed)(MI_INFO *)= mi_killed_standalone; -static int always_valid(const char *filename __attribute__((unused))) -{ - return 0; -} - -int (*myisam_test_invalid_symlink)(const char *filename)= always_valid; - - /* read_vec[] is used for converting between P_READ_KEY.. and SEARCH_ Position is , == , >= , <= , > , < diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c index 7835ab83531..edbe235e190 100644 --- a/storage/myisam/myisamchk.c +++ b/storage/myisam/myisamchk.c @@ -1047,7 +1047,7 @@ static int myisamchk(HA_CHECK *param, char * filename) MYF(MY_WME)); /* Close new file */ error|=change_to_newfile(filename, MI_NAME_DEXT, DATA_TMP_EXT, 0, MYF(0)); - if (mi_open_datafile(info,info->s, NULL, -1)) + if (mi_open_datafile(info, info->s)) error=1; param->out_flag&= ~O_NEW_DATA; /* We are using new datafile */ param->read_cache.file=info->dfile; diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h index 07c4a5b473a..92fc5e45d5d 100644 --- a/storage/myisam/myisamdef.h +++ b/storage/myisam/myisamdef.h @@ -713,8 +713,7 @@ void mi_disable_indexes_for_rebuild(MI_INFO *info, ha_rows rows, my_bool all_keys); extern MI_INFO *test_if_reopen(char *filename); my_bool check_table_is_closed(const char *name, const char *where); -int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share, const char *orn_name, - File file_to_dup); +int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share); int mi_open_keyfile(MYISAM_SHARE *share); void mi_setup_functions(register MYISAM_SHARE *share); diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result index 42c86405503..6b1d0a1854d 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result @@ -33,5 +33,5 @@ FROM `version_history` AS `v` INNER JOIN `db_history` AS `db` ON `db`.`nodeID` = WHERE `latch` = 'breadth_first' AND `origid` = '1' ORDER BY `weight` DESC LIMIT 1; version nodeID 0.0.3 3 -DROP TABLE db_history; DROP TABLE version_history; +DROP TABLE db_history; diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test index 3a7a0e5af08..b8f0ae556c8 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test @@ -40,8 +40,8 @@ --disconnect con2 --connect (con3,localhost,root,,test) -DROP TABLE db_history; DROP TABLE version_history; +DROP TABLE db_history; --disconnect con3 diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result index 8e680e15206..68002ce98a2 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result @@ -8,5 +8,5 @@ latch origid destid weight seq linkid breadth_first 1 6 NULL 0 1 breadth_first 1 6 1 1 2 breadth_first 1 6 1 2 6 -DROP TABLE IF EXISTS oq_backing; DROP TABLE IF EXISTS oq_graph; +DROP TABLE IF EXISTS oq_backing; diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test index 72de926da2e..fefd03a7ed9 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test @@ -14,6 +14,6 @@ CREATE TABLE oq_graph (latch VARCHAR(32) NULL, origid BIGINT UNSIGNED NULL, dest SELECT * FROM oq_graph WHERE latch='breadth_first' AND origid=1 AND destid=6; -DROP TABLE IF EXISTS oq_backing; DROP TABLE IF EXISTS oq_graph; +DROP TABLE IF EXISTS oq_backing; diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index dd61df5f861..c30ff4b6640 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2560,10 +2560,7 @@ start_table_io_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_io_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; - - DBUG_ASSERT(pfs_thread == - my_pthread_getspecific_ptr(PFS_thread*, THR_PFS)); + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); register uint flags; ulonglong timer_start= 0; @@ -2666,7 +2663,7 @@ start_table_lock_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_lock_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); PFS_TL_LOCK_TYPE lock_type; @@ -3068,7 +3065,12 @@ start_socket_wait_v1(PSI_socket_locker_state *state, if (flag_thread_instrumentation) { - PFS_thread *pfs_thread= pfs_socket->m_thread_owner; + /* + Do not use pfs_socket->m_thread_owner here, + as different threads may use concurrently the same socket, + for example during a KILL. + */ + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); if (unlikely(pfs_thread == NULL)) return NULL; @@ -3436,6 +3438,8 @@ static void end_idle_wait_v1(PSI_idle_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3517,6 +3521,8 @@ static void end_mutex_wait_v1(PSI_mutex_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3596,6 +3602,8 @@ static void end_rwlock_rdwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3668,6 +3676,8 @@ static void end_rwlock_wrwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3732,6 +3742,8 @@ static void end_cond_wait_v1(PSI_cond_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3826,6 +3838,8 @@ static void end_table_io_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3895,6 +3909,8 @@ static void end_table_lock_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3935,9 +3951,11 @@ static PSI_file* end_file_open_wait_v1(PSI_file_locker *locker, switch (state->m_operation) { case PSI_FILE_STAT: + case PSI_FILE_RENAME: break; case PSI_FILE_STREAM_OPEN: case PSI_FILE_CREATE: + case PSI_FILE_OPEN: if (result != NULL) { PFS_file_class *klass= reinterpret_cast<PFS_file_class*> (state->m_class); @@ -3948,7 +3966,6 @@ static PSI_file* end_file_open_wait_v1(PSI_file_locker *locker, state->m_file= reinterpret_cast<PSI_file*> (pfs_file); } break; - case PSI_FILE_OPEN: default: DBUG_ASSERT(false); break; @@ -4143,6 +4160,8 @@ static void end_file_wait_v1(PSI_file_locker *locker, if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -5070,6 +5089,8 @@ static void end_socket_wait_v1(PSI_socket_locker *locker, size_t byte_count) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index bf42ada8e67..c4079001b17 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -volatile uint32 digest_index; +volatile uint32 PFS_ALIGNED digest_monotonic_index; bool digest_full= false; LF_HASH digest_hash; @@ -63,7 +63,7 @@ int init_digest(const PFS_global_param *param) */ digest_max= param->m_digest_sizing; digest_lost= 0; - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; if (digest_max == 0) @@ -105,6 +105,9 @@ int init_digest(const PFS_global_param *param) + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Set record[0] as allocated. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + return 0; } @@ -207,9 +210,10 @@ find_or_create_digest(PFS_thread *thread, memcpy(hash_key.m_schema_name, schema_name, schema_name_length); int res; - ulong safe_index; uint retry_count= 0; const uint retry_max= 3; + size_t safe_index; + size_t attempts= 0; PFS_statements_digest_stat **entry; PFS_statements_digest_stat *pfs= NULL; @@ -245,55 +249,71 @@ search: return & pfs->m_stat; } - safe_index= PFS_atomic::add_u32(& digest_index, 1); - if (safe_index >= digest_max) + while (++attempts <= digest_max) { - /* The digest array is now full. */ - digest_full= true; - pfs= &statements_digest_stat_array[0]; - - if (pfs->m_first_seen == 0) - pfs->m_first_seen= now; - pfs->m_last_seen= now; - return & pfs->m_stat; - } - - /* Add a new record in digest stat array. */ - pfs= &statements_digest_stat_array[safe_index]; - - /* Copy digest hash/LF Hash search key. */ - memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); - - /* - Copy digest storage to statement_digest_stat_array so that it could be - used later to generate digest text. - */ - pfs->m_digest_storage.copy(digest_storage); - - pfs->m_first_seen= now; - pfs->m_last_seen= now; + safe_index= PFS_atomic::add_u32(& digest_monotonic_index, 1) % digest_max; + if (safe_index == 0) + { + /* Record [0] is reserved. */ + continue; + } - res= lf_hash_insert(&digest_hash, pins, &pfs); - if (likely(res == 0)) - { - return & pfs->m_stat; - } + /* Add a new record in digest stat array. */ + DBUG_ASSERT(safe_index < digest_max); + pfs= &statements_digest_stat_array[safe_index]; - if (res > 0) - { - /* Duplicate insert by another thread */ - if (++retry_count > retry_max) + if (pfs->m_lock.is_free()) { - /* Avoid infinite loops */ - digest_lost++; - return NULL; + if (pfs->m_lock.free_to_dirty()) + { + /* Copy digest hash/LF Hash search key. */ + memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); + + /* + Copy digest storage to statement_digest_stat_array so that it could be + used later to generate digest text. + */ + pfs->m_digest_storage.copy(digest_storage); + + pfs->m_first_seen= now; + pfs->m_last_seen= now; + + res= lf_hash_insert(&digest_hash, pins, &pfs); + if (likely(res == 0)) + { + pfs->m_lock.dirty_to_allocated(); + return & pfs->m_stat; + } + + pfs->m_lock.dirty_to_free(); + + if (res > 0) + { + /* Duplicate insert by another thread */ + if (++retry_count > retry_max) + { + /* Avoid infinite loops */ + digest_lost++; + return NULL; + } + goto search; + } + + /* OOM in lf_hash_insert */ + digest_lost++; + return NULL; + } } - goto search; } - /* OOM in lf_hash_insert */ - digest_lost++; - return NULL; + /* The digest array is now full. */ + digest_full= true; + pfs= &statements_digest_stat_array[0]; + + if (pfs->m_first_seen == 0) + pfs->m_first_seen= now; + pfs->m_last_seen= now; + return & pfs->m_stat; } void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) @@ -320,10 +340,12 @@ void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) void PFS_statements_digest_stat::reset_data(unsigned char *token_array, uint length) { + m_lock.set_dirty(); m_digest_storage.reset(token_array, length); m_stat.reset(); m_first_seen= 0; m_last_seen= 0; + m_lock.dirty_to_free(); } void PFS_statements_digest_stat::reset_index(PFS_thread *thread) @@ -351,11 +373,14 @@ void reset_esms_by_digest() statements_digest_stat_array[index].reset_data(statements_digest_token_array + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Mark record[0] as allocated again. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + /* Reset index which indicates where the next calculated digest information to be inserted in statements_digest_stat_array. */ - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; } diff --git a/storage/perfschema/pfs_digest.h b/storage/perfschema/pfs_digest.h index 76d6c33d984..429a9f4250a 100644 --- a/storage/perfschema/pfs_digest.h +++ b/storage/perfschema/pfs_digest.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ struct PFS_digest_key /** A statement digest stat record. */ struct PFS_ALIGNED PFS_statements_digest_stat { + /** Internal lock. */ + pfs_lock m_lock; + /** Digest Schema + MD5 Hash. */ PFS_digest_key m_digest_key; diff --git a/storage/perfschema/pfs_lock.h b/storage/perfschema/pfs_lock.h index 09efecd1c5f..be84d0f7fb4 100644 --- a/storage/perfschema/pfs_lock.h +++ b/storage/perfschema/pfs_lock.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -156,6 +156,18 @@ struct pfs_lock } /** + Initialize a lock to dirty. + */ + void set_dirty(void) + { + /* Do not set the version to 0, read the previous value. */ + uint32 copy= PFS_atomic::load_u32(&m_version_state); + /* Increment the version, set the DIRTY state */ + uint32 new_val= (copy & VERSION_MASK) + VERSION_INC + PFS_LOCK_DIRTY; + PFS_atomic::store_u32(&m_version_state, new_val); + } + + /** Execute a dirty to free transition. This transition should be executed by the writer that owns the record. */ diff --git a/storage/perfschema/table_esms_by_digest.cc b/storage/perfschema/table_esms_by_digest.cc index 80fa4077281..5a3145b91d1 100644 --- a/storage/perfschema/table_esms_by_digest.cc +++ b/storage/perfschema/table_esms_by_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -238,11 +238,14 @@ int table_esms_by_digest::rnd_next(void) m_pos.next()) { digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - m_next_pos.set_after(&m_pos); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + m_next_pos.set_after(&m_pos); + return 0; + } } } @@ -260,10 +263,13 @@ table_esms_by_digest::rnd_pos(const void *pos) set_position(pos); digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + return 0; + } } return HA_ERR_RECORD_DELETED; diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 6121fac098f..7883e8070fe 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -27,6 +27,8 @@ #include "stub_print_error.h" #include "stub_pfs_defaults.h" +void unload_performance_schema(); + /* test helpers, to simulate the setup */ void setup_thread(PSI_thread *t, bool enabled) @@ -126,7 +128,7 @@ void test_bootstrap() psi_2= boot->get_interface(PSI_VERSION_2); ok(psi_2 == NULL, "version 2"); - shutdown_performance_schema(); + unload_performance_schema(); } /* @@ -183,6 +185,28 @@ PSI * load_perfschema() return (PSI*) psi; } +void unload_performance_schema() +{ + cleanup_table_share(); + cleanup_instruments(); + cleanup_sync_class(); + cleanup_thread_class(); + cleanup_table_share(); + cleanup_file_class(); + cleanup_stage_class(); + cleanup_statement_class(); + cleanup_socket_class(); + cleanup_events_waits_history_long(); + cleanup_events_stages_history_long(); + cleanup_events_statements_history_long(); + cleanup_table_share_hash(); + cleanup_file_hash(); + cleanup_digest(); + PFS_atomic::cleanup(); + + shutdown_performance_schema(); +} + void test_bad_registration() { PSI *psi; @@ -581,8 +605,7 @@ void test_bad_registration() psi->register_socket("X", bad_socket_3, 1); ok(dummy_socket_key == 2, "assigned key"); - - shutdown_performance_schema(); + unload_performance_schema(); } void test_init_disabled() @@ -1016,7 +1039,7 @@ void test_init_disabled() socket_A1= psi->init_socket(99, NULL, NULL, 0); ok(socket_A1 == NULL, "broken socket key not instrumented"); - shutdown_performance_schema(); + unload_performance_schema(); } void test_locker_disabled() @@ -1316,13 +1339,15 @@ void test_locker_disabled() /* Pretend the socket does not have a thread owner */ /* ---------------------------------------------- */ + psi->delete_current_thread(); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); ok(socket_A1 != NULL, "instrumented"); /* Socket thread owner has not been set */ socket_locker= psi->start_socket_wait(&socket_state, socket_A1, PSI_SOCKET_SEND, 12, "foo.cc", 12); - ok(socket_locker == NULL, "no locker (no thread owner)"); - + ok(socket_locker != NULL, "locker (owner not used)"); + psi->end_socket_wait(socket_locker, 10); + /* Pretend the running thread is not instrumented */ /* ---------------------------------------------- */ @@ -1350,7 +1375,7 @@ void test_locker_disabled() socket_locker= psi->start_socket_wait(&socket_state, socket_A1, PSI_SOCKET_SEND, 12, "foo.cc", 12); ok(socket_locker == NULL, "no locker"); - shutdown_performance_schema(); + unload_performance_schema(); } void test_file_instrumentation_leak() @@ -1437,7 +1462,7 @@ void test_file_instrumentation_leak() file_locker= psi->get_thread_file_descriptor_locker(&file_state, (File) 12, PSI_FILE_WRITE); ok(file_locker == NULL, "no locker, no leak"); - shutdown_performance_schema(); + unload_performance_schema(); } void test_enabled() @@ -1473,7 +1498,7 @@ void test_enabled() { & cond_key_B, "C-B", 0} }; - shutdown_performance_schema(); + unload_performance_schema(); #endif } @@ -1643,5 +1668,5 @@ int main(int argc, char **argv) MY_INIT(argv[0]); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_account-oom-t.cc b/storage/perfschema/unittest/pfs_account-oom-t.cc index f715325401d..59f17a10b64 100644 --- a/storage/perfschema/unittest/pfs_account-oom-t.cc +++ b/storage/perfschema/unittest/pfs_account-oom-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -112,6 +112,6 @@ int main(int, char **) MY_INIT("pfs_account-oom-t"); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_connect_attr-t.cc b/storage/perfschema/unittest/pfs_connect_attr-t.cc index 7bee1d063a1..ecf790eeede 100644 --- a/storage/perfschema/unittest/pfs_connect_attr-t.cc +++ b/storage/perfschema/unittest/pfs_connect_attr-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -341,5 +341,5 @@ int main(int, char **) diag("skipping the cp1251 tests : missing character set"); plan(59 + (cs_cp1251 ? 10 : 0)); do_all_tests(); - return 0; + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_host-oom-t.cc b/storage/perfschema/unittest/pfs_host-oom-t.cc index 56597bdda74..d52022ce754 100644 --- a/storage/perfschema/unittest/pfs_host-oom-t.cc +++ b/storage/perfschema/unittest/pfs_host-oom-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -112,6 +112,6 @@ int main(int, char **) MY_INIT("pfs_host-oom-t"); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_instr-oom-t.cc b/storage/perfschema/unittest/pfs_instr-oom-t.cc index 25caaa6241f..d5c948b41a7 100644 --- a/storage/perfschema/unittest/pfs_instr-oom-t.cc +++ b/storage/perfschema/unittest/pfs_instr-oom-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -355,6 +355,11 @@ void test_oom() rc= init_instruments(& param); ok(rc == 1, "oom (per thread wait)"); + cleanup_sync_class(); + cleanup_thread_class(); + cleanup_file_class(); + cleanup_instruments(); + param.m_enabled= true; param.m_mutex_class_sizing= 0; param.m_rwlock_class_sizing= 0; @@ -432,6 +437,8 @@ void test_oom() init_event_name_sizing(& param); rc= init_instruments(& param); ok(rc == 1, "oom (thread stages history sizing)"); + + cleanup_thread_class(); cleanup_instruments(); param.m_enabled= true; @@ -467,6 +474,9 @@ void test_oom() init_event_name_sizing(& param); rc= init_instruments(& param); ok(rc == 1, "oom (per thread stages)"); + + cleanup_stage_class(); + cleanup_thread_class(); cleanup_instruments(); param.m_enabled= true; @@ -502,6 +512,8 @@ void test_oom() init_event_name_sizing(& param); rc= init_instruments(& param); ok(rc == 1, "oom (thread statements history sizing)"); + + cleanup_thread_class(); cleanup_instruments(); param.m_enabled= true; @@ -537,6 +549,9 @@ void test_oom() init_event_name_sizing(& param); rc= init_instruments(& param); ok(rc == 1, "oom (per thread statements)"); + + cleanup_statement_class(); + cleanup_thread_class(); cleanup_instruments(); param.m_enabled= true; @@ -572,6 +587,8 @@ void test_oom() init_event_name_sizing(& param); rc= init_instruments(& param); ok(rc == 1, "oom (global waits)"); + + cleanup_sync_class(); cleanup_instruments(); param.m_enabled= true; @@ -609,8 +626,10 @@ void test_oom() ok(rc == 0, "init stage class"); rc= init_instruments(& param); ok(rc == 1, "oom (global stages)"); - cleanup_instruments(); + + cleanup_sync_class(); cleanup_stage_class(); + cleanup_instruments(); param.m_enabled= true; param.m_mutex_class_sizing= 10; @@ -647,8 +666,10 @@ void test_oom() ok(rc == 0, "init statement class"); rc= init_instruments(& param); ok(rc == 1, "oom (global statements)"); - cleanup_instruments(); + + cleanup_sync_class(); cleanup_statement_class(); + cleanup_instruments(); } void do_all_tests() @@ -666,6 +687,6 @@ int main(int argc, char **argv) MY_INIT(argv[0]); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_instr-t.cc b/storage/perfschema/unittest/pfs_instr-t.cc index 81df38340df..430b286bf88 100644 --- a/storage/perfschema/unittest/pfs_instr-t.cc +++ b/storage/perfschema/unittest/pfs_instr-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -23,10 +23,11 @@ #include <memory.h> +PFS_global_param param; + void test_no_instruments() { int rc; - PFS_global_param param; memset(& param, 0xFF, sizeof(param)); param.m_enabled= true; @@ -86,7 +87,6 @@ void test_no_instances() PFS_file *file; PFS_socket *socket; PFS_table *table; - PFS_global_param param; memset(& param, 0xFF, sizeof(param)); param.m_enabled= true; @@ -227,7 +227,6 @@ void test_with_instances() PFS_socket *socket_2; PFS_table *table_1; PFS_table *table_2; - PFS_global_param param; memset(& param, 0xFF, sizeof(param)); param.m_enabled= true; @@ -418,6 +417,6 @@ int main(int argc, char **argv) MY_INIT(argv[0]); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_instr_class-oom-t.cc b/storage/perfschema/unittest/pfs_instr_class-oom-t.cc index 1b098ee356e..1a23b761cc5 100644 --- a/storage/perfschema/unittest/pfs_instr_class-oom-t.cc +++ b/storage/perfschema/unittest/pfs_instr_class-oom-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -68,6 +68,6 @@ int main(int argc, char **argv) MY_INIT(argv[0]); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_instr_class-t.cc b/storage/perfschema/unittest/pfs_instr_class-t.cc index 06e3200a8d2..62cc7bbd527 100644 --- a/storage/perfschema/unittest/pfs_instr_class-t.cc +++ b/storage/perfschema/unittest/pfs_instr_class-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -674,5 +674,5 @@ int main(int argc, char **argv) MY_INIT(argv[0]); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_misc-t.cc b/storage/perfschema/unittest/pfs_misc-t.cc index a4b11b9a727..eed9039dfb2 100644 --- a/storage/perfschema/unittest/pfs_misc-t.cc +++ b/storage/perfschema/unittest/pfs_misc-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -67,6 +67,6 @@ int main(int, char **) MY_INIT("pfs_misc-t"); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_timer-t.cc b/storage/perfschema/unittest/pfs_timer-t.cc index 0a2cc63f30e..2ac63d0abd3 100644 --- a/storage/perfschema/unittest/pfs_timer-t.cc +++ b/storage/perfschema/unittest/pfs_timer-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -119,6 +119,6 @@ int main(int, char **) MY_INIT("pfs_timer-t"); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/perfschema/unittest/pfs_user-oom-t.cc b/storage/perfschema/unittest/pfs_user-oom-t.cc index 6c936ba8ffb..861934ebd1e 100644 --- a/storage/perfschema/unittest/pfs_user-oom-t.cc +++ b/storage/perfschema/unittest/pfs_user-oom-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -111,6 +111,6 @@ int main(int, char **) MY_INIT("pfs_user-oom-t"); do_all_tests(); my_end(0); - return exit_status(); + return (exit_status()); } diff --git a/storage/sphinx/ha_sphinx.cc b/storage/sphinx/ha_sphinx.cc index ff3b63465eb..3f035c7bc7d 100644 --- a/storage/sphinx/ha_sphinx.cc +++ b/storage/sphinx/ha_sphinx.cc @@ -911,7 +911,7 @@ bool sphinx_show_status ( THD * thd ) } // show last error or warning (either in addition to stats, or on their own) - if ( pTls && pTls->m_pHeadTable && pTls->m_pHeadTable->m_tStats.m_sLastMessage && pTls->m_pHeadTable->m_tStats.m_sLastMessage[0] ) + if ( pTls && pTls->m_pHeadTable && pTls->m_pHeadTable->m_tStats.m_sLastMessage[0] ) { const char * sMessageType = pTls->m_pHeadTable->m_tStats.m_bLastError ? "error" : "warning"; diff --git a/storage/sphinx/mysql-test/sphinx/suite.pm b/storage/sphinx/mysql-test/sphinx/suite.pm index e4c3c1b9f74..fc127ffd6c0 100644 --- a/storage/sphinx/mysql-test/sphinx/suite.pm +++ b/storage/sphinx/mysql-test/sphinx/suite.pm @@ -16,13 +16,26 @@ sub locate_sphinx_binary { for (@list) { return $_ if -x $_; } } -# Look for Sphinx binaries. +# Look for Sphinx binaries my $exe_sphinx_indexer = &locate_sphinx_binary('indexer'); + +unless ($exe_sphinx_indexer) { + mtr_report("Sphinx 'indexer' binary not found, sphinx suite will be skipped"); + return "No Sphinx"; +} my $exe_sphinx_searchd = &locate_sphinx_binary('searchd'); -return "No Sphinx" unless $exe_sphinx_indexer and $exe_sphinx_searchd; -return "No SphinxSE" unless $ENV{HA_SPHINX_SO} or - $::mysqld_variables{'sphinx'} eq "ON"; +unless ($exe_sphinx_searchd) { + mtr_report("Sphinx 'searchd' binary not found, sphinx suite will be skipped"); + return "No Sphinx"; +} + +# Check for Sphinx engine + +unless ($ENV{HA_SPHINX_SO} or $::mysqld_variables{'sphinx'} eq "ON") { + mtr_report("Sphinx engine not found, sphinx suite will be skipped"); + return "No SphinxSE"; +} { local $_ = `"$exe_sphinx_searchd" --help`; diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index ead335c07ea..66f558ecd91 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -517,7 +517,6 @@ int spider_free_share_alloc( ) { int roop_count; DBUG_ENTER("spider_free_share_alloc"); - if (share->dbton_bitmap) { for (roop_count = SPIDER_DBTON_SIZE - 1; roop_count >= 0; roop_count--) { diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 3017d3cf68e..90a8f4974c4 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(TOKUDB_VERSION 5.6.34-79.1) +SET(TOKUDB_VERSION 5.6.35-80.0) # PerconaFT only supports x86-64 and cmake-2.8.9+ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index 30a8710d7aa..ad9ecb1d074 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -651,10 +651,8 @@ void toku_ftnode_clone_callback(void *value_data, // set new pair attr if necessary if (node->height == 0) { *new_attr = make_ftnode_pair_attr(node); - for (int i = 0; i < node->n_children; i++) { - BLB(node, i)->logical_rows_delta = 0; - BLB(cloned_node, i)->logical_rows_delta = 0; - } + node->logical_rows_delta = 0; + cloned_node->logical_rows_delta = 0; } else { new_attr->is_valid = false; } @@ -702,6 +700,10 @@ void toku_ftnode_flush_callback(CACHEFILE UU(cachefile), if (ftnode->height == 0) { FT_STATUS_INC(FT_FULL_EVICTIONS_LEAF, 1); FT_STATUS_INC(FT_FULL_EVICTIONS_LEAF_BYTES, node_size); + if (!ftnode->dirty) { + toku_ft_adjust_logical_row_count( + ft, -ftnode->logical_rows_delta); + } } else { FT_STATUS_INC(FT_FULL_EVICTIONS_NONLEAF, 1); FT_STATUS_INC(FT_FULL_EVICTIONS_NONLEAF_BYTES, node_size); @@ -714,11 +716,12 @@ void toku_ftnode_flush_callback(CACHEFILE UU(cachefile), BASEMENTNODE bn = BLB(ftnode, i); toku_ft_decrease_stats(&ft->in_memory_stats, bn->stat64_delta); - if (!ftnode->dirty) - toku_ft_adjust_logical_row_count( - ft, -bn->logical_rows_delta); } } + if (!ftnode->dirty) { + toku_ft_adjust_logical_row_count( + ft, -ftnode->logical_rows_delta); + } } } toku_ftnode_free(&ftnode); @@ -944,8 +947,6 @@ int toku_ftnode_pe_callback(void *ftnode_pv, basements_to_destroy[num_basements_to_destroy++] = bn; toku_ft_decrease_stats(&ft->in_memory_stats, bn->stat64_delta); - toku_ft_adjust_logical_row_count(ft, - -bn->logical_rows_delta); set_BNULL(node, i); BP_STATE(node, i) = PT_ON_DISK; num_partial_evictions++; @@ -2652,7 +2653,7 @@ static std::unique_ptr<char[], decltype(&toku_free)> toku_file_get_parent_dir( return result; } -static bool toku_create_subdirs_if_needed(const char *path) { +bool toku_create_subdirs_if_needed(const char *path) { static const mode_t dir_mode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IXOTH; @@ -4563,6 +4564,8 @@ int toku_ft_rename_iname(DB_TXN *txn, bs_new_name); } + if (!toku_create_subdirs_if_needed(new_iname_full.get())) + return get_error_errno(); r = toku_os_rename(old_iname_full.get(), new_iname_full.get()); if (r != 0) return r; diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.h b/storage/tokudb/PerconaFT/ft/ft-ops.h index 70cf045d43c..df8ffe287df 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.h +++ b/storage/tokudb/PerconaFT/ft/ft-ops.h @@ -288,3 +288,8 @@ void toku_ft_set_direct_io(bool direct_io_on); void toku_ft_set_compress_buffers_before_eviction(bool compress_buffers); void toku_note_deserialized_basement_node(bool fixed_key_size); + +// Creates all directories for the path if necessary, +// returns true if all dirs are created successfully or +// all dirs exist, false otherwise. +bool toku_create_subdirs_if_needed(const char* path); diff --git a/storage/tokudb/PerconaFT/ft/logger/recover.cc b/storage/tokudb/PerconaFT/ft/logger/recover.cc index a9c30c0e37a..9eaa56bdc53 100644 --- a/storage/tokudb/PerconaFT/ft/logger/recover.cc +++ b/storage/tokudb/PerconaFT/ft/logger/recover.cc @@ -987,7 +987,8 @@ static int toku_recover_frename(struct logtype_frename *l, RECOVER_ENV renv) { return 1; if (old_exist && !new_exist && - (toku_os_rename(old_iname_full.get(), new_iname_full.get()) == -1 || + (!toku_create_subdirs_if_needed(new_iname_full.get()) || + toku_os_rename(old_iname_full.get(), new_iname_full.get()) == -1 || toku_fsync_directory(old_iname_full.get()) == -1 || toku_fsync_directory(new_iname_full.get()) == -1)) return 1; diff --git a/storage/tokudb/PerconaFT/ft/node.cc b/storage/tokudb/PerconaFT/ft/node.cc index 12e5fda226e..07309ff7f94 100644 --- a/storage/tokudb/PerconaFT/ft/node.cc +++ b/storage/tokudb/PerconaFT/ft/node.cc @@ -386,7 +386,8 @@ static void bnc_apply_messages_to_basement_node( const pivot_bounds & bounds, // contains pivot key bounds of this basement node txn_gc_info *gc_info, - bool *msgs_applied) { + bool *msgs_applied, + int64_t* logical_rows_delta) { int r; NONLEAF_CHILDINFO bnc = BNC(ancestor, childnum); @@ -394,7 +395,6 @@ static void bnc_apply_messages_to_basement_node( // apply messages from this buffer STAT64INFO_S stats_delta = {0, 0}; uint64_t workdone_this_ancestor = 0; - int64_t logical_rows_delta = 0; uint32_t stale_lbi, stale_ube; if (!bn->stale_ancestor_messages_applied) { @@ -470,7 +470,7 @@ static void bnc_apply_messages_to_basement_node( gc_info, &workdone_this_ancestor, &stats_delta, - &logical_rows_delta); + logical_rows_delta); } } else if (stale_lbi == stale_ube) { // No stale messages to apply, we just apply fresh messages, and mark @@ -482,7 +482,7 @@ static void bnc_apply_messages_to_basement_node( .gc_info = gc_info, .workdone = &workdone_this_ancestor, .stats_to_update = &stats_delta, - .logical_rows_delta = &logical_rows_delta}; + .logical_rows_delta = logical_rows_delta}; if (fresh_ube - fresh_lbi > 0) *msgs_applied = true; r = bnc->fresh_message_tree @@ -503,7 +503,7 @@ static void bnc_apply_messages_to_basement_node( .gc_info = gc_info, .workdone = &workdone_this_ancestor, .stats_to_update = &stats_delta, - .logical_rows_delta = &logical_rows_delta}; + .logical_rows_delta = logical_rows_delta}; r = bnc->stale_message_tree .iterate_on_range<struct iterate_do_bn_apply_msg_extra, @@ -521,8 +521,6 @@ static void bnc_apply_messages_to_basement_node( if (stats_delta.numbytes || stats_delta.numrows) { toku_ft_update_stats(&t->ft->in_memory_stats, stats_delta); } - toku_ft_adjust_logical_row_count(t->ft, logical_rows_delta); - bn->logical_rows_delta += logical_rows_delta; } static void @@ -536,6 +534,7 @@ apply_ancestors_messages_to_bn( bool* msgs_applied ) { + int64_t logical_rows_delta = 0; BASEMENTNODE curr_bn = BLB(node, childnum); const pivot_bounds curr_bounds = bounds.next_bounds(node, childnum); for (ANCESTORS curr_ancestors = ancestors; curr_ancestors; curr_ancestors = curr_ancestors->next) { @@ -548,13 +547,16 @@ apply_ancestors_messages_to_bn( curr_ancestors->childnum, curr_bounds, gc_info, - msgs_applied + msgs_applied, + &logical_rows_delta ); // We don't want to check this ancestor node again if the // next time we query it, the msn hasn't changed. curr_bn->max_msn_applied = curr_ancestors->node->max_msn_applied_to_node_on_disk; } } + toku_ft_adjust_logical_row_count(t->ft, logical_rows_delta); + node->logical_rows_delta += logical_rows_delta; // At this point, we know all the stale messages above this // basement node have been applied, and any new messages will be // fresh, so we don't need to look at stale messages for this diff --git a/storage/tokudb/PerconaFT/ft/node.h b/storage/tokudb/PerconaFT/ft/node.h index 52eefec0936..db189e36d59 100644 --- a/storage/tokudb/PerconaFT/ft/node.h +++ b/storage/tokudb/PerconaFT/ft/node.h @@ -157,36 +157,49 @@ private: // TODO: class me up struct ftnode { - MSN max_msn_applied_to_node_on_disk; // max_msn_applied that will be written to disk + // max_msn_applied that will be written to disk + MSN max_msn_applied_to_node_on_disk; unsigned int flags; - BLOCKNUM blocknum; // Which block number is this node? - int layout_version; // What version of the data structure? - int layout_version_original; // different (<) from layout_version if upgraded from a previous version (useful for debugging) - int layout_version_read_from_disk; // transient, not serialized to disk, (useful for debugging) - uint32_t build_id; // build_id (svn rev number) of software that wrote this node to disk - int height; /* height is always >= 0. 0 for leaf, >0 for nonleaf. */ - int dirty; + // Which block number is this node? + BLOCKNUM blocknum; + // What version of the data structure? + int layout_version; + // different (<) from layout_version if upgraded from a previous version + // (useful for debugging) + int layout_version_original; + // transient, not serialized to disk, (useful for debugging) + int layout_version_read_from_disk; + // build_id (svn rev number) of software that wrote this node to disk + uint32_t build_id; + // height is always >= 0. 0 for leaf, >0 for nonleaf. + int height; + int dirty; uint32_t fullhash; + // current count of rows add or removed as a result of message application + // to this node as a basement, irrelevant for internal nodes, gets reset + // when node is undirtied. Used to back out tree scoped LRC id node is + // evicted but not persisted + int64_t logical_rows_delta; - // for internal nodes, if n_children==fanout+1 then the tree needs to be rebalanced. - // for leaf nodes, represents number of basement nodes + // for internal nodes, if n_children==fanout+1 then the tree needs to be + // rebalanced. for leaf nodes, represents number of basement nodes int n_children; ftnode_pivot_keys pivotkeys; - // What's the oldest referenced xid that this node knows about? The real oldest - // referenced xid might be younger, but this is our best estimate. We use it - // as a heuristic to transition provisional mvcc entries from provisional to - // committed (from implicity committed to really committed). + // What's the oldest referenced xid that this node knows about? The real + // oldest referenced xid might be younger, but this is our best estimate. + // We use it as a heuristic to transition provisional mvcc entries from + // provisional to committed (from implicity committed to really committed). // - // A better heuristic would be the oldest live txnid, but we use this since it - // still works well most of the time, and its readily available on the inject - // code path. + // A better heuristic would be the oldest live txnid, but we use this since + // it still works well most of the time, and its readily available on the + // inject code path. TXNID oldest_referenced_xid_known; // array of size n_children, consisting of ftnode partitions - // each one is associated with a child - // for internal nodes, the ith partition corresponds to the ith message buffer - // for leaf nodes, the ith partition corresponds to the ith basement node + // each one is associated with a child for internal nodes, the ith + // partition corresponds to the ith message buffer for leaf nodes, the ith + // partition corresponds to the ith basement node struct ftnode_partition *bp; struct ctpair *ct_pair; }; @@ -199,7 +212,6 @@ struct ftnode_leaf_basement_node { MSN max_msn_applied; // max message sequence number applied bool stale_ancestor_messages_applied; STAT64INFO_S stat64_delta; // change in stat64 counters since basement was last written to disk - int64_t logical_rows_delta; }; typedef struct ftnode_leaf_basement_node *BASEMENTNODE; diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc index 5914f8a1050..56876b474d4 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc @@ -996,7 +996,6 @@ BASEMENTNODE toku_clone_bn(BASEMENTNODE orig_bn) { bn->seqinsert = orig_bn->seqinsert; bn->stale_ancestor_messages_applied = orig_bn->stale_ancestor_messages_applied; bn->stat64_delta = orig_bn->stat64_delta; - bn->logical_rows_delta = orig_bn->logical_rows_delta; bn->data_buffer.clone(&orig_bn->data_buffer); return bn; } @@ -1007,7 +1006,6 @@ BASEMENTNODE toku_create_empty_bn_no_buffer(void) { bn->seqinsert = 0; bn->stale_ancestor_messages_applied = false; bn->stat64_delta = ZEROSTATS; - bn->logical_rows_delta = 0; bn->data_buffer.init_zero(); return bn; } @@ -1432,6 +1430,7 @@ static FTNODE alloc_ftnode_for_deserialize(uint32_t fullhash, BLOCKNUM blocknum) node->fullhash = fullhash; node->blocknum = blocknum; node->dirty = 0; + node->logical_rows_delta = 0; node->bp = nullptr; node->oldest_referenced_xid_known = TXNID_NONE; return node; diff --git a/storage/tokudb/PerconaFT/ft/txn/roll.cc b/storage/tokudb/PerconaFT/ft/txn/roll.cc index 9f3977743a0..4f374d62173 100644 --- a/storage/tokudb/PerconaFT/ft/txn/roll.cc +++ b/storage/tokudb/PerconaFT/ft/txn/roll.cc @@ -227,7 +227,8 @@ int toku_rollback_frename(BYTESTRING old_iname, return 1; if (!old_exist && new_exist && - (toku_os_rename(new_iname_full.get(), old_iname_full.get()) == -1 || + (!toku_create_subdirs_if_needed(old_iname_full.get()) || + toku_os_rename(new_iname_full.get(), old_iname_full.get()) == -1 || toku_fsync_directory(new_iname_full.get()) == -1 || toku_fsync_directory(old_iname_full.get()) == -1)) return 1; diff --git a/storage/tokudb/PerconaFT/util/dmt.h b/storage/tokudb/PerconaFT/util/dmt.h index 71cde8814ab..99be296d0e9 100644 --- a/storage/tokudb/PerconaFT/util/dmt.h +++ b/storage/tokudb/PerconaFT/util/dmt.h @@ -589,7 +589,6 @@ private: void convert_from_tree_to_array(void); - __attribute__((nonnull(2,5))) void delete_internal(subtree *const subtreep, const uint32_t idx, subtree *const subtree_replace, subtree **const rebalance_subtree); template<typename iterate_extra_t, @@ -627,16 +626,12 @@ private: __attribute__((nonnull)) void rebalance(subtree *const subtree); - __attribute__((nonnull(3))) static void copyout(uint32_t *const outlen, dmtdata_t *const out, const dmt_node *const n); - __attribute__((nonnull(3))) static void copyout(uint32_t *const outlen, dmtdata_t **const out, dmt_node *const n); - __attribute__((nonnull(4))) static void copyout(uint32_t *const outlen, dmtdata_t *const out, const uint32_t len, const dmtdata_t *const stored_value_ptr); - __attribute__((nonnull(4))) static void copyout(uint32_t *const outlen, dmtdata_t **const out, const uint32_t len, dmtdata_t *const stored_value_ptr); template<typename dmtcmp_t, diff --git a/storage/tokudb/PerconaFT/util/omt.h b/storage/tokudb/PerconaFT/util/omt.h index 799ed0eae7c..c7ed2ca546f 100644 --- a/storage/tokudb/PerconaFT/util/omt.h +++ b/storage/tokudb/PerconaFT/util/omt.h @@ -284,7 +284,6 @@ public: * By taking ownership of the array, we save a malloc and memcpy, * and possibly a free (if the caller is done with the array). */ - __attribute__((nonnull)) void create_steal_sorted_array(omtdata_t **const values, const uint32_t numvalues, const uint32_t new_capacity); /** @@ -667,7 +666,6 @@ private: void set_at_internal(const subtree &subtree, const omtdata_t &value, const uint32_t idx); - __attribute__((nonnull(2,5))) void delete_internal(subtree *const subtreep, const uint32_t idx, omt_node *const copyn, subtree **const rebalance_subtree); template<typename iterate_extra_t, diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index cd5fe52e214..b3b6edd9803 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -29,6 +29,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "tokudb_status.h" #include "tokudb_card.h" #include "ha_tokudb.h" +#include "sql_db.h" #if TOKU_INCLUDE_EXTENDED_KEYS @@ -6123,8 +6124,6 @@ int ha_tokudb::info(uint flag) { stats.deleted = 0; if (!(flag & HA_STATUS_NO_LOCK)) { uint64_t num_rows = 0; - TOKU_DB_FRAGMENTATION_S frag_info; - memset(&frag_info, 0, sizeof frag_info); error = txn_begin(db_env, NULL, &txn, DB_READ_UNCOMMITTED, ha_thd()); if (error) { @@ -6141,11 +6140,6 @@ int ha_tokudb::info(uint flag) { } else { goto cleanup; } - error = share->file->get_fragmentation(share->file, &frag_info); - if (error) { - goto cleanup; - } - stats.delete_length = frag_info.unused_bytes; DB_BTREE_STAT64 dict_stats; error = share->file->stat64(share->file, txn, &dict_stats); @@ -6157,6 +6151,7 @@ int ha_tokudb::info(uint flag) { stats.update_time = dict_stats.bt_modify_time_sec; stats.check_time = dict_stats.bt_verify_time_sec; stats.data_file_length = dict_stats.bt_dsize; + stats.delete_length = dict_stats.bt_fsize - dict_stats.bt_dsize; if (hidden_primary_key) { // // in this case, we have a hidden primary key, do not @@ -6192,30 +6187,21 @@ int ha_tokudb::info(uint flag) { // // this solution is much simpler than trying to maintain an // accurate number of valid keys at the handlerton layer. - uint curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); + uint curr_num_DBs = + table->s->keys + tokudb_test(hidden_primary_key); for (uint i = 0; i < curr_num_DBs; i++) { // skip the primary key, skip dropped indexes if (i == primary_key || share->key_file[i] == NULL) { continue; } - error = - share->key_file[i]->stat64( - share->key_file[i], - txn, - &dict_stats); + error = share->key_file[i]->stat64( + share->key_file[i], txn, &dict_stats); if (error) { goto cleanup; } stats.index_file_length += dict_stats.bt_dsize; - - error = - share->file->get_fragmentation( - share->file, - &frag_info); - if (error) { - goto cleanup; - } - stats.delete_length += frag_info.unused_bytes; + stats.delete_length += + dict_stats.bt_fsize - dict_stats.bt_dsize; } } @@ -7652,6 +7638,27 @@ int ha_tokudb::delete_table(const char *name) { TOKUDB_HANDLER_DBUG_RETURN(error); } +static bool tokudb_check_db_dir_exist_from_table_name(const char *table_name) { + DBUG_ASSERT(table_name); + bool mysql_dir_exists; + char db_name[FN_REFLEN]; + const char *db_name_begin = strchr(table_name, FN_LIBCHAR); + const char *db_name_end = strrchr(table_name, FN_LIBCHAR); + DBUG_ASSERT(db_name_begin); + DBUG_ASSERT(db_name_end); + DBUG_ASSERT(db_name_begin != db_name_end); + + ++db_name_begin; + size_t db_name_size = db_name_end - db_name_begin; + + DBUG_ASSERT(db_name_size < FN_REFLEN); + + memcpy(db_name, db_name_begin, db_name_size); + db_name[db_name_size] = '\0'; + mysql_dir_exists = (check_db_dir_existence(db_name) == 0); + + return mysql_dir_exists; +} // // renames table from "from" to "to" @@ -7674,15 +7681,33 @@ int ha_tokudb::rename_table(const char *from, const char *to) { TOKUDB_SHARE::drop_share(share); } int error; - error = delete_or_rename_table(from, to, false); - if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && - error == DB_LOCK_NOTGRANTED) { + bool to_db_dir_exist = tokudb_check_db_dir_exist_from_table_name(to); + if (!to_db_dir_exist) { sql_print_error( - "Could not rename table from %s to %s because another transaction " - "has accessed the table. To rename the table, make sure no " - "transactions touch the table.", + "Could not rename table from %s to %s because " + "destination db does not exist", from, to); +#ifndef __WIN__ + /* Small hack. tokudb_check_db_dir_exist_from_table_name calls + * my_access, which sets my_errno on Windows, but doesn't on + * unix. Set it for unix too. + */ + my_errno= errno; +#endif + error= my_errno; + } + else { + error = delete_or_rename_table(from, to, false); + if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && + error == DB_LOCK_NOTGRANTED) { + sql_print_error( + "Could not rename table from %s to %s because another transaction " + "has accessed the table. To rename the table, make sure no " + "transactions touch the table.", + from, + to); + } } TOKUDB_HANDLER_DBUG_RETURN(error); } diff --git a/storage/tokudb/ha_tokudb.h b/storage/tokudb/ha_tokudb.h index 3d7a3a7fa05..4a7e395d0d1 100644 --- a/storage/tokudb/ha_tokudb.h +++ b/storage/tokudb/ha_tokudb.h @@ -816,6 +816,8 @@ public: int index_first(uchar * buf); int index_last(uchar * buf); + bool has_gap_locks() const { return true; } + int rnd_init(bool scan); int rnd_end(); int rnd_next(uchar * buf); diff --git a/storage/tokudb/mysql-test/tokudb/r/dir_per_db_rename_to_nonexisting_schema.result b/storage/tokudb/mysql-test/tokudb/r/dir_per_db_rename_to_nonexisting_schema.result new file mode 100644 index 00000000000..74148bd4e74 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/dir_per_db_rename_to_nonexisting_schema.result @@ -0,0 +1,46 @@ +SET GLOBAL tokudb_dir_per_db=true; +###### +# Tokudb and mysql data dirs are the same, rename to existent db +### +CREATE DATABASE new_db; +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +ALTER TABLE test.t1 RENAME new_db.t1; +The content of "test" directory: +The content of "new_db" directory: +db.opt +t1.frm +t1_main_id.tokudb +t1_status_id.tokudb +DROP DATABASE new_db; +###### +# Tokudb and mysql data dirs are the same, rename to nonexistent db +### +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +CALL mtr.add_suppression("because destination db does not exist"); +ALTER TABLE test.t1 RENAME foo.t1; +ERROR HY000: Error on rename of './test/t1' to './foo/t1' (errno: 2 "No such file or directory") +DROP TABLE t1; +SELECT @@tokudb_data_dir; +@@tokudb_data_dir +NULL +SELECT @@tokudb_dir_per_db; +@@tokudb_dir_per_db +0 +###### +# Tokudb and mysql data dirs are different, rename to existent db +### +CREATE DATABASE new_db; +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +ALTER TABLE test.t1 RENAME new_db.t1; +The content of "test" direcotry: +The content of "new_db" directory: +DROP DATABASE new_db; +###### +# Tokudb and mysql data dirs are different, rename to nonexistent db +### +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +CALL mtr.add_suppression("because destination db does not exist"); +ALTER TABLE test.t1 RENAME foo.t1; +ERROR HY000: Error on rename of './test/t1' to './foo/t1' (errno: 2 "No such file or directory") +DROP TABLE t1; +SET GLOBAL tokudb_dir_per_db=default; diff --git a/storage/tokudb/mysql-test/tokudb/r/gap_lock_error.result b/storage/tokudb/mysql-test/tokudb/r/gap_lock_error.result new file mode 100644 index 00000000000..41e294f7d8d --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/gap_lock_error.result @@ -0,0 +1,469 @@ +CREATE TABLE gap1 (id1 INT, id2 INT, id3 INT, c1 INT, value INT, +PRIMARY KEY (id1, id2, id3), +INDEX i (c1)) ENGINE=tokudb; +CREATE TABLE gap2 like gap1; +CREATE TABLE gap3 (id INT, value INT, +PRIMARY KEY (id), +UNIQUE KEY ui(value)) ENGINE=tokudb; +CREATE TABLE gap4 (id INT, value INT, +PRIMARY KEY (id)) ENGINE=tokudb +PARTITION BY HASH(id) PARTITIONS 2; +insert into gap3 values (1,1), (2,2),(3,3),(4,4),(5,5); +insert into gap4 values (1,1), (2,2),(3,3),(4,4),(5,5); +set session autocommit=0; +select * from gap1 limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where value != 100 limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where id1=1 for update; +id1 id2 id3 c1 value +1 0 2 2 2 +1 0 3 3 3 +select * from gap1 where id1=1 and id2= 1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 != 1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 +between 1 and 3 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 asc +limit 1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 desc +limit 1 for update; +id1 id2 id3 c1 value +select * from gap1 order by id1 asc limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 desc limit 1 for update; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 order by id1 desc, id2 desc, id3 desc +limit 1 for update; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 force index(i) where c1=1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap3 force index(ui) where value=1 for update; +id value +1 1 +select * from gap1 where id1=1 and id2=1 and id3=1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 +order by c1 for update; +id1 id2 id3 c1 value +select * from gap3 where id=1 for update; +id value +1 1 +select * from gap4 where id=1 for update; +id value +1 1 +select * from gap4 where id in (1, 2, 3) for update; +id value +1 1 +2 2 +3 3 +select * from gap4 for update; +id value +2 2 +4 4 +1 1 +3 3 +5 5 +select * from gap4 where id between 3 and 7 for update; +id value +3 3 +4 4 +5 5 +set session autocommit=1; +select * from gap1 limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where value != 100 limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where id1=1 for update; +id1 id2 id3 c1 value +1 0 2 2 2 +1 0 3 3 3 +select * from gap1 where id1=1 and id2= 1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 != 1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 +between 1 and 3 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 asc +limit 1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 desc +limit 1 for update; +id1 id2 id3 c1 value +select * from gap1 order by id1 asc limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 desc limit 1 for update; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 order by id1 desc, id2 desc, id3 desc +limit 1 for update; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 force index(i) where c1=1 for update; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap3 force index(ui) where value=1 for update; +id value +1 1 +select * from gap1 where id1=1 and id2=1 and id3=1 for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) for update; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 +order by c1 for update; +id1 id2 id3 c1 value +select * from gap3 where id=1 for update; +id value +1 1 +select * from gap4 where id=1 for update; +id value +1 1 +select * from gap4 where id in (1, 2, 3) for update; +id value +1 1 +2 2 +3 3 +select * from gap4 for update; +id value +2 2 +4 4 +1 1 +3 3 +5 5 +select * from gap4 where id between 3 and 7 for update; +id value +3 3 +4 4 +5 5 +set session autocommit=0; +select * from gap1 limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where value != 100 limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where id1=1 lock in share mode; +id1 id2 id3 c1 value +1 0 2 2 2 +1 0 3 3 3 +select * from gap1 where id1=1 and id2= 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 != 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 +between 1 and 3 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 asc +limit 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 desc +limit 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 order by id1 asc limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 desc limit 1 lock in share mode; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 order by id1 desc, id2 desc, id3 desc +limit 1 lock in share mode; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 force index(i) where c1=1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap3 force index(ui) where value=1 lock in share mode; +id value +1 1 +select * from gap1 where id1=1 and id2=1 and id3=1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 +order by c1 lock in share mode; +id1 id2 id3 c1 value +select * from gap3 where id=1 lock in share mode; +id value +1 1 +select * from gap4 where id=1 lock in share mode; +id value +1 1 +select * from gap4 where id in (1, 2, 3) lock in share mode; +id value +1 1 +2 2 +3 3 +select * from gap4 lock in share mode; +id value +2 2 +4 4 +1 1 +3 3 +5 5 +select * from gap4 where id between 3 and 7 lock in share mode; +id value +3 3 +4 4 +5 5 +set session autocommit=1; +select * from gap1 limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where value != 100 limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where id1=1 lock in share mode; +id1 id2 id3 c1 value +1 0 2 2 2 +1 0 3 3 3 +select * from gap1 where id1=1 and id2= 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 != 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 +between 1 and 3 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 asc +limit 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 desc +limit 1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 order by id1 asc limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 desc limit 1 lock in share mode; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 order by id1 desc, id2 desc, id3 desc +limit 1 lock in share mode; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 force index(i) where c1=1 lock in share mode; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap3 force index(ui) where value=1 lock in share mode; +id value +1 1 +select * from gap1 where id1=1 and id2=1 and id3=1 lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) lock in share mode; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 +order by c1 lock in share mode; +id1 id2 id3 c1 value +select * from gap3 where id=1 lock in share mode; +id value +1 1 +select * from gap4 where id=1 lock in share mode; +id value +1 1 +select * from gap4 where id in (1, 2, 3) lock in share mode; +id value +1 1 +2 2 +3 3 +select * from gap4 lock in share mode; +id value +2 2 +4 4 +1 1 +3 3 +5 5 +select * from gap4 where id between 3 and 7 lock in share mode; +id value +3 3 +4 4 +5 5 +set session autocommit=0; +select * from gap1 limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where value != 100 limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where id1=1 ; +id1 id2 id3 c1 value +1 0 2 2 2 +1 0 3 3 3 +select * from gap1 where id1=1 and id2= 1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 != 1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 +between 1 and 3 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 asc +limit 1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 desc +limit 1 ; +id1 id2 id3 c1 value +select * from gap1 order by id1 asc limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 desc limit 1 ; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 order by id1 desc, id2 desc, id3 desc +limit 1 ; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 force index(i) where c1=1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap3 force index(ui) where value=1 ; +id value +1 1 +select * from gap1 where id1=1 and id2=1 and id3=1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 +order by c1 ; +id1 id2 id3 c1 value +select * from gap3 where id=1 ; +id value +1 1 +select * from gap4 where id=1 ; +id value +1 1 +select * from gap4 where id in (1, 2, 3) ; +id value +1 1 +2 2 +3 3 +select * from gap4 ; +id value +2 2 +4 4 +1 1 +3 3 +5 5 +select * from gap4 where id between 3 and 7 ; +id value +3 3 +4 4 +5 5 +set session autocommit=1; +select * from gap1 limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where value != 100 limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 where id1=1 ; +id1 id2 id3 c1 value +1 0 2 2 2 +1 0 3 3 3 +select * from gap1 where id1=1 and id2= 1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 != 1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 and id3 +between 1 and 3 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 asc +limit 1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2= 1 order by id3 desc +limit 1 ; +id1 id2 id3 c1 value +select * from gap1 order by id1 asc limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 asc, id2 asc, id3 asc limit 1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap1 order by id1 desc limit 1 ; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 order by id1 desc, id2 desc, id3 desc +limit 1 ; +id1 id2 id3 c1 value +500 100 1000 1000 1000 +select * from gap1 force index(i) where c1=1 ; +id1 id2 id3 c1 value +0 0 1 1 1 +select * from gap3 force index(ui) where value=1 ; +id value +1 1 +select * from gap1 where id1=1 and id2=1 and id3=1 ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3 in (1, 2, 3) ; +id1 id2 id3 c1 value +select * from gap1 where id1=1 and id2=1 and id3=1 and value=1 +order by c1 ; +id1 id2 id3 c1 value +select * from gap3 where id=1 ; +id value +1 1 +select * from gap4 where id=1 ; +id value +1 1 +select * from gap4 where id in (1, 2, 3) ; +id value +1 1 +2 2 +3 3 +select * from gap4 ; +id value +2 2 +4 4 +1 1 +3 3 +5 5 +select * from gap4 where id between 3 and 7 ; +id value +3 3 +4 4 +5 5 +set session autocommit=0; +insert into gap1 (id1, id2, id3) values (-1,-1,-1); +insert into gap1 (id1, id2, id3) values (-1,-1,-1) +on duplicate key update value=100; +update gap1 set value=100 where id1=1; +update gap1 set value=100 where id1=1 and id2=1 and id3=1; +delete from gap1 where id1=2; +delete from gap1 where id1=-1 and id2=-1 and id3=-1; +commit; +set session autocommit=1; +insert into gap1 (id1, id2, id3) values (-1,-1,-1); +insert into gap1 (id1, id2, id3) values (-1,-1,-1) +on duplicate key update value=100; +update gap1 set value=100 where id1=1; +update gap1 set value=100 where id1=1 and id2=1 and id3=1; +delete from gap1 where id1=2; +delete from gap1 where id1=-1 and id2=-1 and id3=-1; +commit; +drop table gap1, gap2, gap3, gap4; diff --git a/storage/tokudb/mysql-test/tokudb/r/locks-select-update-3.result b/storage/tokudb/mysql-test/tokudb/r/locks-select-update-3.result index 8b31bf5a280..c1f05cdafd8 100644 --- a/storage/tokudb/mysql-test/tokudb/r/locks-select-update-3.result +++ b/storage/tokudb/mysql-test/tokudb/r/locks-select-update-3.result @@ -8,6 +8,7 @@ select * from t where a=1 for update; a b 1 0 update t set b=b+1 where a=1; +set session tokudb_lock_timeout= 60000; set session transaction isolation level read committed; begin; select * from t where a=1 for update; diff --git a/storage/tokudb/mysql-test/tokudb/r/percona_kill_idle_trx_tokudb.result b/storage/tokudb/mysql-test/tokudb/r/percona_kill_idle_trx_tokudb.result new file mode 100644 index 00000000000..089d1d2b136 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/percona_kill_idle_trx_tokudb.result @@ -0,0 +1,43 @@ +SET default_storage_engine=TokuDB; +# +# Test kill_idle_transaction_timeout feature with TokuDB +# +CREATE TABLE t1 (a INT); +SET GLOBAL kill_idle_transaction= 1; +BEGIN; +INSERT INTO t1 VALUES (1),(2); +COMMIT; +SELECT * FROM t1; +a +1 +2 +BEGIN; +INSERT INTO t1 VALUES (3); +# Current connection idle transaction killed, reconnecting +SELECT * FROM t1; +a +1 +2 +# +# Test that row locks are released on idle transaction kill +# +SET GLOBAL kill_idle_transaction= 2; +# Take row locks in connection conn1 +BEGIN; +SELECT * FROM t1 FOR UPDATE; +a +1 +2 +# Take row locks in connection default +UPDATE t1 SET a=4; +SELECT * FROM t1; +a +4 +4 +# Show that connection conn1 has been killed +SELECT * FROM t1; +ERROR HY000: MySQL server has gone away +# connection default +# Cleanup +DROP TABLE t1; +SET GLOBAL kill_idle_transaction= saved_kill_idle_transaction; diff --git a/storage/tokudb/mysql-test/tokudb/t/dir_per_db_rename_to_nonexisting_schema.test b/storage/tokudb/mysql-test/tokudb/t/dir_per_db_rename_to_nonexisting_schema.test new file mode 100644 index 00000000000..17fe0188a6e --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/dir_per_db_rename_to_nonexisting_schema.test @@ -0,0 +1,64 @@ +--source include/have_tokudb.inc + +SET GLOBAL tokudb_dir_per_db=true; +--let DATADIR=`SELECT @@datadir` + +--echo ###### +--echo # Tokudb and mysql data dirs are the same, rename to existent db +--echo ### +CREATE DATABASE new_db; +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +ALTER TABLE test.t1 RENAME new_db.t1; +--echo The content of "test" directory: +--source include/table_files_replace_pattern.inc +--sorted_result +--list_files $DATADIR/test +--echo The content of "new_db" directory: +--source include/table_files_replace_pattern.inc +--sorted_result +--list_files $DATADIR/new_db +DROP DATABASE new_db; + +--echo ###### +--echo # Tokudb and mysql data dirs are the same, rename to nonexistent db +--echo ### +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +CALL mtr.add_suppression("because destination db does not exist"); +--error ER_ERROR_ON_RENAME +ALTER TABLE test.t1 RENAME foo.t1; +DROP TABLE t1; + +--let $custom_tokudb_data_dir=$MYSQL_TMP_DIR/custom_tokudb_data_dir +--mkdir $custom_tokudb_data_dir +--replace_result $custom_tokudb_data_dir CUSTOM_TOKUDB_DATA_DIR +--source include/restart_mysqld.inc + +--replace_result $custom_tokudb_data_dir CUSTOM_TOKUDB_DATA_DIR +SELECT @@tokudb_data_dir; +SELECT @@tokudb_dir_per_db; + +--echo ###### +--echo # Tokudb and mysql data dirs are different, rename to existent db +--echo ### +CREATE DATABASE new_db; +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +ALTER TABLE test.t1 RENAME new_db.t1; +--echo The content of "test" direcotry: +--source include/table_files_replace_pattern.inc +--sorted_result +--echo The content of "new_db" directory: +--source include/table_files_replace_pattern.inc +--sorted_result +DROP DATABASE new_db; + +--echo ###### +--echo # Tokudb and mysql data dirs are different, rename to nonexistent db +--echo ### +CREATE TABLE t1 (id INT AUTO_INCREMENT PRIMARY KEY NOT NULL) ENGINE=tokudb; +CALL mtr.add_suppression("because destination db does not exist"); +--error ER_ERROR_ON_RENAME +ALTER TABLE test.t1 RENAME foo.t1; +DROP TABLE t1; + +SET GLOBAL tokudb_dir_per_db=default; + diff --git a/storage/tokudb/mysql-test/tokudb/t/gap_lock_error.test b/storage/tokudb/mysql-test/tokudb/t/gap_lock_error.test new file mode 100644 index 00000000000..8c52cef9e27 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/gap_lock_error.test @@ -0,0 +1,5 @@ +--source include/have_tokudb.inc + +let $engine=tokudb; +let $expect_gap_lock_errors=0; +--source include/gap_lock_error_all.inc diff --git a/storage/tokudb/mysql-test/tokudb/t/locks-select-update-3.test b/storage/tokudb/mysql-test/tokudb/t/locks-select-update-3.test index a563f061add..5b54fa7313e 100644 --- a/storage/tokudb/mysql-test/tokudb/t/locks-select-update-3.test +++ b/storage/tokudb/mysql-test/tokudb/t/locks-select-update-3.test @@ -15,6 +15,7 @@ select * from t where a=1 for update; # t2 update update t set b=b+1 where a=1; connect(conn1,localhost,root); +set session tokudb_lock_timeout= 60000; set session transaction isolation level read committed; begin; # t2 select for update, should hang until t1 commits diff --git a/storage/tokudb/mysql-test/tokudb/t/percona_kill_idle_trx_tokudb.test b/storage/tokudb/mysql-test/tokudb/t/percona_kill_idle_trx_tokudb.test new file mode 100644 index 00000000000..743fb9a55a7 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/percona_kill_idle_trx_tokudb.test @@ -0,0 +1,5 @@ +--source include/have_tokudb.inc +--skip MariaDB doesn't support kill_idle_trx variable for all SE + +SET default_storage_engine=TokuDB; +--source include/percona_kill_idle_trx.inc diff --git a/storage/tokudb/mysql-test/tokudb_backup/t/suite.opt b/storage/tokudb/mysql-test/tokudb_backup/t/suite.opt index 0d80cf85a91..5d4cb245e27 100644 --- a/storage/tokudb/mysql-test/tokudb_backup/t/suite.opt +++ b/storage/tokudb/mysql-test/tokudb_backup/t/suite.opt @@ -1 +1 @@ -$TOKUDB_OPT $TOKUDB_LOAD_ADD $TOKUDB_BACKUP_OPT $TOKUDB_BACKUP_LOAD_ADD --loose-tokudb-check-jemalloc=0 --loose-tokudb-cache-size=512M --loose-tokudb-block-size=1M +$TOKUDB_OPT $TOKUDB_LOAD_ADD_PATH $TOKUDB_BACKUP_OPT $TOKUDB_BACKUP_LOAD_ADD_PATH --loose-tokudb-check-jemalloc=0 --loose-tokudb-cache-size=512M --loose-tokudb-block-size=1M diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index ecea98fccfe..6bf8823124c 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -3400,7 +3400,7 @@ Removes a page from the level list of pages. /*************************************************************//** Removes a page from the level list of pages. */ -static MY_ATTRIBUTE((nonnull)) +static void btr_level_list_remove_func( /*=======================*/ @@ -3416,8 +3416,6 @@ btr_level_list_remove_func( ulint prev_page_no; ulint next_page_no; - ut_ad(page != NULL); - ut_ad(mtr != NULL); ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); ut_ad(space == page_get_space_id(page)); /* Get the previous and next page numbers of page */ diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index 214d050d562..dcb197a761f 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -1782,7 +1782,7 @@ btr_cur_pessimistic_insert( /*************************************************************//** For an update, checks the locks and does the undo logging. @return DB_SUCCESS, DB_WAIT_LOCK, or error number */ -UNIV_INLINE MY_ATTRIBUTE((warn_unused_result, nonnull(2,3,6,7))) +UNIV_INLINE MY_ATTRIBUTE((warn_unused_result)) dberr_t btr_cur_upd_lock_and_undo( /*======================*/ @@ -2014,7 +2014,6 @@ btr_cur_update_alloc_zip_func( const page_t* page = page_cur_get_page(cursor); ut_ad(page_zip == page_cur_get_page_zip(cursor)); - ut_ad(page_zip); ut_ad(!dict_index_is_ibuf(index)); ut_ad(rec_offs_validate(page_cur_get_rec(cursor), index, offsets)); @@ -3141,7 +3140,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && (thr != NULL)) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr)->id, index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -3855,19 +3854,42 @@ inexact: return(n_rows); } -/*******************************************************************//** -Estimates the number of rows in a given index range. -@return estimated number of rows */ -UNIV_INTERN +/** If the tree gets changed too much between the two dives for the left +and right boundary then btr_estimate_n_rows_in_range_low() will retry +that many times before giving up and returning the value stored in +rows_in_range_arbitrary_ret_val. */ +static const unsigned rows_in_range_max_retries = 4; + +/** We pretend that a range has that many records if the tree keeps changing +for rows_in_range_max_retries retries while we try to estimate the records +in a given range. */ +static const ib_int64_t rows_in_range_arbitrary_ret_val = 10; + +/** Estimates the number of rows in a given index range. +@param[in] index index +@param[in] tuple1 range start, may also be empty tuple +@param[in] mode1 search mode for range start +@param[in] tuple2 range end, may also be empty tuple +@param[in] mode2 search mode for range end +@param[in] trx trx +@param[in] nth_attempt if the tree gets modified too much while +we are trying to analyze it, then we will retry (this function will call +itself, incrementing this parameter) +@return estimated number of rows; if after rows_in_range_max_retries +retries the tree keeps changing, then we will just return +rows_in_range_arbitrary_ret_val as a result (if +nth_attempt >= rows_in_range_max_retries and the tree is modified between +the two dives). */ +static ib_int64_t -btr_estimate_n_rows_in_range( -/*=========================*/ - dict_index_t* index, /*!< in: index */ - const dtuple_t* tuple1, /*!< in: range start, may also be empty tuple */ - ulint mode1, /*!< in: search mode for range start */ - const dtuple_t* tuple2, /*!< in: range end, may also be empty tuple */ - ulint mode2, /*!< in: search mode for range end */ - trx_t* trx) /*!< in: trx */ +btr_estimate_n_rows_in_range_low( + dict_index_t* index, + const dtuple_t* tuple1, + ulint mode1, + const dtuple_t* tuple2, + ulint mode2, + trx_t* trx, + unsigned nth_attempt) { btr_path_t path1[BTR_PATH_ARRAY_N_SLOTS]; btr_path_t path2[BTR_PATH_ARRAY_N_SLOTS]; @@ -3905,6 +3927,12 @@ btr_estimate_n_rows_in_range( mtr_start_trx(&mtr, trx); +#ifdef UNIV_DEBUG + if (!strcmp(index->name, "iC")) { + DEBUG_SYNC_C("btr_estimate_n_rows_in_range_between_dives"); + } +#endif + cursor.path_arr = path2; if (dtuple_get_n_fields(tuple2) > 0) { @@ -3971,6 +3999,33 @@ btr_estimate_n_rows_in_range( if (!diverged && slot1->nth_rec != slot2->nth_rec) { + /* If both slots do not point to the same page or if + the paths have crossed and the same page on both + apparently contains a different number of records, + this means that the tree must have changed between + the dive for slot1 and the dive for slot2 at the + beginning of this function. */ + if (slot1->page_no != slot2->page_no + || slot1->page_level != slot2->page_level + || (slot1->nth_rec >= slot2->nth_rec + && slot1->n_recs != slot2->n_recs)) { + + /* If the tree keeps changing even after a + few attempts, then just return some arbitrary + number. */ + if (nth_attempt >= rows_in_range_max_retries) { + return(rows_in_range_arbitrary_ret_val); + } + + const ib_int64_t ret = + btr_estimate_n_rows_in_range_low( + index, tuple1, mode1, + tuple2, mode2, trx, + nth_attempt + 1); + + return(ret); + } + diverged = TRUE; if (slot1->nth_rec < slot2->nth_rec) { @@ -3989,7 +4044,7 @@ btr_estimate_n_rows_in_range( in this case slot1->nth_rec will point to the supr record and slot2->nth_rec will point to 6 */ - n_rows = 0; + return(0); } } else if (diverged && !diverged_lot) { @@ -4020,6 +4075,30 @@ btr_estimate_n_rows_in_range( } } +/** Estimates the number of rows in a given index range. +@param[in] index index +@param[in] tuple1 range start, may also be empty tuple +@param[in] mode1 search mode for range start +@param[in] tuple2 range end, may also be empty tuple +@param[in] mode2 search mode for range end +@param[in] trx trx +@return estimated number of rows */ +ib_int64_t +btr_estimate_n_rows_in_range( + dict_index_t* index, + const dtuple_t* tuple1, + ulint mode1, + const dtuple_t* tuple2, + ulint mode2, + trx_t* trx) +{ + const ib_int64_t ret = btr_estimate_n_rows_in_range_low( + index, tuple1, mode1, tuple2, mode2, trx, + 1 /* first attempt */); + + return(ret); +} + /*******************************************************************//** Record the number of non_null key values in a given index for each n-column prefix of the index where 1 <= n <= dict_index_get_n_unique(index). @@ -4482,7 +4561,6 @@ btr_cur_disown_inherited_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(mtr); for (i = 0; i < rec_offs_n_fields(offsets); i++) { if (rec_offs_nth_extern(offsets, i) @@ -4545,9 +4623,6 @@ btr_push_update_extern_fields( ulint n; const upd_field_t* uf; - ut_ad(tuple); - ut_ad(update); - uf = update->fields; n = upd_get_n_fields(update); @@ -4731,7 +4806,6 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(btr_mtr); ut_ad(mtr_memo_contains(btr_mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains(btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX)); diff --git a/storage/xtradb/buf/buf0buddy.cc b/storage/xtradb/buf/buf0buddy.cc index 8cb880c1169..2ee39c6c992 100644 --- a/storage/xtradb/buf/buf0buddy.cc +++ b/storage/xtradb/buf/buf0buddy.cc @@ -485,7 +485,6 @@ buf_buddy_alloc_low( { buf_block_t* block; - ut_ad(lru); ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); ut_ad(i >= buf_buddy_get_slot(UNIV_ZIP_SIZE_MIN)); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 21b10196a25..07b84c60c76 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -3691,15 +3691,6 @@ buf_page_init( /* Set the state of the block */ buf_block_set_file_page(block, space, offset); -#ifdef UNIV_DEBUG_VALGRIND - if (!space) { - /* Silence valid Valgrind warnings about uninitialized - data being written to data files. There are some unused - bytes on some pages that InnoDB does not initialize. */ - UNIV_MEM_VALID(block->frame, UNIV_PAGE_SIZE); - } -#endif /* UNIV_DEBUG_VALGRIND */ - buf_block_init_low(block); block->lock_hash_val = lock_rec_hash(space, offset); @@ -4703,7 +4694,8 @@ buf_all_freed_instance( block->page.space, block->page.offset); ib_logf(IB_LOG_LEVEL_ERROR, - "Page oldest_modification %lu fix_count %d io_fix %d.", + "Page oldest_modification " LSN_PF + " fix_count %d io_fix %d.", block->page.oldest_modification, block->page.buf_fix_count, buf_page_get_io_fix(&block->page)); @@ -5601,23 +5593,22 @@ buf_print_io_instance( pool_info->pages_written_rate); if (pool_info->n_page_get_delta) { - double hit_rate = ((1000 * pool_info->page_read_delta) - / pool_info->n_page_get_delta); + double hit_rate = double(pool_info->page_read_delta) + / pool_info->n_page_get_delta; - if (hit_rate > 1000) { - hit_rate = 1000; + if (hit_rate > 1) { + hit_rate = 1; } - hit_rate = 1000 - hit_rate; - fprintf(file, - "Buffer pool hit rate %lu / 1000," - " young-making rate %lu / 1000 not %lu / 1000\n", - (ulint) hit_rate, - (ulint) (1000 * pool_info->young_making_delta - / pool_info->n_page_get_delta), - (ulint) (1000 * pool_info->not_young_making_delta - / pool_info->n_page_get_delta)); + "Buffer pool hit rate " ULINTPF " / 1000," + " young-making rate " ULINTPF " / 1000 not " + ULINTPF " / 1000\n", + ulint(1000 * (1 - hit_rate)), + ulint(1000 * double(pool_info->young_making_delta) + / pool_info->n_page_get_delta), + ulint(1000 * double(pool_info->not_young_making_delta) + / pool_info->n_page_get_delta)); } else { fputs("No buffer pool page gets since the last printout\n", file); diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc index cae6099f03e..df62c945288 100644 --- a/storage/xtradb/buf/buf0dump.cc +++ b/storage/xtradb/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -52,8 +53,8 @@ enum status_severity { /* Flags that tell the buffer pool dump/load thread which action should it take after being waked up. */ -static ibool buf_dump_should_start = FALSE; -static ibool buf_load_should_start = FALSE; +static volatile bool buf_dump_should_start; +static volatile bool buf_load_should_start; static ibool buf_load_abort_flag = FALSE; @@ -78,7 +79,7 @@ void buf_dump_start() /*============*/ { - buf_dump_should_start = TRUE; + buf_dump_should_start = true; os_event_set(srv_buf_dump_event); } @@ -92,7 +93,7 @@ void buf_load_start() /*============*/ { - buf_load_should_start = TRUE; + buf_load_should_start = true; os_event_set(srv_buf_dump_event); } @@ -695,15 +696,18 @@ DECLARE_THREAD(buf_dump_thread)( os_event_wait(srv_buf_dump_event); if (buf_dump_should_start) { - buf_dump_should_start = FALSE; + buf_dump_should_start = false; buf_dump(TRUE /* quit on shutdown */); } if (buf_load_should_start) { - buf_load_should_start = FALSE; + buf_load_should_start = false; buf_load(); } + if (buf_dump_should_start || buf_load_should_start) { + continue; + } os_event_reset(srv_buf_dump_event); } diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 601e79d8923..018b29b7d95 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -720,9 +721,6 @@ buf_flush_write_complete( buf_pool->n_flush[flush_type]--; - /* fprintf(stderr, "n pending flush %lu\n", - buf_pool->n_flush[flush_type]); */ - if (buf_pool->n_flush[flush_type] == 0 && buf_pool->init_flush[flush_type] == FALSE) { diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc index c599653706d..d31773c96cc 100644 --- a/storage/xtradb/buf/buf0lru.cc +++ b/storage/xtradb/buf/buf0lru.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1301,6 +1302,71 @@ buf_LRU_check_size_of_non_data_objects( } } +/** Diagnose failure to get a free page and request InnoDB monitor output in +the error log if more than two seconds have been spent already. +@param[in] n_iterations how many buf_LRU_get_free_page iterations + already completed +@param[in] started_ms timestamp in ms of when the attempt to get the + free page started +@param[in] flush_failures how many times single-page flush, if allowed, + has failed +@param[out] mon_value_was previous srv_print_innodb_monitor value +@param[out] started_monitor whether InnoDB monitor print has been requested +*/ +static +void +buf_LRU_handle_lack_of_free_blocks(ulint n_iterations, ulint started_ms, + ulint flush_failures, + ibool *mon_value_was, + ibool *started_monitor) +{ + static ulint last_printout_ms = 0; + + /* Legacy algorithm started warning after at least 2 seconds, we + emulate this. */ + const ulint current_ms = ut_time_ms(); + + if ((current_ms > started_ms + 2000) + && (current_ms > last_printout_ms + 2000)) { + + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Warning: difficult to find free blocks in\n" + "InnoDB: the buffer pool (%lu search iterations)!\n" + "InnoDB: %lu failed attempts to flush a page!" + " Consider\n" + "InnoDB: increasing the buffer pool size.\n" + "InnoDB: It is also possible that" + " in your Unix version\n" + "InnoDB: fsync is very slow, or" + " completely frozen inside\n" + "InnoDB: the OS kernel. Then upgrading to" + " a newer version\n" + "InnoDB: of your operating system may help." + " Look at the\n" + "InnoDB: number of fsyncs in diagnostic info below.\n" + "InnoDB: Pending flushes (fsync) log: %lu;" + " buffer pool: %lu\n" + "InnoDB: %lu OS file reads, %lu OS file writes," + " %lu OS fsyncs\n" + "InnoDB: Starting InnoDB Monitor to print further\n" + "InnoDB: diagnostics to the standard output.\n", + (ulong) n_iterations, + (ulong) flush_failures, + (ulong) fil_n_pending_log_flushes, + (ulong) fil_n_pending_tablespace_flushes, + (ulong) os_n_file_reads, (ulong) os_n_file_writes, + (ulong) os_n_fsyncs); + + last_printout_ms = current_ms; + *mon_value_was = srv_print_innodb_monitor; + *started_monitor = TRUE; + srv_print_innodb_monitor = TRUE; + os_event_set(lock_sys->timeout_event); + } + +} + /** The maximum allowed backoff sleep time duration, microseconds */ #define MAX_FREE_LIST_BACKOFF_SLEEP 10000 @@ -1348,6 +1414,7 @@ buf_LRU_get_free_block( ulint flush_failures = 0; ibool mon_value_was = FALSE; ibool started_monitor = FALSE; + ulint started_ms = 0; ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); @@ -1356,7 +1423,24 @@ loop: buf_LRU_check_size_of_non_data_objects(buf_pool); /* If there is a block in the free list, take it */ - block = buf_LRU_get_free_only(buf_pool); + if (DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { + + block = NULL; + + if (srv_debug_monitor_printed) + DBUG_SET("-d,simulate_lack_of_pages"); + + } else if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", + recv_recovery_on, false)) { + + block = NULL; + + if (srv_debug_monitor_printed) + DBUG_SUICIDE(); + } else { + + block = buf_LRU_get_free_only(buf_pool); + } if (block) { @@ -1371,6 +1455,9 @@ loop: return(block); } + if (!started_ms) + started_ms = ut_time_ms(); + if (srv_empty_free_list_algorithm == SRV_EMPTY_FREE_LIST_BACKOFF && buf_lru_manager_is_active && (srv_shutdown_state == SRV_SHUTDOWN_NONE @@ -1408,11 +1495,17 @@ loop: : FREE_LIST_BACKOFF_LOW_PRIO_DIVIDER)); } - /* In case of backoff, do not ever attempt single page flushes - and wait for the cleaner to free some pages instead. */ + buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, + flush_failures, + &mon_value_was, + &started_monitor); n_iterations++; + srv_stats.buf_pool_wait_free.inc(); + + /* In case of backoff, do not ever attempt single page flushes + and wait for the cleaner to free some pages instead. */ goto loop; } else { @@ -1444,6 +1537,12 @@ loop: mutex_exit(&buf_pool->flush_state_mutex); + if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", true, false) + || DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { + + buf_pool->try_LRU_scan = false; + } + freed = FALSE; if (buf_pool->try_LRU_scan || n_iterations > 0) { @@ -1469,41 +1568,9 @@ loop: } - if (n_iterations > 20) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: difficult to find free blocks in\n" - "InnoDB: the buffer pool (%lu search iterations)!\n" - "InnoDB: %lu failed attempts to flush a page!" - " Consider\n" - "InnoDB: increasing the buffer pool size.\n" - "InnoDB: It is also possible that" - " in your Unix version\n" - "InnoDB: fsync is very slow, or" - " completely frozen inside\n" - "InnoDB: the OS kernel. Then upgrading to" - " a newer version\n" - "InnoDB: of your operating system may help." - " Look at the\n" - "InnoDB: number of fsyncs in diagnostic info below.\n" - "InnoDB: Pending flushes (fsync) log: %lu;" - " buffer pool: %lu\n" - "InnoDB: %lu OS file reads, %lu OS file writes," - " %lu OS fsyncs\n" - "InnoDB: Starting InnoDB Monitor to print further\n" - "InnoDB: diagnostics to the standard output.\n", - (ulong) n_iterations, - (ulong) flush_failures, - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - mon_value_was = srv_print_innodb_monitor; - started_monitor = TRUE; - srv_print_innodb_monitor = TRUE; - os_event_set(srv_monitor_event); - } + buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, + flush_failures, &mon_value_was, + &started_monitor); /* If we have scanned the whole LRU and still are unable to find a free block then we should sleep here to let the @@ -1529,7 +1596,7 @@ loop: ++flush_failures; } - srv_stats.buf_pool_wait_free.add(n_iterations, 1); + srv_stats.buf_pool_wait_free.inc(); n_iterations++; diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index c28df72df92..9053f38924e 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -986,15 +986,11 @@ not_to_recover: count++; if (count > 1000) { - fprintf(stderr, - "InnoDB: Error: InnoDB has waited for" - " 10 seconds for pending\n" - "InnoDB: reads to the buffer pool to" - " be finished.\n" - "InnoDB: Number of pending reads %lu," - " pending pread calls %lu\n", - (ulong) buf_pool->n_pend_reads, - (ulong) os_file_n_pending_preads); + ib_logf(IB_LOG_LEVEL_ERROR, + "waited for 10 seconds for " ULINTPF + " pending reads to the buffer pool to" + " be finished", + buf_pool->n_pend_reads); os_aio_print_debug = TRUE; } diff --git a/storage/xtradb/dict/dict0boot.cc b/storage/xtradb/dict/dict0boot.cc index c0bb0298bea..20ab5d75205 100644 --- a/storage/xtradb/dict/dict0boot.cc +++ b/storage/xtradb/dict/dict0boot.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -272,10 +273,6 @@ dict_boot(void) ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_FOR_NAME == 2); ut_ad(DICT_NUM_COLS__SYS_FOREIGN_COLS == 4); ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_COLS == 6); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT == 5); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT_COLS == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS == 5); mtr_start(&mtr); diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index db0ca638de4..9a4040421d7 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1932,135 +1933,6 @@ dict_create_or_check_sys_tablespace(void) return(err); } -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void) -{ - trx_t* trx; - my_bool srv_file_per_table_backup; - dberr_t err; - dberr_t sys_zip_dict_err; - dberr_t sys_zip_dict_cols_err; - - ut_a(srv_get_active_thread_type() == SRV_NONE); - - /* Note: The master thread has not been started at this point. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, - 1); - - if (sys_zip_dict_err == DB_SUCCESS && - sys_zip_dict_cols_err == DB_SUCCESS) - return (DB_SUCCESS); - - trx = trx_allocate_for_mysql(); - - trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); - - trx->op_info = "creating zip_dict and zip_dict_cols sys tables"; - - row_mysql_lock_data_dictionary(trx); - - /* Check which incomplete table definition to drop. */ - - if (sys_zip_dict_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT table."); - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - } - if (sys_zip_dict_cols_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT_COLS table."); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - } - - ib_logf(IB_LOG_LEVEL_INFO, - "Creating zip_dict and zip_dict_cols system tables."); - - /* We always want SYSTEM tables to be created inside the system - tablespace. */ - srv_file_per_table_backup = srv_file_per_table; - srv_file_per_table = 0; - - err = que_eval_sql( - NULL, - "PROCEDURE CREATE_SYS_ZIP_DICT_PROC () IS\n" - "BEGIN\n" - "CREATE TABLE SYS_ZIP_DICT(\n" - " ID INT UNSIGNED NOT NULL,\n" - " NAME CHAR(" - STRINGIFY_ARG(ZIP_DICT_MAX_NAME_LENGTH) - ") NOT NULL,\n" - " DATA BLOB NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_ID" - " ON SYS_ZIP_DICT (ID);\n" - "CREATE UNIQUE INDEX SYS_ZIP_DICT_NAME" - " ON SYS_ZIP_DICT (NAME);\n" - "CREATE TABLE SYS_ZIP_DICT_COLS(\n" - " TABLE_ID INT UNSIGNED NOT NULL,\n" - " COLUMN_POS INT UNSIGNED NOT NULL,\n" - " DICT_ID INT UNSIGNED NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_COLS_COMPOSITE" - " ON SYS_ZIP_DICT_COLS (TABLE_ID, COLUMN_POS);\n" - "END;\n", - FALSE, trx); - - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Creation of SYS_ZIP_DICT and SYS_ZIP_DICT_COLS" - "has failed with error %lu. Tablespace is full. " - "Dropping incompletely created tables.", - (ulong) err); - - ut_a(err == DB_OUT_OF_FILE_SPACE - || err == DB_TOO_MANY_CONCURRENT_TRXS); - - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - - if (err == DB_OUT_OF_FILE_SPACE) { - err = DB_MUST_GET_MORE_FILE_SPACE; - } - } - - trx_commit_for_mysql(trx); - - row_mysql_unlock_data_dictionary(trx); - - trx_free_for_mysql(trx); - - srv_file_per_table = srv_file_per_table_backup; - - if (err == DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_INFO, - "zip_dict and zip_dict_cols system tables created."); - } - - /* Note: The master thread has not been started at this point. */ - /* Confirm and move to the non-LRU part of the table LRU list. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - ut_a(sys_zip_dict_err == DB_SUCCESS); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, 1); - ut_a(sys_zip_dict_cols_err == DB_SUCCESS); - - return(err); -} - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -2114,456 +1986,3 @@ dict_create_add_tablespace_to_dictionary( return(error); } - -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_add_literal(info, "data", data, data_len, - DATA_BLOB, DATA_BINARY_TYPE | DATA_NOT_NULL); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - " max_id INT;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " ORDER BY ID DESC;\n" - "BEGIN\n" - " max_id := 0;\n" - " OPEN cur;\n" - " FETCH cur INTO max_id;\n" - " IF (cur % NOTFOUND) THEN\n" - " max_id := 0;\n" - " END IF;\n" - " CLOSE cur;\n" - " INSERT INTO SYS_ZIP_DICT VALUES" - " (max_id + 1, :name, :data);\n" - "END;\n", - FALSE, trx); - - return error; -} - -/** Fetch callback, just stores extracted zip_dict id in the external -variable. -@return TRUE if all OK */ -static -ibool -dict_create_extract_int_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: int32 id */ -{ - sel_node_t* node = static_cast<sel_node_t*>(row); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - ut_a(len == sizeof(ib_uint32_t)); - - memcpy(user_arg, dfield_get_data(dfield), sizeof(ib_uint32_t)); - - return(TRUE); -} - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_add_int4_literal(info, "dict_id", dict_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " INSERT INTO SYS_ZIP_DICT_COLS VALUES" - " (:table_id, :column_pos, :dict_id);\n" - "END;\n", - FALSE, trx); - return error; -} - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast<byte*>(&dict_id_buf ), - ULINT32_UNDEFINED); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT DICT_ID FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id AND\n" - " COLUMN_POS = :column_pos;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast<const byte*>(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_name); - ut_ad(dict_name_len); - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "dict_name", dict_name, dict_name_len, - DATA_VARCHAR, DATA_ENGLISH); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast<byte*>(&dict_id_buf), - ULINT32_UNDEFINED); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :dict_name;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast<const byte*>(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Auxiliary enum used to indicate zip dict data extraction result code */ -enum zip_dict_info_aux_code { - zip_dict_info_success, /*!< success */ - zip_dict_info_not_found, /*!< zip dict record not found */ - zip_dict_info_oom, /*!< out of memory */ - zip_dict_info_corrupted_name, /*!< corrupted zip dict name */ - zip_dict_info_corrupted_data /*!< corrupted zip dict data */ -}; - -/** Auxiliary struct used to return zip dict info aling with result code */ -struct zip_dict_info_aux { - LEX_STRING name; /*!< zip dict name */ - LEX_STRING data; /*!< zip dict data */ - int code; /*!< result code (0 - success) */ -}; - -/** Fetch callback, just stores extracted zip_dict data in the external -variable. -@return always returns TRUE */ -static -ibool -dict_create_get_zip_dict_info_by_id_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to zip_dict_info_aux* */ -{ - sel_node_t* node = static_cast<sel_node_t*>(row); - zip_dict_info_aux* result = - static_cast<zip_dict_info_aux*>(user_arg); - - result->code = zip_dict_info_success; - result->name.str = 0; - result->name.length = 0; - result->data.str = 0; - result->data.length = 0; - - /* NAME field */ - que_node_t* exp = node->select_list; - ut_a(exp != 0); - - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_name; - } - else { - result->name.str = - static_cast<char*>(my_malloc(len + 1, MYF(0))); - if (result->name.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->name.str, data, len); - result->name.str[len] = '\0'; - result->name.length = len; - } - } - - /* DATA field */ - exp = que_node_get_next(exp); - ut_a(exp != 0); - - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_BLOB); - - len = dfield_get_len(dfield); - data = dfield_get_data(dfield); - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_data; - } - else { - result->data.str = - static_cast<char*>(my_malloc( - len == 0 ? 1 : len, MYF(0))); - if (result->data.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->data.str, data, len); - result->data.length = len; - } - } - - ut_ad(que_node_get_next(exp) == 0); - - if (result->code != zip_dict_info_success) { - if (result->name.str == 0) { - mem_free(result->name.str); - result->name.str = 0; - result->name.length = 0; - } - if (result->data.str == 0) { - mem_free(result->data.str); - result->data.str = 0; - result->data.length = 0; - } - } - - return TRUE; -} - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length*/ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length*/ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - zip_dict_info_aux rec; - rec.code = zip_dict_info_not_found; - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "id", dict_id); - pars_info_bind_function( - info, "my_func", dict_create_get_zip_dict_info_by_id_aux, - &rec); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT NAME, DATA FROM SYS_ZIP_DICT\n" - " WHERE ID = :id;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - switch (rec.code) { - case zip_dict_info_success: - *name = rec.name.str; - *name_len = rec.name.length; - *data = rec.data.str; - *data_len = rec.data.length; - break; - case zip_dict_info_not_found: - error = DB_RECORD_NOT_FOUND; - break; - case zip_dict_info_oom: - error = DB_OUT_OF_MEMORY; - break; - case zip_dict_info_corrupted_name: - case zip_dict_info_corrupted_data: - error = DB_INVALID_NULL; - break; - default: - ut_error; - } - } - return error; -} - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast<byte*>(&dict_id_buf), - ULINT32_UNDEFINED); - ib_uint32_t counter_buf; - mach_write_to_4(reinterpret_cast<byte*>(&counter_buf), - ULINT32_UNDEFINED); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_bind_int4_literal(info, "dict_id", &dict_id_buf); - pars_info_bind_function(info, "find_dict_func", - dict_create_extract_int_aux, &dict_id_buf); - pars_info_bind_function(info, "count_func", - dict_create_extract_int_aux, &counter_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION find_dict_func;\n" - "DECLARE FUNCTION count_func;\n" - "DECLARE CURSOR dict_cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :name\n" - " FOR UPDATE;\n" - "DECLARE CURSOR ref_cur IS\n" - " SELECT 1 FROM SYS_ZIP_DICT_COLS\n" - " WHERE DICT_ID = :dict_id;\n" - "BEGIN\n" - " OPEN dict_cur;\n" - " FETCH dict_cur INTO find_dict_func();\n" - " IF NOT (SQL % NOTFOUND) THEN\n" - " OPEN ref_cur;\n" - " FETCH ref_cur INTO count_func();\n" - " IF SQL % NOTFOUND THEN\n" - " DELETE FROM SYS_ZIP_DICT WHERE CURRENT OF dict_cur;\n" - " END IF;\n" - " CLOSE ref_cur;\n" - " END IF;\n" - " CLOSE dict_cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast<const byte*>(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) { - error = DB_RECORD_NOT_FOUND; - } - else { - ib_uint32_t local_counter = mach_read_from_4( - reinterpret_cast<const byte*>(&counter_buf)); - if (local_counter != ULINT32_UNDEFINED) - error = DB_ROW_IS_REFERENCED; - } - } - return error; -} - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " DELETE FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id;\n" - "END;\n", - FALSE, trx); - return error; -} diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 3845138c8bf..01eaecb860d 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -6156,7 +6156,6 @@ dict_set_corrupted( row_mysql_lock_data_dictionary(trx); } - ut_ad(index); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); @@ -7288,161 +7287,3 @@ dict_tf_to_row_format_string( return(0); } #endif /* !UNIV_HOTBACKUP */ - -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len) /*!< in: zip_dict data length */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - ut_ad(data); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "insert zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_add_zip_dict(name, name_len, data, data_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id) /*!< out: zip_dict id */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict id by composite key"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_id_by_reference(table_id, column_pos, - dict_id, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len) /*!< out: dictionary data length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict name and data by id"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_info_by_id(dict_id, name, name_len, - data, data_len, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len) /*!< in: zip_dict name length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "delete zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_remove_zip_dict(name, name_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 9ea4f4d873a..6cbd0a3d488 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,9 +57,7 @@ static const char* SYSTEM_TABLE_NAME[] = { "SYS_FOREIGN", "SYS_FOREIGN_COLS", "SYS_TABLESPACES", - "SYS_DATAFILES", - "SYS_ZIP_DICT", - "SYS_ZIP_DICT_COLS" + "SYS_DATAFILES" }; /* If this flag is TRUE, then we will load the cluster index's (and tables') @@ -730,161 +729,6 @@ err_len: return(NULL); } -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len) /*!< out: dict data length */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *id = ULINT_UNDEFINED; - *name = NULL; - *data = NULL; - *data_len = 0; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT"); - } - - if (UNIV_UNLIKELY( - rec_get_n_fields_old(rec)!= DICT_NUM_FIELDS__SYS_ZIP_DICT)) { - return("wrong number of columns in SYS_ZIP_DICT record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *id = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__NAME, &len); - if (UNIV_UNLIKELY(len == 0 || len == UNIV_SQL_NULL)) { - goto err_len; - } - *name = mem_heap_strdupl(heap, (char*) field, len); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__DATA, &len); - if (UNIV_UNLIKELY(len == UNIV_SQL_NULL)) { - goto err_len; - } - - if (rec_get_1byte_offs_flag(rec) == 0 && - rec_2_is_field_extern(rec, DICT_FLD__SYS_ZIP_DICT__DATA)) { - ut_a(len >= BTR_EXTERN_FIELD_REF_SIZE); - - if (UNIV_UNLIKELY - (!memcmp(field + len - BTR_EXTERN_FIELD_REF_SIZE, - field_ref_zero, - BTR_EXTERN_FIELD_REF_SIZE))) { - goto err_len; - } - *data = reinterpret_cast<char*>( - btr_copy_externally_stored_field(data_len, field, - zip_size, len, heap, 0)); - } - else { - *data_len = len; - *data = static_cast<char*>(mem_heap_dup(heap, field, len)); - } - - return(NULL); - -err_len: - return("incorrect column length in SYS_ZIP_DICT"); -} - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id) /*!< out: dict id */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *table_id = ULINT_UNDEFINED; - *column_pos = ULINT_UNDEFINED; - *dict_id = ULINT_UNDEFINED; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT_COLS"); - } - - if (UNIV_UNLIKELY(rec_get_n_fields_old(rec) != - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS)) { - return("wrong number of columns in SYS_ZIP_DICT_COLS" - " record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { -err_len: - return("incorrect column length in SYS_ZIP_DICT_COLS"); - } - *table_id = mach_read_from_4(field); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *column_pos = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *dict_id = mach_read_from_4(field); - - return(NULL); -} /********************************************************************//** Determine the flags of a table as stored in SYS_TABLES.TYPE and N_COLS. @return ULINT_UNDEFINED if error, else a valid dict_table_t::flags. */ diff --git a/storage/xtradb/dict/dict0stats.cc b/storage/xtradb/dict/dict0stats.cc index a4aa43651f8..c0a83c951a5 100644 --- a/storage/xtradb/dict/dict0stats.cc +++ b/storage/xtradb/dict/dict0stats.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2009, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1109,7 +1109,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1295,8 +1296,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1572,6 +1577,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/xtradb/dict/dict0stats_bg.cc b/storage/xtradb/dict/dict0stats_bg.cc index 6f01c379776..b3c8ef018f7 100644 --- a/storage/xtradb/dict/dict0stats_bg.cc +++ b/storage/xtradb/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -39,8 +40,9 @@ Created Apr 25, 2012 Vasil Dimov #define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) -/** Event to wake up the stats thread */ -UNIV_INTERN os_event_t dict_stats_event = NULL; +/** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() +or shutdown. Not protected by any mutex. */ +UNIV_INTERN os_event_t dict_stats_event; /** This mutex protects the "recalc_pool" variable. */ static ib_mutex_t recalc_pool_mutex; diff --git a/storage/xtradb/dyn/dyn0dyn.cc b/storage/xtradb/dyn/dyn0dyn.cc index 3ef5297a7c9..dd1f6863c14 100644 --- a/storage/xtradb/dyn/dyn0dyn.cc +++ b/storage/xtradb/dyn/dyn0dyn.cc @@ -40,7 +40,6 @@ dyn_array_add_block( mem_heap_t* heap; dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index a7b0377d2a4..1a866a693ca 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -4993,15 +4994,12 @@ fil_extend_space_to_desired_size( byte* buf; ulint buf_size; ulint start_page_no; - ulint file_start_page_no; ulint page_size; - ulint pages_added; ibool success; ut_ad(!srv_read_only_mode); retry: - pages_added = 0; success = TRUE; fil_mutex_enter_and_prepare_for_io(space_id); @@ -5055,27 +5053,33 @@ retry: mutex_exit(&fil_system->mutex); start_page_no = space->size; - file_start_page_no = space->size - node->size; - + const ulint file_start_page_no = space->size - node->size; #ifdef HAVE_POSIX_FALLOCATE if (srv_use_posix_fallocate) { - os_offset_t start_offset = start_page_no * page_size; - os_offset_t n_pages = (size_after_extend - start_page_no); - os_offset_t len = n_pages * page_size; - - if (posix_fallocate(node->handle, start_offset, len) == -1) { - ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " - "space for file \'%s\' failed. Current size " - INT64PF ", desired size " INT64PF, - node->name, start_offset, len+start_offset); - os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE); - success = FALSE; - } else { - success = TRUE; + const os_offset_t start_offset + = os_offset_t(start_page_no - file_start_page_no) + * page_size; + const ulint n_pages + = size_after_extend - start_page_no; + const os_offset_t len = os_offset_t(n_pages) * page_size; + + int err; + do { + err = posix_fallocate(node->handle, start_offset, len); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); + + success = !err; + if (!success) { + ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" + " from " INT64PF " to " INT64PF " bytes" + " failed with error %d", + node->name, start_offset, len + start_offset, + err); } DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28;os_has_said_disk_full = TRUE;); + success = FALSE; os_has_said_disk_full = TRUE;); mutex_enter(&fil_system->mutex); @@ -5095,14 +5099,24 @@ retry: } #endif +#ifdef _WIN32 + /* Write 1 page of zeroes at the desired end. */ + start_page_no = size_after_extend - 1; + buf_size = page_size; +#else /* Extend at most 64 pages at a time */ buf_size = ut_min(64, size_after_extend - start_page_no) * page_size; - buf2 = static_cast<byte*>(mem_alloc(buf_size + page_size)); +#endif + buf2 = static_cast<byte*>(calloc(1, buf_size + page_size)); + if (!buf2) { + ib_logf(IB_LOG_LEVEL_ERROR, "Cannot allocate " ULINTPF + " bytes to extend file", + buf_size + page_size); + success = FALSE; + } buf = static_cast<byte*>(ut_align(buf2, page_size)); - memset(buf, 0, buf_size); - - while (start_page_no < size_after_extend) { + while (success && start_page_no < size_after_extend) { ulint n_pages = ut_min(buf_size / page_size, size_after_extend - start_page_no); @@ -5111,55 +5125,47 @@ retry: = ((os_offset_t) (start_page_no - file_start_page_no)) * page_size; - const char* name = node->name == NULL ? space->name : node->name; - #ifdef UNIV_HOTBACKUP - success = os_file_write(name, node->handle, buf, + success = os_file_write(node->name, node->handle, buf, offset, page_size * n_pages); #else success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, - name, node->handle, buf, + node->name, node->handle, buf, offset, page_size * n_pages, NULL, NULL, space_id, NULL); #endif /* UNIV_HOTBACKUP */ DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28; os_has_said_disk_full = TRUE;); - - if (success) { - os_has_said_disk_full = FALSE; - } else { - /* Let us measure the size of the file to determine - how much we were able to extend it */ - os_offset_t size; - - size = os_file_get_size(node->handle); - ut_a(size != (os_offset_t) -1); + success = FALSE; os_has_said_disk_full = TRUE;); - n_pages = ((ulint) (size / page_size)) - - node->size - pages_added; + /* Let us measure the size of the file to determine + how much we were able to extend it */ + os_offset_t size = os_file_get_size(node->handle); + ut_a(size != (os_offset_t) -1); - pages_added += n_pages; - break; - } - - start_page_no += n_pages; - pages_added += n_pages; + start_page_no = (ulint) (size / page_size) + + file_start_page_no; } - mem_free(buf2); + free(buf2); mutex_enter(&fil_system->mutex); ut_a(node->being_extended); + ut_a(start_page_no - file_start_page_no >= node->size); - space->size += pages_added; - node->size += pages_added; + if (buf) { + ulint file_size = start_page_no - file_start_page_no; + space->size += file_size - node->size; + node->size = file_size; + } fil_node_complete_io(node, fil_system, OS_FILE_WRITE); /* At this point file has been extended */ +#ifdef HAVE_POSIX_FALLOCATE file_extended: +#endif /* HAVE_POSIX_FALLOCATE */ node->being_extended = FALSE; *actual_size = space->size; diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 0bdb2ea0d4c..9292f84ab02 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -132,7 +132,7 @@ fsp_fill_free_list( ulint space, /*!< in: space */ fsp_header_t* header, /*!< in/out: space header */ mtr_t* mtr) /*!< in/out: mini-transaction */ - UNIV_COLD MY_ATTRIBUTE((nonnull)); + UNIV_COLD; /**********************************************************************//** Allocates a single free page from a segment. This function implements the intelligent allocation strategy which tries to minimize file space @@ -161,7 +161,7 @@ fseg_alloc_free_page_low( in which the page should be initialized. If init_mtr!=mtr, but the page is already latched in mtr, do not initialize the page. */ - MY_ATTRIBUTE((warn_unused_result, nonnull)); + MY_ATTRIBUTE((warn_unused_result)); #endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** @@ -1067,8 +1067,6 @@ fsp_fill_free_list( ulint i; mtr_t ibuf_mtr; - ut_ad(header != NULL); - ut_ad(mtr != NULL); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); /* Check if we can fill free list from above the free list limit */ @@ -1331,7 +1329,7 @@ Allocates a single free page from a space. The page is marked as used. @retval block, rw_lock_x_lock_count(&block->lock) == 1 if allocation succeeded (init_mtr == mtr, or the page was not previously freed in mtr) @retval block (not allocated or initialized) otherwise */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +static MY_ATTRIBUTE((warn_unused_result)) buf_block_t* fsp_alloc_free_page( /*================*/ @@ -1351,9 +1349,6 @@ fsp_alloc_free_page( ulint page_no; ulint space_size; - ut_ad(mtr); - ut_ad(init_mtr); - header = fsp_get_space_header(space, zip_size, mtr); /* Get the hinted descriptor */ @@ -2372,7 +2367,6 @@ fseg_alloc_free_page_low( ibool success; ulint n; - ut_ad(mtr); ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); @@ -2810,6 +2804,7 @@ try_again: } } else { ut_a(alloc_type == FSP_CLEANING); + reserve = 0; } success = fil_space_reserve_free_extents(space, n_free, n_ext); diff --git a/storage/xtradb/fts/fts0fts.cc b/storage/xtradb/fts/fts0fts.cc index 4c54afae8cd..c1f0b0bd5fe 100644 --- a/storage/xtradb/fts/fts0fts.cc +++ b/storage/xtradb/fts/fts0fts.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1934,6 +1935,8 @@ func_exit: /*************************************************************//** Wrapper function of fts_create_index_tables_low(), create auxiliary tables for an FTS index + +@see row_merge_create_fts_sort_index() @return: DB_SUCCESS or error code */ static dict_table_t* @@ -1965,13 +1968,12 @@ fts_create_one_index_table( (int)(field->col->prtype & DATA_MYSQL_TYPE_MASK), (uint) dtype_get_charset_coll(field->col->prtype)); - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARCHAR, - field->col->prtype, FTS_MAX_WORD_LEN); - } else { - dict_mem_table_add_col(new_table, heap, "word", DATA_VARMYSQL, - field->col->prtype, FTS_MAX_WORD_LEN); - } + dict_mem_table_add_col(new_table, heap, "word", + charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL, + field->col->prtype, + FTS_MAX_WORD_LEN_IN_CHAR + * DATA_MBMAXLEN(field->col->mbminmaxlen)); dict_mem_table_add_col(new_table, heap, "first_doc_id", DATA_INT, DATA_NOT_NULL | DATA_UNSIGNED, diff --git a/storage/xtradb/fts/fts0opt.cc b/storage/xtradb/fts/fts0opt.cc index ea937c20752..cb30122adcb 100644 --- a/storage/xtradb/fts/fts0opt.cc +++ b/storage/xtradb/fts/fts0opt.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -281,7 +282,7 @@ fts_zip_initialize( zip->last_big_block = 0; zip->word.f_len = 0; - memset(zip->word.f_str, 0, FTS_MAX_WORD_LEN); + *zip->word.f_str = 0; ib_vector_reset(zip->blocks); @@ -578,9 +579,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -655,10 +653,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -669,7 +666,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index d072e61a237..12ae71508c7 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -107,14 +107,6 @@ this program; if not, write to the Free Software Foundation, Inc., #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define SQLCOM_CREATE_COMPRESSION_DICTIONARY 0xDECAF -#define SQLCOM_DROP_COMPRESSION_DICTIONARY 0xC0FFEE -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -const static LEX_CSTRING null_lex_cstr={0,0}; -#endif - #ifdef MYSQL_DYNAMIC_PLUGIN #define tc_size 400 #define tdc_size 400 @@ -1476,30 +1468,6 @@ normalize_table_name_low( ibool set_lower_case); /* in: TRUE if we want to set name to lower case */ -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len); - /*!< in/out: zip dictionary data length */ - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len); - /*!< in/out: zip dictionary name length */ -#endif /*************************************************************//** Checks if buffer pool is big enough to enable backoff algorithm. InnoDB empty free list algorithm backoff requires free pages @@ -1550,6 +1518,7 @@ static bool innobase_purge_archive_logs( } #endif + /*************************************************************//** Check for a valid value of innobase_commit_concurrency. @return 0 for valid innodb_commit_concurrency */ @@ -2357,12 +2326,11 @@ innobase_get_stmt( { const char* query = NULL; LEX_STRING *stmt = NULL; - if (thd) { - stmt = thd_query_string(thd); - if (stmt) { - *length = stmt->length; - query = stmt->str; - } + ut_ad(thd != NULL); + stmt = thd_query_string(thd); + if (stmt) { + *length = stmt->length; + query = stmt->str; } return (query); } @@ -3591,10 +3559,6 @@ innobase_init( if (srv_file_per_table) innobase_hton->tablefile_extensions = ha_innobase_exts; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - innobase_hton->create_zip_dict = innobase_create_zip_dict; - innobase_hton->drop_zip_dict = innobase_drop_zip_dict; -#endif ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); #ifndef DBUG_OFF @@ -4039,19 +4003,13 @@ innobase_change_buffering_inited_ok: and consequently we do not need to know the ordering internally in InnoDB. */ - ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); srv_latin1_ordering = my_charset_latin1.sort_order; innobase_commit_concurrency_init_default(); -#ifndef EXTENDED_FOR_KILLIDLE - srv_kill_idle_transaction = 0; -#endif - #ifdef HAVE_POSIX_FALLOCATE srv_use_posix_fallocate = (ibool) innobase_use_fallocate; #endif - /* Do not enable backoff algorithm for small buffer pool. */ if (!innodb_empty_free_list_algorithm_backoff_allowed( static_cast<srv_empty_free_list_t>( @@ -4288,90 +4246,6 @@ innobase_purge_changed_page_bitmaps( return (my_bool)log_online_purge_changed_page_bitmaps(lsn); } -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len) - /*!< in/out: zip dictionary data length */ -{ - handler_create_zip_dict_result result = - HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_create_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_CREATE_ZIP_DICT_READ_ONLY); - } - - if (UNIV_UNLIKELY(*name_len > ZIP_DICT_MAX_NAME_LENGTH)) { - *name_len = ZIP_DICT_MAX_NAME_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_NAME_TOO_LONG); - } - - if (UNIV_UNLIKELY(*data_len > ZIP_DICT_MAX_DATA_LENGTH)) { - *data_len = ZIP_DICT_MAX_DATA_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_DATA_TOO_LONG); - } - - switch (dict_create_zip_dict(name, *name_len, data, *data_len)) { - case DB_SUCCESS: - result = HA_CREATE_ZIP_DICT_OK; - break; - case DB_DUPLICATE_KEY: - result = HA_CREATE_ZIP_DICT_ALREADY_EXISTS; - break; - default: - ut_ad(0); - result = HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len) - /*!< in/out: zip dictionary name length */ -{ - handler_drop_zip_dict_result result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_drop_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_DROP_ZIP_DICT_READ_ONLY); - } - - switch (dict_drop_zip_dict(name, *name_len)) { - case DB_SUCCESS: - result = HA_DROP_ZIP_DICT_OK; - break; - case DB_RECORD_NOT_FOUND: - result = HA_DROP_ZIP_DICT_DOES_NOT_EXIST; - break; - case DB_ROW_IS_REFERENCED: - result = HA_DROP_ZIP_DICT_IS_REFERENCED; - break; - default: - ut_ad(0); - result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} -#endif /*****************************************************************//** Check whether this is a fake change transaction. @return TRUE if a fake change transaction */ @@ -5941,88 +5815,6 @@ func_exit: DBUG_RETURN(ret); } -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name) /*!< out: the name of the - zip_dict which does not exist. */ -{ - DBUG_ENTER("innobase_check_zip_dicts"); - - bool res = true; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - Field* field_ptr; - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - field_ptr = table->field[field_idx]; - if (field_ptr->has_associated_compression_dictionary()) { - err = dict_create_get_zip_dict_id_by_name( - field_ptr->zip_dict_name.str, - field_ptr->zip_dict_name.length, - &dict_ids[field_idx], - trx); - ut_a(err == DB_SUCCESS || err == DB_RECORD_NOT_FOUND); - } - else { - dict_ids[field_idx] = ULINT_UNDEFINED; - } - - } - - if (err != DB_SUCCESS) { - res = false; - *err_dict_name = field_ptr->zip_dict_name.str; - } - -#endif - DBUG_RETURN(res); -} - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx) /*!< in: transaction */ -{ - DBUG_ENTER("innobase_create_zip_dict_references"); - - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - if (zip_dict_ids[field_idx] != ULINT_UNDEFINED) { - err = dict_create_add_zip_dict_reference(ib_table_id, - table->field[field_idx]->field_index, - zip_dict_ids[field_idx], trx); - ut_a(err == DB_SUCCESS); - } - } - - DBUG_VOID_RETURN; -} - /*******************************************************************//** This function uses index translation table to quickly locate the requested index structure. @@ -7041,18 +6833,16 @@ get_innobase_type_from_mysql_type( case MYSQL_TYPE_VARCHAR: /* new >= 5.0.3 true VARCHAR */ if (field->binary()) { return(DATA_BINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_VARCHAR); } else { return(DATA_VARMYSQL); } case MYSQL_TYPE_BIT: - case MYSQL_TYPE_STRING: if (field->binary()) { - + case MYSQL_TYPE_STRING: + if (field->binary()) { return(DATA_FIXBINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_CHAR); } else { return(DATA_MYSQL); @@ -7289,16 +7079,7 @@ ha_innobase::store_key_val_for_row( blob_data = row_mysql_read_blob_ref(&blob_len, (byte*) (record + (ulint) get_field_offset(table, field)), - (ulint) field->pack_length(), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast<const byte*>( - field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0, 0, 0, prebuilt); -#endif + (ulint) field->pack_length()); true_len = blob_len; @@ -7613,14 +7394,6 @@ build_template_field( templ->mbminlen = dict_col_get_mbminlen(col); templ->mbmaxlen = dict_col_get_mbmaxlen(col); templ->is_unsigned = col->prtype & DATA_UNSIGNED; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - templ->compressed = (field->column_format() - == COLUMN_FORMAT_TYPE_COMPRESSED); - templ->zip_dict_data = field->zip_dict_data; -#else - templ->compressed = 0; - templ->zip_dict_data = null_lex_cstr; -#endif if (!dict_index_is_clust(index) && templ->rec_field_no == ULINT_UNDEFINED) { @@ -8409,10 +8182,8 @@ calc_row_difference( case DATA_BLOB: /* Do not compress blob column while comparing*/ - o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len, - false, 0, 0, prebuilt); - n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len, - false, 0, 0, prebuilt); + o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len); + n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len); break; @@ -8482,17 +8253,7 @@ calc_row_difference( TRUE, new_mysql_row_col, col_pack_len, - dict_table_is_comp(prebuilt->table), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast<const byte*>( - field->zip_dict_data.str), - field->zip_dict_data.length, -#else - 0, 0, 0, -#endif - prebuilt); + dict_table_is_comp(prebuilt->table)); dfield_copy(&ufield->new_val, &dfield); } else { dfield_set_null(&ufield->new_val); @@ -10179,7 +9940,6 @@ create_table_def( ulint unsigned_type; ulint binary_type; ulint long_true_varchar; - ulint compressed; ulint charset_no; ulint i; ulint doc_id_col = 0; @@ -10329,13 +10089,6 @@ create_table_def( } } - /* Check if the the field has COMPRESSED attribute */ - compressed = 0; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - compressed = DATA_COMPRESSED; - } - /* First check whether the column to be added has a system reserved name. */ if (dict_col_name_is_reserved(field->field_name)){ @@ -10356,8 +10109,7 @@ err_col: dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type - | binary_type | long_true_varchar - | compressed, + | binary_type | long_true_varchar, charset_no), col_len); } @@ -11187,9 +10939,6 @@ ha_innobase::create( const char* stmt; size_t stmt_len; - mem_heap_t* heap = 0; - ulint* zip_dict_ids = 0; - DBUG_ENTER("ha_innobase::create"); DBUG_ASSERT(thd != NULL); @@ -11280,25 +11029,6 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); - heap = mem_heap_create(form->s->fields * sizeof(ulint)); - zip_dict_ids = static_cast<ulint*>( - mem_heap_alloc(heap, form->s->fields * sizeof(ulint))); - - /* This is currently required for valgrind because MariaDB does - not currently support compressed columns. */ - for (size_t field_idx = 0; field_idx < form->s->fields; ++field_idx) { - zip_dict_ids[field_idx] = ULINT_UNDEFINED; - } - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(form, zip_dict_ids, - trx, &err_zip_dict_name)) { - error = -1; - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto cleanup; - } - error = create_table_def(trx, form, norm_name, temp_path, remote_path, flags, flags2); if (error) { @@ -11406,22 +11136,6 @@ ha_innobase::create( dict_table_get_all_fts_indexes(innobase_table, fts->indexes); } - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - ut_a(zip_dict_ids != 0); - { - dict_table_t* local_table = dict_table_open_on_name( - norm_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); - - ut_a(local_table); - table_id_t table_id = local_table->id; - dict_table_close(local_table, TRUE, FALSE); - innobase_create_zip_dict_references(form, - table_id, zip_dict_ids, trx); - } - stmt = innobase_get_stmt(thd, &stmt_len); if (stmt) { @@ -11538,9 +11252,6 @@ ha_innobase::create( trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(0); cleanup: @@ -11550,9 +11261,6 @@ cleanup: trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(error); } @@ -12643,9 +12351,13 @@ ha_innobase::info_low( /* If this table is already queued for background analyze, remove it from the queue as we are about to do the same */ - dict_mutex_enter_for_mysql(); - dict_stats_recalc_pool_del(ib_table); - dict_mutex_exit_for_mysql(); + if (!srv_read_only_mode) { + + dict_mutex_enter_for_mysql(); + dict_stats_recalc_pool_del( + ib_table); + dict_mutex_exit_for_mysql(); + } opt = DICT_STATS_RECALC_PERSISTENT; } else { @@ -13834,10 +13546,6 @@ ha_innobase::extra( row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - break; case HA_EXTRA_RESET_STATE: reset_template(); @@ -13889,10 +13597,6 @@ ha_innobase::reset() row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - reset_template(); ds_mrr.dsmrr_close(); @@ -14092,11 +13796,7 @@ ha_innobase::external_lock( && lock_type == F_WRLCK) || thd_sql_command(thd) == SQLCOM_CREATE_INDEX || thd_sql_command(thd) == SQLCOM_DROP_INDEX - || thd_sql_command(thd) == SQLCOM_DELETE - || thd_sql_command(thd) == - SQLCOM_CREATE_COMPRESSION_DICTIONARY - || thd_sql_command(thd) == - SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || thd_sql_command(thd) == SQLCOM_DELETE)) { if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { @@ -14864,9 +14564,7 @@ ha_innobase::store_lock( && lock_type <= TL_WRITE)) || sql_command == SQLCOM_CREATE_INDEX || sql_command == SQLCOM_DROP_INDEX - || sql_command == SQLCOM_DELETE - || sql_command == SQLCOM_CREATE_COMPRESSION_DICTIONARY - || sql_command == SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || sql_command == SQLCOM_DELETE)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); @@ -15157,6 +14855,37 @@ ha_innobase::get_auto_increment( ulonglong col_max_value = innobase_get_int_col_max_value( table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { @@ -15807,84 +15536,6 @@ ha_innobase::check_if_incompatible_data( return(COMPATIBLE_DATA_YES); } -/** This function reads zip dict-related info from SYS_ZIP_DICT -and SYS_ZIP_DICT_COLS for all columns marked with -COLUMN_FORMAT_TYPE_COMPRESSED flag and updates -zip_dict_name / zip_dict_data for those which have associated -compression dictionaries. -*/ -UNIV_INTERN -void -ha_innobase::update_field_defs_with_zip_dict_info() -{ - DBUG_ENTER("update_field_defs_with_zip_dict_info"); - ut_ad(!mutex_own(&dict_sys->mutex)); - - char norm_name[FN_REFLEN]; - normalize_table_name(norm_name, table_share->normalized_path.str); - - dict_table_t* ib_table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - - /* if dict_table_open_on_name() returns NULL, then it means that - TABLE_SHARE is populated for a table being created and we can - skip filling zip dict info here */ - if (ib_table == 0) - DBUG_VOID_RETURN; - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - table_id_t ib_table_id = ib_table->id; - dict_table_close(ib_table, FALSE, FALSE); - Field* field; - for (uint i = 0; i < table_share->fields; ++i) { - field = table_share->field[i]; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - bool reference_found = false; - ulint dict_id = 0; - switch (dict_get_dictionary_id_by_key(ib_table_id, i, - &dict_id)) { - case DB_SUCCESS: - reference_found = true; - break; - case DB_RECORD_NOT_FOUND: - reference_found = false; - break; - default: - ut_error; - } - if (reference_found) { - char* local_name = 0; - ulint local_name_len = 0; - char* local_data = 0; - ulint local_data_len = 0; - if (dict_get_dictionary_info_by_id(dict_id, - &local_name, &local_name_len, - &local_data, &local_data_len) != - DB_SUCCESS) { - ut_error; - } - else { - field->zip_dict_name.str = - local_name; - field->zip_dict_name.length = - local_name_len; - field->zip_dict_data.str = - local_data; - field->zip_dict_data.length = - local_data_len; - } - } - else { - field->zip_dict_name = null_lex_cstr; - field->zip_dict_data = null_lex_cstr; - } - } - } -#endif - DBUG_VOID_RETURN; -} - /****************************************************************//** Update the system variable innodb_io_capacity_max using the "saved" value. This function is registered as a callback with MySQL. */ @@ -17693,32 +17344,6 @@ innobase_fts_retrieve_ranking( } /*********************************************************************** -functions for kill session of idle transaction */ -ibool -innobase_thd_is_idle( -/*=================*/ - const void* thd) /*!< in: thread handle (THD*) */ -{ -#ifdef EXTENDED_FOR_KILLIDLE - return(thd_command((const THD*) thd) == COM_SLEEP); -#else - return(FALSE); -#endif -} - -ib_int64_t -innobase_thd_get_start_time( -/*========================*/ - const void* thd) /*!< in: thread handle (THD*) */ -{ -#ifdef EXTENDED_FOR_KILLIDLE - return((ib_int64_t)thd_start_time((const THD*) thd)); -#else - return(0); /*dummy value*/ -#endif -} - -/*********************************************************************** Free the memory for the FTS handler */ UNIV_INTERN void @@ -17737,19 +17362,6 @@ innobase_fts_close_ranking( return; } -UNIV_INTERN -void -innobase_thd_kill( -/*==============*/ - ulong thd_id) -{ -#ifdef EXTENDED_FOR_KILLIDLE - thd_kill(thd_id); -#else - return; -#endif -} - /*********************************************************************** Find and Retrieve the FTS Relevance Ranking result for doc with doc_id of prebuilt->fts_doc_id @@ -17947,16 +17559,6 @@ innobase_fts_retrieve_docid( } -ulong -innobase_thd_get_thread_id( -/*=======================*/ - const void* thd) -{ - return(thd_get_thread_id((const THD*) thd)); -} - - - /*********************************************************************** Find and retrieve the size of the current result @return number of matching rows */ @@ -18245,6 +17847,12 @@ static MYSQL_SYSVAR_BOOL(doublewrite, innobase_use_doublewrite, "Disable with --skip-innodb-doublewrite.", NULL, NULL, TRUE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Scan delete marked records for persistent stat", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_BOOL(use_atomic_writes, innobase_use_atomic_writes, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, "Prevent partial page writes, via atomic writes (beta). " @@ -18887,13 +18495,6 @@ static MYSQL_SYSVAR_ULONG(force_recovery, srv_force_recovery, "Helps to save your data in case the disk image of the database becomes corrupt.", NULL, NULL, 0, 0, 6, 0); -#ifndef DBUG_OFF -static MYSQL_SYSVAR_ULONG(force_recovery_crash, srv_force_recovery_crash, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Kills the server during crash recovery.", - NULL, NULL, 0, 0, 10, 0); -#endif /* !DBUG_OFF */ - static MYSQL_SYSVAR_ULONG(page_size, srv_page_size, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, "Page size to use for all InnoDB tablespaces.", @@ -19276,21 +18877,6 @@ static MYSQL_SYSVAR_BOOL(use_stacktrace, srv_use_stacktrace, "Print stacktrace on long semaphore wait (off by default supported only on linux)", NULL, NULL, FALSE); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -static MYSQL_SYSVAR_UINT(compressed_columns_zip_level, - srv_compressed_columns_zip_level, - PLUGIN_VAR_RQCMDARG, - "Compression level used for compressed columns. 0 is no compression" - ", 1 is fastest and 9 is best compression. Default is 6.", - NULL, NULL, DEFAULT_COMPRESSION_LEVEL, 0, 9, 0); - -static MYSQL_SYSVAR_ULONG(compressed_columns_threshold, - srv_compressed_columns_threshold, - PLUGIN_VAR_RQCMDARG, - "Compress column data if its length exceeds this value. Default is 96", - NULL, NULL, 96, 1, ~0UL, 0); -#endif - static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), MYSQL_SYSVAR(additional_mem_pool_size), @@ -19322,6 +18908,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(data_file_path), MYSQL_SYSVAR(data_home_dir), MYSQL_SYSVAR(doublewrite), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), @@ -19340,9 +18927,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(use_global_flush_log_at_trx_commit), MYSQL_SYSVAR(flush_method), MYSQL_SYSVAR(force_recovery), -#ifndef DBUG_OFF - MYSQL_SYSVAR(force_recovery_crash), -#endif /* !DBUG_OFF */ MYSQL_SYSVAR(ft_cache_size), MYSQL_SYSVAR(ft_total_cache_size), MYSQL_SYSVAR(ft_result_cache_limit), @@ -19494,10 +19078,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(use_stacktrace), MYSQL_SYSVAR(simulate_comp_failures), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - MYSQL_SYSVAR(compressed_columns_zip_level), - MYSQL_SYSVAR(compressed_columns_threshold), -#endif NULL }; @@ -19520,10 +19100,6 @@ maria_declare_plugin(xtradb) i_s_xtradb_read_view, i_s_xtradb_internal_hash_tables, i_s_xtradb_rseg, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -i_s_xtradb_zip_dict, -i_s_xtradb_zip_dict_cols, -#endif i_s_innodb_trx, i_s_innodb_locks, i_s_innodb_lock_waits, diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index f01faaf41d8..38e45647a7c 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -144,6 +144,8 @@ class ha_innobase: public handler int index_first(uchar * buf); int index_last(uchar * buf); + bool has_gap_locks() const { return true; } + int rnd_init(bool scan); int rnd_end(); int rnd_next(uchar *buf); @@ -290,14 +292,6 @@ class ha_innobase: public handler bool check_if_supported_virtual_columns(void) { return TRUE; } - /** This function reads zip dict-related info from SYS_ZIP_DICT - and SYS_ZIP_DICT_COLS for all columns marked with - COLUMN_FORMAT_TYPE_COMPRESSED flag and updates - zip_dict_name / zip_dict_data for those which have associated - compression dictionaries. - */ - virtual void update_field_defs_with_zip_dict_info(); - private: /** Builds a 'template' to the prebuilt struct. @@ -691,31 +685,3 @@ ib_push_frm_error( TABLE* table, /*!< in: MySQL table */ ulint n_keys, /*!< in: InnoDB #keys */ bool push_warning); /*!< in: print warning ? */ - -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name); /*!< out: the name of the - zip_dict which does not exist. */ - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx); /*!< in: transaction */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 362ac9c1b89..3d20c9abccf 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -21,11 +22,6 @@ this program; if not, write to the Free Software Foundation, Inc., Smart ALTER TABLE *******************************************************/ -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -#endif - #include <my_global.h> #include <unireg.h> #include <mysqld_error.h> @@ -134,7 +130,6 @@ my_error_innodb( break; case DB_OUT_OF_FILE_SPACE: my_error(ER_RECORD_FILE_FULL, MYF(0), table); - ut_error; break; case DB_TEMP_FILE_WRITE_FAILURE: my_error(ER_GET_ERRMSG, MYF(0), @@ -1185,15 +1180,6 @@ innobase_col_to_mysql( field->reset(); if (field->type() == MYSQL_TYPE_VARCHAR) { - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - /* Skip compressed varchar column when - reporting an erroneous row - during index creation or table rebuild. */ - field->set_null(); - break; - } - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -1852,6 +1838,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -1920,26 +1907,11 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); - - if (maybe_null) { - new_primary = false; - break; - } - } + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id @@ -1958,8 +1930,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -2492,14 +2470,7 @@ innobase_build_col_map_add( byte* buf = static_cast<byte*>(mem_heap_alloc(heap, size)); row_mysql_store_col_in_innobase_format( - dfield, buf, TRUE, field->ptr, size, comp, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast<const byte*>(field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0,0,0, prebuilt); -#endif + dfield, buf, TRUE, field->ptr, size, comp); } /** Construct the translation table for reordering, dropping or @@ -2753,7 +2724,6 @@ prepare_inplace_alter_table_dict( ulint num_fts_index; ha_innobase_inplace_ctx*ctx; uint sql_idx; - ulint* zip_dict_ids = 0; DBUG_ENTER("prepare_inplace_alter_table_dict"); @@ -2890,26 +2860,6 @@ prepare_inplace_alter_table_dict( ulint n_cols; dtuple_t* add_cols; - zip_dict_ids = static_cast<ulint*>( - mem_heap_alloc(ctx->heap, - altered_table->s->fields * sizeof(ulint))); - - /* This is currently required for valgrind because MariaDB does - not currently support compressed columns. */ - for (size_t field_idx = 0; - field_idx < altered_table->s->fields; - ++field_idx) { - zip_dict_ids[field_idx] = ULINT_UNDEFINED; - } - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(altered_table, zip_dict_ids, - ctx->trx, &err_zip_dict_name)) { - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto new_clustered_failed; - } - if (innobase_check_foreigns( ha_alter_info, altered_table, old_table, user_table, ctx->drop_fk, ctx->num_to_drop_fk)) { @@ -3016,12 +2966,6 @@ prepare_inplace_alter_table_dict( } } - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - field_type |= DATA_COMPRESSED; - } - - if (dict_col_name_is_reserved(field->field_name)) { dict_mem_table_free(ctx->new_table); my_error(ER_WRONG_COLUMN_NAME, MYF(0), @@ -3105,6 +3049,8 @@ prepare_inplace_alter_table_dict( ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info, old_table)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); if (!ctx->new_table->fts && innobase_fulltext_exist(altered_table)) { @@ -3279,17 +3225,6 @@ op_ok: DBUG_ASSERT(error == DB_SUCCESS); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - if (zip_dict_ids != 0) { - innobase_create_zip_dict_references(altered_table, - ctx->trx->table_id, zip_dict_ids, ctx->trx); - } -#endif - /* Commit the data dictionary transaction in order to release the table locks on the system tables. This means that if MySQL crashes while creating a new primary key inside @@ -3962,7 +3897,7 @@ check_if_can_drop_indexes: index->name, TRUE); my_error(ER_INDEX_CORRUPT, MYF(0), index_name); - DBUG_RETURN(true); + goto err_exit; } } } @@ -4145,6 +4080,27 @@ found_col: add_fts_doc_id_idx, prebuilt)); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} + /** Alter the table structure in-place with operations specified using Alter_inplace_info. The level of concurrency allowed during this operation depends @@ -4262,17 +4218,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; default: my_error_innodb(error, @@ -5085,7 +5037,6 @@ innobase_update_foreign_cache( "Foreign key constraints for table '%s'" " are loaded with charset check off", user_table->name); - } } @@ -5185,14 +5136,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc index 59cad1c2e7a..aac1074e152 100644 --- a/storage/xtradb/handler/i_s.cc +++ b/storage/xtradb/handler/i_s.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -155,6 +156,7 @@ struct buf_page_info_t{ if ((expr) != 0) { \ DBUG_RETURN(1); \ } +#define BREAK_IF(expr) if ((expr)) break #define RETURN_IF_INNODB_NOT_STARTED(plugin_name) \ do { \ @@ -2989,14 +2991,16 @@ i_s_fts_deleted_generic_fill( fields = table->field; + int ret = 0; + for (ulint j = 0; j < ib_vector_size(deleted->doc_ids); ++j) { doc_id_t doc_id; doc_id = *(doc_id_t*) ib_vector_get_const(deleted->doc_ids, j); - OK(fields[I_S_FTS_DOC_ID]->store((longlong) doc_id, true)); + BREAK_IF(ret = fields[I_S_FTS_DOC_ID]->store(doc_id, true)); - OK(schema_table_store_record(thd, table)); + BREAK_IF(ret = schema_table_store_record(thd, table)); } trx_free_for_background(trx); @@ -3007,7 +3011,7 @@ i_s_fts_deleted_generic_fill( rw_lock_s_unlock(&dict_operation_lock); - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** @@ -3245,13 +3249,13 @@ i_s_fts_index_cache_fill_one_index( /*===============================*/ fts_index_cache_t* index_cache, /*!< in: FTS index cache */ THD* thd, /*!< in: thread */ + fts_string_t* conv_str, /*!< in/out: buffer */ TABLE_LIST* tables) /*!< in/out: tables to fill */ { TABLE* table = (TABLE*) tables->table; Field** fields; CHARSET_INFO* index_charset; const ib_rbt_node_t* rbt_node; - fts_string_t conv_str; uint dummy_errors; char* word_str; @@ -3260,10 +3264,9 @@ i_s_fts_index_cache_fill_one_index( fields = table->field; index_charset = index_cache->charset; - conv_str.f_len = system_charset_info->mbmaxlen - * FTS_MAX_WORD_LEN_IN_CHAR; - conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); - conv_str.f_n_char = 0; + conv_str->f_n_char = 0; + + int ret = 0; /* Go through each word in the index cache */ for (rbt_node = rbt_first(index_cache->words); @@ -3275,16 +3278,16 @@ i_s_fts_index_cache_fill_one_index( /* Convert word from index charset to system_charset_info */ if (index_charset->cset != system_charset_info->cset) { - conv_str.f_n_char = my_convert( - reinterpret_cast<char*>(conv_str.f_str), - static_cast<uint32>(conv_str.f_len), + conv_str->f_n_char = my_convert( + reinterpret_cast<char*>(conv_str->f_str), + static_cast<uint32>(conv_str->f_len), system_charset_info, reinterpret_cast<char*>(word->text.f_str), static_cast<uint32>(word->text.f_len), index_charset, &dummy_errors); - ut_ad(conv_str.f_n_char <= conv_str.f_len); - conv_str.f_str[conv_str.f_n_char] = 0; - word_str = reinterpret_cast<char*>(conv_str.f_str); + ut_ad(conv_str->f_n_char <= conv_str->f_len); + conv_str->f_str[conv_str->f_n_char] = 0; + word_str = reinterpret_cast<char*>(conv_str->f_str); } else { word_str = reinterpret_cast<char*>(word->text.f_str); } @@ -3342,9 +3345,7 @@ i_s_fts_index_cache_fill_one_index( } } - ut_free(conv_str.f_str); - - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** Fill the dynamic table INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHED @@ -3388,18 +3389,27 @@ i_s_fts_index_cache_fill( ut_a(cache); + int ret = 0; + fts_string_t conv_str; + conv_str.f_len = system_charset_info->mbmaxlen + * FTS_MAX_WORD_LEN_IN_CHAR; + conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); + for (ulint i = 0; i < ib_vector_size(cache->indexes); i++) { fts_index_cache_t* index_cache; index_cache = static_cast<fts_index_cache_t*> ( ib_vector_get(cache->indexes, i)); - i_s_fts_index_cache_fill_one_index(index_cache, thd, tables); + BREAK_IF(ret = i_s_fts_index_cache_fill_one_index( + index_cache, thd, &conv_str, tables)); } + ut_free(conv_str.f_str); + dict_table_close(user_table, FALSE, FALSE); - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** @@ -3702,8 +3712,6 @@ i_s_fts_index_table_fill_one_fetch( } } - i_s_fts_index_table_free_one_fetch(words); - DBUG_RETURN(ret); } @@ -3717,13 +3725,13 @@ i_s_fts_index_table_fill_one_index( /*===============================*/ dict_index_t* index, /*!< in: FTS index */ THD* thd, /*!< in: thread */ + fts_string_t* conv_str, /*!< in/out: buffer */ TABLE_LIST* tables) /*!< in/out: tables to fill */ { ib_vector_t* words; mem_heap_t* heap; fts_string_t word; CHARSET_INFO* index_charset; - fts_string_t conv_str; dberr_t error; int ret = 0; @@ -3740,10 +3748,6 @@ i_s_fts_index_table_fill_one_index( word.f_n_char = 0; index_charset = fts_index_get_charset(index); - conv_str.f_len = system_charset_info->mbmaxlen - * FTS_MAX_WORD_LEN_IN_CHAR; - conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); - conv_str.f_n_char = 0; /* Iterate through each auxiliary table as described in fts_index_selector */ @@ -3777,17 +3781,17 @@ i_s_fts_index_table_fill_one_index( /* Fill into tables */ ret = i_s_fts_index_table_fill_one_fetch( - index_charset, thd, tables, words, &conv_str, has_more); + index_charset, thd, tables, words, conv_str, + has_more); + i_s_fts_index_table_free_one_fetch(words); if (ret != 0) { - i_s_fts_index_table_free_one_fetch(words); goto func_exit; } } while (has_more); } func_exit: - ut_free(conv_str.f_str); mem_heap_free(heap); DBUG_RETURN(ret); @@ -3829,10 +3833,17 @@ i_s_fts_index_table_fill( DBUG_RETURN(0); } + int ret = 0; + fts_string_t conv_str; + conv_str.f_len = system_charset_info->mbmaxlen + * FTS_MAX_WORD_LEN_IN_CHAR; + conv_str.f_str = static_cast<byte*>(ut_malloc(conv_str.f_len)); + for (index = dict_table_get_first_index(user_table); index; index = dict_table_get_next_index(index)) { if (index->type & DICT_FTS) { - i_s_fts_index_table_fill_one_index(index, thd, tables); + BREAK_IF(ret = i_s_fts_index_table_fill_one_index( + index, thd, &conv_str, tables)); } } @@ -3840,7 +3851,9 @@ i_s_fts_index_table_fill( rw_lock_s_unlock(&dict_operation_lock); - DBUG_RETURN(0); + ut_free(conv_str.f_str); + + DBUG_RETURN(ret); } /*******************************************************************//** @@ -4005,6 +4018,8 @@ i_s_fts_config_fill( DBUG_ASSERT(!dict_index_is_online_ddl(index)); } + int ret = 0; + while (fts_config_key[i]) { fts_string_t value; char* key_name; @@ -4029,13 +4044,14 @@ i_s_fts_config_fill( ut_free(key_name); } - OK(field_store_string( - fields[FTS_CONFIG_KEY], fts_config_key[i])); + BREAK_IF(ret = field_store_string( + fields[FTS_CONFIG_KEY], fts_config_key[i])); - OK(field_store_string( - fields[FTS_CONFIG_VALUE], (const char*) value.f_str)); + BREAK_IF(ret = field_store_string( + fields[FTS_CONFIG_VALUE], + reinterpret_cast<const char*>(value.f_str))); - OK(schema_table_store_record(thd, table)); + BREAK_IF(ret = schema_table_store_record(thd, table)); i++; } @@ -4048,7 +4064,7 @@ i_s_fts_config_fill( rw_lock_s_unlock(&dict_operation_lock); - DBUG_RETURN(0); + DBUG_RETURN(ret); } /*******************************************************************//** @@ -4883,34 +4899,29 @@ i_s_innodb_buffer_page_fill( state_str = NULL; OK(fields[IDX_BUFFER_POOL_ID]->store( - static_cast<double>(page_info->pool_id))); + page_info->pool_id, true)); OK(fields[IDX_BUFFER_BLOCK_ID]->store( - static_cast<double>(page_info->block_id))); + page_info->block_id, true)); OK(fields[IDX_BUFFER_PAGE_SPACE]->store( - static_cast<double>(page_info->space_id))); + page_info->space_id, true)); OK(fields[IDX_BUFFER_PAGE_NUM]->store( - static_cast<double>(page_info->page_num))); + page_info->page_num, true)); OK(field_store_string( fields[IDX_BUFFER_PAGE_TYPE], i_s_page_type[page_info->page_type].type_str)); OK(fields[IDX_BUFFER_PAGE_FLUSH_TYPE]->store( - page_info->flush_type)); + page_info->flush_type, true)); OK(fields[IDX_BUFFER_PAGE_FIX_COUNT]->store( - page_info->fix_count)); + page_info->fix_count, true)); - if (page_info->hashed) { - OK(field_store_string( - fields[IDX_BUFFER_PAGE_HASHED], "YES")); - } else { - OK(field_store_string( - fields[IDX_BUFFER_PAGE_HASHED], "NO")); - } + OK(field_store_string(fields[IDX_BUFFER_PAGE_HASHED], + page_info->hashed ? "YES" : "NO")); OK(fields[IDX_BUFFER_PAGE_NEWEST_MOD]->store( (longlong) page_info->newest_mod, true)); @@ -4919,7 +4930,7 @@ i_s_innodb_buffer_page_fill( (longlong) page_info->oldest_mod, true)); OK(fields[IDX_BUFFER_PAGE_ACCESS_TIME]->store( - page_info->access_time)); + page_info->access_time, true)); fields[IDX_BUFFER_PAGE_TABLE_NAME]->set_null(); @@ -4928,44 +4939,48 @@ i_s_innodb_buffer_page_fill( /* If this is an index page, fetch the index name and table name */ if (page_info->page_type == I_S_PAGE_TYPE_INDEX) { - const dict_index_t* index; + bool ret = false; mutex_enter(&dict_sys->mutex); - index = dict_index_get_if_in_cache_low( - page_info->index_id); - - if (index) { + if (const dict_index_t* index = + dict_index_get_if_in_cache_low( + page_info->index_id)) { table_name_end = innobase_convert_name( table_name, sizeof(table_name), index->table_name, strlen(index->table_name), thd, TRUE); - OK(fields[IDX_BUFFER_PAGE_TABLE_NAME]->store( - table_name, - static_cast<uint>(table_name_end - table_name), - system_charset_info)); - fields[IDX_BUFFER_PAGE_TABLE_NAME]->set_notnull(); - - OK(field_store_index_name( - fields[IDX_BUFFER_PAGE_INDEX_NAME], - index->name)); + ret = fields[IDX_BUFFER_PAGE_TABLE_NAME] + ->store(table_name, + static_cast<uint>( + table_name_end + - table_name), + system_charset_info) + || field_store_index_name( + fields + [IDX_BUFFER_PAGE_INDEX_NAME], + index->name); } mutex_exit(&dict_sys->mutex); + + OK(ret); + + fields[IDX_BUFFER_PAGE_TABLE_NAME]->set_notnull(); } OK(fields[IDX_BUFFER_PAGE_NUM_RECS]->store( - page_info->num_recs)); + page_info->num_recs, true)); OK(fields[IDX_BUFFER_PAGE_DATA_SIZE]->store( - page_info->data_size)); + page_info->data_size, true)); OK(fields[IDX_BUFFER_PAGE_ZIP_SIZE]->store( - page_info->zip_ssize - ? (UNIV_ZIP_SIZE_MIN >> 1) << page_info->zip_ssize - : 0)); + page_info->zip_ssize + ? (UNIV_ZIP_SIZE_MIN >> 1) << page_info->zip_ssize + : 0, true)); #if BUF_PAGE_STATE_BITS > 3 # error "BUF_PAGE_STATE_BITS > 3, please ensure that all 1<<BUF_PAGE_STATE_BITS values are checked for" @@ -5003,32 +5018,29 @@ i_s_innodb_buffer_page_fill( switch (page_info->io_fix) { case BUF_IO_NONE: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_NONE")); + state_str = "IO_NONE"; break; case BUF_IO_READ: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_READ")); + state_str = "IO_READ"; break; case BUF_IO_WRITE: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_WRITE")); + state_str = "IO_WRITE"; break; case BUF_IO_PIN: - OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], - "IO_PIN")); + state_str = "IO_PIN"; break; } + OK(field_store_string(fields[IDX_BUFFER_PAGE_IO_FIX], + state_str)); + OK(field_store_string(fields[IDX_BUFFER_PAGE_IS_OLD], (page_info->is_old) ? "YES" : "NO")); OK(fields[IDX_BUFFER_PAGE_FREE_CLOCK]->store( page_info->freed_page_clock)); - if (schema_table_store_record(thd, table)) { - DBUG_RETURN(1); - } + OK(schema_table_store_record(thd, table)); } DBUG_RETURN(0); @@ -5569,17 +5581,10 @@ i_s_innodb_buf_page_lru_fill( ulint num_page) /*!< in: number of page info cached */ { - TABLE* table; - Field** fields; - mem_heap_t* heap; - DBUG_ENTER("i_s_innodb_buf_page_lru_fill"); - table = tables->table; - - fields = table->field; - - heap = mem_heap_create(1000); + TABLE* table = tables->table; + Field** fields = table->field; /* Iterate through the cached array and fill the I_S table rows */ for (ulint i = 0; i < num_page; i++) { @@ -5594,43 +5599,37 @@ i_s_innodb_buf_page_lru_fill( page_info = info_array + i; OK(fields[IDX_BUF_LRU_POOL_ID]->store( - static_cast<double>(page_info->pool_id))); - + page_info->pool_id, true)); OK(fields[IDX_BUF_LRU_POS]->store( - static_cast<double>(page_info->block_id))); + page_info->block_id, true)); OK(fields[IDX_BUF_LRU_PAGE_SPACE]->store( - static_cast<double>(page_info->space_id))); + page_info->space_id, true)); OK(fields[IDX_BUF_LRU_PAGE_NUM]->store( - static_cast<double>(page_info->page_num))); + page_info->page_num, true)); OK(field_store_string( - fields[IDX_BUF_LRU_PAGE_TYPE], - i_s_page_type[page_info->page_type].type_str)); + fields[IDX_BUF_LRU_PAGE_TYPE], + i_s_page_type[page_info->page_type].type_str)); OK(fields[IDX_BUF_LRU_PAGE_FLUSH_TYPE]->store( - static_cast<double>(page_info->flush_type))); + page_info->flush_type, true)); OK(fields[IDX_BUF_LRU_PAGE_FIX_COUNT]->store( - static_cast<double>(page_info->fix_count))); + page_info->fix_count, true)); - if (page_info->hashed) { - OK(field_store_string( - fields[IDX_BUF_LRU_PAGE_HASHED], "YES")); - } else { - OK(field_store_string( - fields[IDX_BUF_LRU_PAGE_HASHED], "NO")); - } + OK(field_store_string(fields[IDX_BUF_LRU_PAGE_HASHED], + page_info->hashed ? "YES" : "NO")); OK(fields[IDX_BUF_LRU_PAGE_NEWEST_MOD]->store( - page_info->newest_mod, true)); + page_info->newest_mod, true)); OK(fields[IDX_BUF_LRU_PAGE_OLDEST_MOD]->store( - page_info->oldest_mod, true)); + page_info->oldest_mod, true)); OK(fields[IDX_BUF_LRU_PAGE_ACCESS_TIME]->store( - page_info->access_time)); + page_info->access_time, true)); fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->set_null(); @@ -5639,43 +5638,47 @@ i_s_innodb_buf_page_lru_fill( /* If this is an index page, fetch the index name and table name */ if (page_info->page_type == I_S_PAGE_TYPE_INDEX) { - const dict_index_t* index; + bool ret = false; mutex_enter(&dict_sys->mutex); - index = dict_index_get_if_in_cache_low( - page_info->index_id); - - if (index) { + if (const dict_index_t* index = + dict_index_get_if_in_cache_low( + page_info->index_id)) { table_name_end = innobase_convert_name( table_name, sizeof(table_name), index->table_name, strlen(index->table_name), thd, TRUE); - OK(fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->store( - table_name, - static_cast<uint>(table_name_end - table_name), - system_charset_info)); - fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->set_notnull(); - - OK(field_store_index_name( - fields[IDX_BUF_LRU_PAGE_INDEX_NAME], - index->name)); + ret = fields[IDX_BUF_LRU_PAGE_TABLE_NAME] + ->store(table_name, + static_cast<uint>( + table_name_end + - table_name), + system_charset_info) + || field_store_index_name( + fields + [IDX_BUF_LRU_PAGE_INDEX_NAME], + index->name); } mutex_exit(&dict_sys->mutex); + + OK(ret); + + fields[IDX_BUF_LRU_PAGE_TABLE_NAME]->set_notnull(); } OK(fields[IDX_BUF_LRU_PAGE_NUM_RECS]->store( - page_info->num_recs)); + page_info->num_recs, true)); OK(fields[IDX_BUF_LRU_PAGE_DATA_SIZE]->store( - page_info->data_size)); + page_info->data_size, true)); OK(fields[IDX_BUF_LRU_PAGE_ZIP_SIZE]->store( - page_info->zip_ssize ? - 512 << page_info->zip_ssize : 0)); + page_info->zip_ssize + ? 512 << page_info->zip_ssize : 0, true)); state = static_cast<enum buf_page_state>(page_info->page_state); @@ -5704,35 +5707,31 @@ i_s_innodb_buf_page_lru_fill( switch (page_info->io_fix) { case BUF_IO_NONE: - OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], - "IO_NONE")); + state_str = "IO_NONE"; break; case BUF_IO_READ: - OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], - "IO_READ")); + state_str = "IO_READ"; break; case BUF_IO_WRITE: - OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], - "IO_WRITE")); + state_str = "IO_WRITE"; + break; + case BUF_IO_PIN: + state_str = "IO_PIN"; break; } + OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IO_FIX], + state_str)); + OK(field_store_string(fields[IDX_BUF_LRU_PAGE_IS_OLD], - (page_info->is_old) ? "YES" : "NO")); + page_info->is_old ? "YES" : "NO")); OK(fields[IDX_BUF_LRU_PAGE_FREE_CLOCK]->store( - page_info->freed_page_clock)); - - if (schema_table_store_record(thd, table)) { - mem_heap_free(heap); - DBUG_RETURN(1); - } + page_info->freed_page_clock, true)); - mem_heap_empty(heap); + OK(schema_table_store_record(thd, table)); } - mem_heap_free(heap); - DBUG_RETURN(0); } @@ -8343,29 +8342,7 @@ i_s_innodb_changed_pages_fill( while(log_online_bitmap_iterator_next(&i) && (!srv_max_changed_pages || - output_rows_num < srv_max_changed_pages) && - /* - There is no need to compare both start LSN and end LSN fields - with maximum value. It's enough to compare only start LSN. - Example: - - max_lsn = 100 - \\\\\\\\\\\\\\\\\\\\\\\\\|\\\\\\\\ - Query 1 - I------I I-------I I-------------I I----I - ////////////////// | - Query 2 - 1 2 3 4 - - Query 1: - SELECT * FROM INNODB_CHANGED_PAGES WHERE start_lsn < 100 - will select 1,2,3 bitmaps - Query 2: - SELECT * FROM INNODB_CHANGED_PAGES WHERE end_lsn < 100 - will select 1,2 bitmaps - - The condition start_lsn <= 100 will be false after reading - 1,2,3 bitmaps which suits for both cases. - */ - LOG_BITMAP_ITERATOR_START_LSN(i) <= max_lsn) + output_rows_num < srv_max_changed_pages)) { if (!LOG_BITMAP_ITERATOR_PAGE_CHANGED(i)) continue; diff --git a/storage/xtradb/handler/xtradb_i_s.cc b/storage/xtradb/handler/xtradb_i_s.cc index eb6637dad03..84c1e5852bc 100644 --- a/storage/xtradb/handler/xtradb_i_s.cc +++ b/storage/xtradb/handler/xtradb_i_s.cc @@ -2,6 +2,7 @@ Copyright (c) 2007, 2012, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,7 +37,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include <btr0sea.h> /* btr_search_sys */ #include <log0recv.h> /* recv_sys */ #include <fil0fil.h> -#include <dict0crea.h> /* for ZIP_DICT_MAX_* constants */ /* for XTRADB_RSEG table */ #include "trx0trx.h" /* for TRX_QUE_STATE_STR_MAX_LEN */ @@ -629,331 +629,3 @@ UNIV_INTERN struct st_mysql_plugin i_s_xtradb_rseg = STRUCT_FLD(version_info, INNODB_VERSION_STR), STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_STABLE), }; - - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/************************************************************************/ -enum zip_dict_field_type -{ - zip_dict_field_id, - zip_dict_field_name, - zip_dict_field_zip_dict -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_fields_info[] = -{ - { STRUCT_FLD(field_name, "id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "name"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_NAME_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "zip_dict"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_DATA_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_BLOB), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT with information -collected by scanning SYS_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict( - THD* thd, /*!< in: thread */ - ulint id, /*!< in: dict ID */ - const char* name, /*!< in: dict name */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_field_id], id)); - OK(field_store_string(fields[zip_dict_field_name], name)); - OK(field_store_blob(fields[zip_dict_field_zip_dict], data, - data_len)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -Loop through each record in SYS_ZIP_DICT, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT); - ulint zip_size = dict_table_zip_size(pcur.btr_cur.index->table); - - while (rec) { - const char* err_msg; - ulint id; - const char* name; - const char* data; - ulint data_len; - - /* Extract necessary information from a SYS_ZIP_DICT row */ - err_msg = dict_process_sys_zip_dict( - heap, zip_size, rec, &id, &name, &data, &data_len); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict( - thd, id, name, data, data_len, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_init"); - - ST_SCHEMA_TABLE* schema = static_cast<ST_SCHEMA_TABLE*>(p); - - schema->fields_info = xtradb_sys_zip_dict_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compression dictionaries information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; - -enum zip_dict_cols_field_type -{ - zip_dict_cols_field_table_id, - zip_dict_cols_field_column_pos, - zip_dict_cols_field_dict_id -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_cols_fields_info[] = -{ - { STRUCT_FLD(field_name, "table_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "column_pos"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "dict_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS with information -collected by scanning SYS_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict_cols( - THD* thd, /*!< in: thread */ - ulint table_id, /*!< in: table ID */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict ID */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict_cols"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_cols_field_table_id], - table_id)); - OK(field_store_ulint(fields[zip_dict_cols_field_column_pos], - column_pos)); - OK(field_store_ulint(fields[zip_dict_cols_field_dict_id], - dict_id)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -Loop through each record in SYS_ZIP_DICT_COLS, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_cols_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_cols_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT_COLS); - - while (rec) { - const char* err_msg; - ulint table_id; - ulint column_pos; - ulint dict_id; - - /* Extract necessary information from a SYS_ZIP_DICT_COLS - row */ - err_msg = dict_process_sys_zip_dict_cols( - heap, rec, &table_id, &column_pos, &dict_id); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict_cols( - thd, table_id, column_pos, dict_id, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_cols_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_cols_init"); - - ST_SCHEMA_TABLE* schema = static_cast<ST_SCHEMA_TABLE*>(p); - - schema->fields_info = xtradb_sys_zip_dict_cols_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_cols_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict_cols = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT_COLS"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compressed columns information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_cols_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; -#endif diff --git a/storage/xtradb/handler/xtradb_i_s.h b/storage/xtradb/handler/xtradb_i_s.h index 905d84587af..994bc11c1b8 100644 --- a/storage/xtradb/handler/xtradb_i_s.h +++ b/storage/xtradb/handler/xtradb_i_s.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -22,7 +23,5 @@ this program; if not, write to the Free Software Foundation, Inc., extern struct st_mysql_plugin i_s_xtradb_read_view; extern struct st_mysql_plugin i_s_xtradb_internal_hash_tables; extern struct st_mysql_plugin i_s_xtradb_rseg; -extern struct st_mysql_plugin i_s_xtradb_zip_dict; -extern struct st_mysql_plugin i_s_xtradb_zip_dict_cols; #endif /* XTRADB_I_S_H */ diff --git a/storage/xtradb/include/btr0cur.h b/storage/xtradb/include/btr0cur.h index f14288d0f89..19ab7d8bc3a 100644 --- a/storage/xtradb/include/btr0cur.h +++ b/storage/xtradb/include/btr0cur.h @@ -294,11 +294,7 @@ btr_cur_update_alloc_zip_func( false=update-in-place */ mtr_t* mtr, /*!< in/out: mini-transaction */ trx_t* trx) /*!< in: NULL or transaction */ -#ifdef UNIV_DEBUG - MY_ATTRIBUTE((nonnull (1, 2, 3, 4, 7), warn_unused_result)); -#else - MY_ATTRIBUTE((nonnull (1, 2, 3, 6), warn_unused_result)); -#endif + MY_ATTRIBUTE((warn_unused_result)); #ifdef UNIV_DEBUG # define btr_cur_update_alloc_zip(page_zip,cursor,index,offsets,len,cr,mtr,trx) \ @@ -428,7 +424,7 @@ btr_cur_del_mark_set_clust_rec( const ulint* offsets,/*!< in: rec_get_offsets(rec) */ que_thr_t* thr, /*!< in: query thread */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /***********************************************************//** Sets a secondary index record delete mark to TRUE or FALSE. @return DB_SUCCESS, DB_LOCK_WAIT, or error number */ @@ -441,7 +437,7 @@ btr_cur_del_mark_set_sec_rec( ibool val, /*!< in: value to set */ que_thr_t* thr, /*!< in: query thread */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /*************************************************************//** Tries to compress a page of the tree if it seems useful. It is assumed that mtr holds an x-latch on the tree and on the cursor page. To avoid @@ -609,8 +605,7 @@ btr_cur_disown_inherited_fields( dict_index_t* index, /*!< in: index of the page */ const ulint* offsets,/*!< in: array returned by rec_get_offsets() */ const upd_t* update, /*!< in: update vector */ - mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull(2,3,4,5,6))); + mtr_t* mtr); /*!< in/out: mini-transaction */ /** Operation code for btr_store_big_rec_extern_fields(). */ enum blob_op { @@ -655,7 +650,7 @@ btr_store_big_rec_extern_fields( mtr_t* btr_mtr, /*!< in: mtr containing the latches to the clustered index */ enum blob_op op) /*! in: operation code */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /*******************************************************************//** Frees the space in an externally stored field to the file space @@ -751,8 +746,7 @@ btr_push_update_extern_fields( /*==========================*/ dtuple_t* tuple, /*!< in/out: data tuple */ const upd_t* update, /*!< in: update vector */ - mem_heap_t* heap) /*!< in: memory heap */ - MY_ATTRIBUTE((nonnull)); + mem_heap_t* heap); /*!< in: memory heap */ /***********************************************************//** Sets a secondary index record's delete mark to the given value. This function is only used by the insert buffer merge mechanism. */ diff --git a/storage/xtradb/include/btr0sea.h b/storage/xtradb/include/btr0sea.h index 8f438bf640e..bfe2c43defb 100644 --- a/storage/xtradb/include/btr0sea.h +++ b/storage/xtradb/include/btr0sea.h @@ -200,7 +200,7 @@ hash_table_t* btr_search_get_hash_table( /*======================*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((pure,warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Returns the adaptive hash index latch for a given index key. @@ -210,7 +210,7 @@ prio_rw_lock_t* btr_search_get_latch( /*=================*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((pure,warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Returns the AHI partition number corresponding to a given index ID. */ @@ -227,8 +227,7 @@ UNIV_INLINE void btr_search_index_init( /*===============*/ - dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((nonnull)); + dict_index_t* index); /*!< in: index */ /********************************************************************//** Latches all adaptive hash index latches in exclusive mode. */ diff --git a/storage/xtradb/include/btr0sea.ic b/storage/xtradb/include/btr0sea.ic index 3cbcff75f31..e963d8a8449 100644 --- a/storage/xtradb/include/btr0sea.ic +++ b/storage/xtradb/include/btr0sea.ic @@ -90,7 +90,6 @@ btr_search_get_hash_table( /*======================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->search_table); return(index->search_table); @@ -105,7 +104,6 @@ btr_search_get_latch( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->search_latch >= btr_search_latch_arr && index->search_latch < btr_search_latch_arr + btr_search_index_num); @@ -132,8 +130,6 @@ btr_search_index_init( /*===============*/ dict_index_t* index) /*!< in: index */ { - ut_ad(index); - index->search_latch = &btr_search_latch_arr[btr_search_get_key(index->id)]; index->search_table = diff --git a/storage/xtradb/include/buf0buddy.ic b/storage/xtradb/include/buf0buddy.ic index 9bc8e9e8762..a5fb510dd19 100644 --- a/storage/xtradb/include/buf0buddy.ic +++ b/storage/xtradb/include/buf0buddy.ic @@ -50,7 +50,7 @@ buf_buddy_alloc_low( allocated from the LRU list and buf_pool->LRU_list_mutex was temporarily released */ - MY_ATTRIBUTE((malloc, nonnull)); + MY_ATTRIBUTE((malloc)); /**********************************************************************//** Deallocate a block. */ diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 8110fbc4808..18c1cb4ff9d 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -242,8 +243,7 @@ buf_relocate( buf_page_t* bpage, /*!< in/out: control block being relocated; buf_page_get_state(bpage) must be BUF_BLOCK_ZIP_DIRTY or BUF_BLOCK_ZIP_PAGE */ - buf_page_t* dpage) /*!< in/out: destination control block */ - MY_ATTRIBUTE((nonnull)); + buf_page_t* dpage); /*!< in/out: destination control block */ /*********************************************************************//** Gets the current size of buffer buf_pool in bytes. @return size in bytes */ @@ -737,7 +737,7 @@ buf_page_print( ulint flags) /*!< in: 0 or BUF_PAGE_PRINT_NO_CRASH or BUF_PAGE_PRINT_NO_FULL */ - UNIV_COLD MY_ATTRIBUTE((nonnull)); + UNIV_COLD; /********************************************************************//** Decompress a block. @return TRUE if successful */ @@ -1949,7 +1949,10 @@ struct buf_pool_t{ os_event_t no_flush[BUF_FLUSH_N_TYPES]; /*!< this is in the set state when there is no flush batch - of the given type running */ + of the given type running; + os_event_set() and os_event_reset() + are protected by + buf_pool_t::flush_state_mutex */ ib_rbt_t* flush_rbt; /*!< a red-black tree is used exclusively during recovery to speed up insertions in the diff --git a/storage/xtradb/include/buf0dblwr.h b/storage/xtradb/include/buf0dblwr.h index a62a6400d97..5582778825c 100644 --- a/storage/xtradb/include/buf0dblwr.h +++ b/storage/xtradb/include/buf0dblwr.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -134,11 +135,13 @@ struct buf_dblwr_t{ ulint b_reserved;/*!< number of slots currently reserved for batch flush. */ os_event_t b_event;/*!< event where threads wait for a - batch flush to end. */ + batch flush to end; + os_event_set() and os_event_reset() + are protected by buf_dblwr_t::mutex */ ulint s_reserved;/*!< number of slots currently reserved for single page flushes. */ os_event_t s_event;/*!< event where threads wait for a - single page flush slot. */ + single page flush slot. Protected by mutex. */ bool* in_use; /*!< flag used to indicate if a slot is in use. Only used for single page flushes. */ diff --git a/storage/xtradb/include/data0type.h b/storage/xtradb/include/data0type.h index f269c266efb..df6b6a41c11 100644 --- a/storage/xtradb/include/data0type.h +++ b/storage/xtradb/include/data0type.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -170,9 +171,6 @@ be less than 256 */ type when the column is true VARCHAR where MySQL uses 2 bytes to store the data len; for shorter VARCHARs MySQL uses only 1 byte */ -#define DATA_COMPRESSED 16384 /* this is ORed to the precise data - type when the column has COLUMN_FORMAT = - COMPRESSED attribute*/ /*-------------------------------------------*/ /* This many bytes we need to store the type information affecting the @@ -503,17 +501,6 @@ dtype_print( /*========*/ const dtype_t* type); /*!< in: type */ -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype); /*!< in: precise type */ - /* Structure for an SQL data type. If you add fields to this structure, be sure to initialize them everywhere. This structure is initialized in the following functions: diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index 29dc480a19c..555852474aa 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -710,18 +711,3 @@ dtype_get_sql_null_size( 0, 0)); #endif /* !UNIV_HOTBACKUP */ } - -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype) /*!< in: precise type */ -{ - return (prtype & DATA_COMPRESSED) != 0 ? - ZIP_COLUMN_HEADER_LENGTH : 0; -} diff --git a/storage/xtradb/include/dict0boot.h b/storage/xtradb/include/dict0boot.h index d5bee886cbf..4fd9b0b7f98 100644 --- a/storage/xtradb/include/dict0boot.h +++ b/storage/xtradb/include/dict0boot.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -324,38 +325,6 @@ enum dict_fld_sys_datafiles_enum { DICT_FLD__SYS_DATAFILES__PATH = 3, DICT_NUM_FIELDS__SYS_DATAFILES = 4 }; -/* The columns in SYS_DICT */ -enum dict_col_sys_zip_dict_enum { - DICT_COL__SYS_ZIP_DICT__ID = 0, - DICT_COL__SYS_ZIP_DICT__NAME = 1, - DICT_COL__SYS_ZIP_DICT__DATA = 2, - DICT_NUM_COLS__SYS_ZIP_DICT = 3 -}; -/* The field numbers in the SYS_DICT clustered index */ -enum dict_fld_sys_zip_dict_enum { - DICT_FLD__SYS_ZIP_DICT__ID = 0, - DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID = 1, - DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR = 2, - DICT_FLD__SYS_ZIP_DICT__NAME = 3, - DICT_FLD__SYS_ZIP_DICT__DATA = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT = 5 -}; -/* The columns in SYS_DICT_COLS */ -enum dict_col_sys_zip_dict_cols_enum { - DICT_COL__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_COL__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_COL__SYS_ZIP_DICT_COLS__DICT_ID = 2, - DICT_NUM_COLS__SYS_ZIP_DICT_COLS = 3 -}; -/* The field numbers in the SYS_DICT_COLS clustered index */ -enum dict_fld_sys_zip_dict_cols_enum { - DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID = 2, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR = 3, - DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS = 5 -}; /* A number of the columns above occur in multiple tables. These are the length of thos fields. */ diff --git a/storage/xtradb/include/dict0boot.ic b/storage/xtradb/include/dict0boot.ic index 2b156a4f672..42e91ee930e 100644 --- a/storage/xtradb/include/dict0boot.ic +++ b/storage/xtradb/include/dict0boot.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -92,5 +93,3 @@ dict_is_sys_table( { return(id < DICT_HDR_FIRST_ID); } - - diff --git a/storage/xtradb/include/dict0crea.h b/storage/xtradb/include/dict0crea.h index 686f56ad58c..e5977cbabf6 100644 --- a/storage/xtradb/include/dict0crea.h +++ b/storage/xtradb/include/dict0crea.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -164,18 +165,6 @@ dberr_t dict_create_or_check_sys_tablespace(void); /*=====================================*/ -#define ZIP_DICT_MAX_NAME_LENGTH 64 -/* Max window size (2^15) minus 262 */ -#define ZIP_DICT_MAX_DATA_LENGTH 32506 - -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void); - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -192,83 +181,6 @@ dict_create_add_tablespace_to_dictionary( bool commit); /*!< in: if true then commit the transaction */ -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length */ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx); /*!< in/out: transaction */ - /********************************************************************//** Add a foreign key definition to the data dictionary tables. @return error code or DB_SUCCESS */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 1b6110dd010..a2481fbad6f 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -747,7 +747,7 @@ ulint dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((nonnull, pure, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Check whether the index is unique. @return nonzero for unique index, zero for other indexes */ @@ -756,7 +756,7 @@ ulint dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((nonnull, pure, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Check whether the index is the insert buffer tree. @return nonzero for insert buffer, zero for other indexes */ @@ -765,7 +765,7 @@ ulint dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((nonnull, pure, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Check whether the index is a secondary index or the insert buffer tree. @return nonzero for insert buffer, zero for other indexes */ @@ -774,7 +774,7 @@ ulint dict_index_is_sec_or_ibuf( /*======================*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((nonnull, pure, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /************************************************************************ Gets the all the FTS indexes for the table. NOTE: must not be called for @@ -796,7 +796,7 @@ ulint dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull, pure, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Gets the number of system columns in a table in the dictionary cache. @return number of system (e.g., ROW_ID) columns of a table */ @@ -815,7 +815,7 @@ ulint dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull, pure, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Gets the approximately estimated number of rows in the table. @return estimated number of rows */ @@ -1745,7 +1745,7 @@ ulint dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); #endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** @@ -1758,7 +1758,7 @@ dict_set_corrupted( dict_index_t* index, /*!< in/out: index */ trx_t* trx, /*!< in/out: transaction */ const char* ctx) /*!< in: context */ - UNIV_COLD MY_ATTRIBUTE((nonnull)); + UNIV_COLD; /**********************************************************************//** Flags an index corrupted in the data dictionary cache only. This @@ -1769,8 +1769,7 @@ void dict_set_corrupted_index_cache_only( /*================================*/ dict_index_t* index, /*!< in/out: index */ - dict_table_t* table) /*!< in/out: table */ - MY_ATTRIBUTE((nonnull)); + dict_table_t* table); /*!< in/out: table */ /**********************************************************************//** Flags a table with specified space_id corrupted in the table dictionary @@ -1871,52 +1870,6 @@ dict_table_set_corrupt_by_space( ulint space_id, ibool need_mutex); -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len); /*!< in: zip_dict data length */ - -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id); /*!< out: zip_dict id */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len); /*!< out: dictionary data length*/ - -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len); /*!< in: zip_dict name length*/ - #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index 58a9ef4d65d..efc2c3f0e68 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -266,7 +266,6 @@ dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(index->type & DICT_CLUSTERED); @@ -280,7 +279,6 @@ dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(index->type & DICT_UNIQUE); @@ -295,7 +293,6 @@ dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(index->type & DICT_IBUF); @@ -327,7 +324,6 @@ dict_index_is_sec_or_ibuf( { ulint type; - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); type = index->type; @@ -345,7 +341,6 @@ dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols - DATA_N_SYS_COLS); @@ -377,7 +372,6 @@ dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols); @@ -1373,7 +1367,6 @@ dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return((index->type & DICT_CORRUPT) diff --git a/storage/xtradb/include/dict0load.h b/storage/xtradb/include/dict0load.h index 85e3e565637..1a720de5bb6 100644 --- a/storage/xtradb/include/dict0load.h +++ b/storage/xtradb/include/dict0load.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,8 +45,6 @@ enum dict_system_id_t { SYS_FOREIGN_COLS, SYS_TABLESPACES, SYS_DATAFILES, - SYS_ZIP_DICT, - SYS_ZIP_DICT_COLS, /* This must be last item. Defines the number of system tables. */ SYS_NUM_SYSTEM_TABLES @@ -389,32 +388,6 @@ dict_process_sys_datafiles( ulint* space, /*!< out: pace id */ const char** path); /*!< out: datafile path */ -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len); /*!< out: dict data length */ - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id); /*!< out: dict id */ - /********************************************************************//** Get the filepath for a spaceid from SYS_DATAFILES. This function provides a temporary heap which is used for the table lookup, but not for the path. diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 942f8ba1b06..b4307f7d168 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -138,7 +138,7 @@ allows InnoDB to update_create_info() accordingly. */ + DICT_TF_WIDTH_DATA_DIR) /** A mask of all the known/used bits in table flags */ -#define DICT_TF_BIT_MASK (~(~0 << DICT_TF_BITS)) +#define DICT_TF_BIT_MASK (~(~0U << DICT_TF_BITS)) /** Zero relative shift position of the COMPACT field */ #define DICT_TF_POS_COMPACT 0 diff --git a/storage/xtradb/include/dict0stats_bg.h b/storage/xtradb/include/dict0stats_bg.h index 82cd2b468b9..1973380d197 100644 --- a/storage/xtradb/include/dict0stats_bg.h +++ b/storage/xtradb/include/dict0stats_bg.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,7 +33,8 @@ Created Apr 26, 2012 Vasil Dimov #include "os0sync.h" /* os_event_t */ #include "os0thread.h" /* DECLARE_THREAD */ -/** Event to wake up the stats thread */ +/** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() +or shutdown. Not protected by any mutex. */ extern os_event_t dict_stats_event; /*****************************************************************//** diff --git a/storage/xtradb/include/dyn0dyn.h b/storage/xtradb/include/dyn0dyn.h index 1bd10b6bf58..20963a1472b 100644 --- a/storage/xtradb/include/dyn0dyn.h +++ b/storage/xtradb/include/dyn0dyn.h @@ -46,9 +46,8 @@ UNIV_INLINE dyn_array_t* dyn_array_create( /*=============*/ - dyn_array_t* arr) /*!< in/out memory buffer of + dyn_array_t* arr); /*!< in/out memory buffer of size sizeof(dyn_array_t) */ - MY_ATTRIBUTE((nonnull)); /************************************************************//** Frees a dynamic array. */ UNIV_INLINE @@ -69,7 +68,7 @@ dyn_array_open( dyn_array_t* arr, /*!< in: dynamic array */ ulint size) /*!< in: size in bytes of the buffer; MUST be smaller than DYN_ARRAY_DATA_SIZE! */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Closes the buffer returned by dyn_array_open. */ UNIV_INLINE @@ -77,8 +76,7 @@ void dyn_array_close( /*============*/ dyn_array_t* arr, /*!< in: dynamic array */ - const byte* ptr) /*!< in: end of used space */ - MY_ATTRIBUTE((nonnull)); + const byte* ptr); /*!< in: end of used space */ /*********************************************************************//** Makes room on top of a dyn array and returns a pointer to the added element. The caller must copy the element to @@ -90,7 +88,7 @@ dyn_array_push( /*===========*/ dyn_array_t* arr, /*!< in/out: dynamic array */ ulint size) /*!< in: size in bytes of the element */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /************************************************************//** Returns pointer to an element in dyn array. @return pointer to element */ @@ -101,7 +99,7 @@ dyn_array_get_element( const dyn_array_t* arr, /*!< in: dyn array */ ulint pos) /*!< in: position of element in bytes from array start */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /************************************************************//** Returns the size of stored data in a dyn array. @return data size in bytes */ @@ -110,7 +108,7 @@ ulint dyn_array_get_data_size( /*====================*/ const dyn_array_t* arr) /*!< in: dyn array */ - MY_ATTRIBUTE((nonnull, warn_unused_result, pure)); + MY_ATTRIBUTE((warn_unused_result)); /************************************************************//** Gets the first block in a dyn array. @param arr dyn array @@ -144,7 +142,7 @@ ulint dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ - MY_ATTRIBUTE((nonnull, warn_unused_result, pure)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Gets pointer to the start of data in a dyn array block. @return pointer to data */ @@ -153,7 +151,7 @@ byte* dyn_block_get_data( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ - MY_ATTRIBUTE((nonnull, warn_unused_result, pure)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************//** Pushes n bytes to a dyn array. */ UNIV_INLINE diff --git a/storage/xtradb/include/dyn0dyn.ic b/storage/xtradb/include/dyn0dyn.ic index f18f2e6dff9..6e97649245e 100644 --- a/storage/xtradb/include/dyn0dyn.ic +++ b/storage/xtradb/include/dyn0dyn.ic @@ -36,7 +36,7 @@ dyn_block_t* dyn_array_add_block( /*================*/ dyn_array_t* arr) /*!< in/out: dyn array */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Gets the number of used bytes in a dyn array block. @@ -47,8 +47,6 @@ dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((block->used) & ~DYN_BLOCK_FULL_FLAG); } @@ -76,7 +74,6 @@ dyn_array_create( dyn_array_t* arr) /*!< in/out: memory buffer of size sizeof(dyn_array_t) */ { - ut_ad(arr); #if DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG # error "DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG" #endif @@ -119,7 +116,6 @@ dyn_array_push( dyn_block_t* block; ulint used; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -159,7 +155,6 @@ dyn_array_open( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -195,7 +190,6 @@ dyn_array_close( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); block = dyn_array_get_last_block(arr); @@ -222,7 +216,6 @@ dyn_array_get_element( { const dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); /* Get the first array block */ @@ -260,7 +253,6 @@ dyn_array_get_data_size( const dyn_block_t* block; ulint sum = 0; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 2581384ff22..3023b29b793 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -199,7 +200,9 @@ struct fil_node_t { ibool open; /*!< TRUE if file open */ os_file_t handle; /*!< OS handle to the file, if file open */ os_event_t sync_event;/*!< Condition event to group and - serialize calls to fsync */ + serialize calls to fsync; + os_event_set() and os_event_reset() + are protected by fil_system_t::mutex */ ibool is_raw_disk;/*!< TRUE if the 'file' is actually a raw device or a raw disk partition */ ulint size; /*!< size of the file in database pages, 0 if diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 099cb8edc14..0097537e4d6 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -61,7 +61,7 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR) /** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 diff --git a/storage/xtradb/include/fsp0types.h b/storage/xtradb/include/fsp0types.h index 94fd908ab0c..9734f4c8abc 100644 --- a/storage/xtradb/include/fsp0types.h +++ b/storage/xtradb/include/fsp0types.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software diff --git a/storage/xtradb/include/fts0fts.h b/storage/xtradb/include/fts0fts.h index 3e2f359bbeb..7aa7055640c 100644 --- a/storage/xtradb/include/fts0fts.h +++ b/storage/xtradb/include/fts0fts.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -366,8 +367,8 @@ extern ulong fts_min_token_size; need a sync to free some memory */ extern bool fts_need_sync; -/** Maximum possible Fulltext word length */ -#define FTS_MAX_WORD_LEN HA_FT_MAXBYTELEN +/** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */ +#define FTS_MAX_WORD_LEN (HA_FT_MAXCHARLEN * 4) /** Maximum possible Fulltext word length (in characters) */ #define FTS_MAX_WORD_LEN_IN_CHAR HA_FT_MAXCHARLEN diff --git a/storage/xtradb/include/fts0types.h b/storage/xtradb/include/fts0types.h index e495fe72a60..0dad75d8f1b 100644 --- a/storage/xtradb/include/fts0types.h +++ b/storage/xtradb/include/fts0types.h @@ -126,7 +126,9 @@ struct fts_sync_t { bool in_progress; /*!< flag whether sync is in progress.*/ bool unlock_cache; /*!< flag whether unlock cache when write fts node */ - os_event_t event; /*!< sync finish event */ + os_event_t event; /*!< sync finish event; + only os_event_set() and os_event_wait() + are used */ }; /** The cache for the FTS system. It is a memory-based inverted index diff --git a/storage/xtradb/include/lock0lock.h b/storage/xtradb/include/lock0lock.h index ee89f3512fb..20c7ac888a4 100644 --- a/storage/xtradb/include/lock0lock.h +++ b/storage/xtradb/include/lock0lock.h @@ -931,7 +931,12 @@ struct lock_sys_t{ srv_slot_t* waiting_threads; /*!< Array of user threads suspended while waiting for locks within InnoDB, protected - by the lock_sys->wait_mutex */ + by the lock_sys->wait_mutex; + os_event_set() and + os_event_reset() on + waiting_threads[]->event + are protected by + trx_t::mutex */ srv_slot_t* last_slot; /*!< highest slot ever used in the waiting_threads array, protected by @@ -944,10 +949,11 @@ struct lock_sys_t{ ulint n_lock_max_wait_time; /*!< Max wait time */ - os_event_t timeout_event; /*!< Set to the event that is - created in the lock wait monitor - thread. A value of 0 means the - thread is not active */ + os_event_t timeout_event; /*!< An event waited for by + lock_wait_timeout_thread. + Not protected by a mutex, + but the waits are timed. + Signaled on shutdown only. */ bool timeout_thread_active; /*!< True if the timeout thread is running */ diff --git a/storage/xtradb/include/log0log.h b/storage/xtradb/include/log0log.h index f716e534b01..259c4301de9 100644 --- a/storage/xtradb/include/log0log.h +++ b/storage/xtradb/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -924,10 +925,8 @@ struct log_t{ be 'flush_or_write'! */ os_event_t no_flush_event; /*!< this event is in the reset state when a flush or a write is running; - a thread should wait for this without - owning the log mutex, but NOTE that - to set or reset this event, the - thread MUST own the log mutex! */ + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ ibool one_flushed; /*!< during a flush, this is first FALSE and becomes TRUE when one log group has been @@ -936,11 +935,9 @@ struct log_t{ flush or write has not yet completed for any log group; e.g., this means that a transaction has been committed - when this is set; a thread should wait - for this without owning the log mutex, - but NOTE that to set or reset this - event, the thread MUST own the log - mutex! */ + when this is set; + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ ulint n_log_ios; /*!< number of log i/os initiated thus far */ ulint n_log_ios_old; /*!< number of log i/o's at the @@ -1026,9 +1023,9 @@ struct log_t{ byte* archive_buf_ptr;/*!< unaligned archived_buf */ byte* archive_buf; /*!< log segment is written to the archive from this buffer */ - os_event_t archiving_on; /*!< if archiving has been stopped, - a thread can wait for this event to - become signaled */ + os_event_t archiving_on; /*!< if archiving has been stopped; + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ /* @} */ #endif /* UNIV_LOG_ARCHIVE */ lsn_t tracked_lsn; /*!< log tracking has advanced to this diff --git a/storage/xtradb/include/log0online.h b/storage/xtradb/include/log0online.h index 5706f3af4b0..722336dd6b4 100644 --- a/storage/xtradb/include/log0online.h +++ b/storage/xtradb/include/log0online.h @@ -38,19 +38,25 @@ log_online_bitmap_file_range_t; /** An iterator over changed page info */ typedef struct log_bitmap_iterator_struct log_bitmap_iterator_t; -/*********************************************************************//** -Initializes the online log following subsytem. */ +/** Initialize the constant part of the log tracking subsystem */ +UNIV_INTERN +void +log_online_init(void); + +/** Initialize the dynamic part of the log tracking subsystem */ UNIV_INTERN void log_online_read_init(void); -/*=======================*/ -/*********************************************************************//** -Shuts down the online log following subsystem. */ +/** Shut down the dynamic part of the log tracking subsystem */ UNIV_INTERN void log_online_read_shutdown(void); -/*===========================*/ + +/** Shut down the constant part of the log tracking subsystem */ +UNIV_INTERN +void +log_online_shutdown(void); /*********************************************************************//** Reads and parses the redo log up to last checkpoint LSN to build the changed @@ -147,6 +153,8 @@ struct log_online_bitmap_file_range_struct { /** Struct for an iterator through all bits of changed pages bitmap blocks */ struct log_bitmap_iterator_struct { + lsn_t max_lsn; /*!< End LSN of the + range */ ibool failed; /*!< Has the iteration stopped prematurely */ log_online_bitmap_file_range_t in_files; /*!< The bitmap files diff --git a/storage/xtradb/include/log0recv.h b/storage/xtradb/include/log0recv.h index 6955491bac8..9e6001caf71 100644 --- a/storage/xtradb/include/log0recv.h +++ b/storage/xtradb/include/log0recv.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -299,20 +300,12 @@ void recv_sys_var_init(void); /*===================*/ #endif /* !UNIV_HOTBACKUP */ -/*******************************************************************//** -Empties the hash table of stored log records, applying them to appropriate -pages. */ +/** Apply the hash table of stored log records to persistent data pages. +@param[in] last_batch whether the change buffer merge will be + performed as part of the operation */ UNIV_INTERN void -recv_apply_hashed_log_recs( -/*=======================*/ - ibool allow_ibuf); /*!< in: if TRUE, also ibuf operations are - allowed during the application; if FALSE, - no ibuf operations are allowed, and after - the application all file pages are flushed to - disk and invalidated in buffer pool: this - alternative means that no new log records - can be generated during the application */ +recv_apply_hashed_log_recs(bool last_batch); #ifdef UNIV_HOTBACKUP /*******************************************************************//** Applies log records in the hash table to a backup. */ @@ -438,6 +431,8 @@ struct recv_sys_t{ scan find a corrupt log block, or a corrupt log record, or there is a log parsing buffer overflow */ + /** the time when progress was last reported */ + ib_time_t progress_time; #ifdef UNIV_LOG_ARCHIVE log_group_t* archive_group; /*!< in archive recovery: the log group whose @@ -450,6 +445,20 @@ struct recv_sys_t{ addresses in the hash table */ recv_dblwr_t dblwr; + + /** Determine whether redo log recovery progress should be reported. + @param[in] time the current time + @return whether progress should be reported + (the last report was at least 15 seconds ago) */ + bool report(ib_time_t time) + { + if (time - progress_time < 15) { + return false; + } + + progress_time = time; + return true; + } }; /** The recovery system */ diff --git a/storage/xtradb/include/mach0data.h b/storage/xtradb/include/mach0data.h index 9859def0adc..2e16634a6c2 100644 --- a/storage/xtradb/include/mach0data.h +++ b/storage/xtradb/include/mach0data.h @@ -53,7 +53,7 @@ ulint mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*******************************************************//** The following function is used to store data in two consecutive bytes. We store the most significant byte to the lower address. */ @@ -114,7 +114,7 @@ ulint mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*******************************************************//** The following function is used to store data in four consecutive bytes. We store the most significant byte to the lowest address. */ @@ -133,7 +133,7 @@ ulint mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************//** Writes a ulint in a compressed form (1..5 bytes). @return stored size in bytes */ @@ -160,7 +160,7 @@ ulint mach_read_compressed( /*=================*/ const byte* b) /*!< in: pointer to memory from where to read */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*******************************************************//** The following function is used to store data in 6 consecutive bytes. We store the most significant byte to the lowest address. */ @@ -179,7 +179,7 @@ ib_uint64_t mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*******************************************************//** The following function is used to store data in 7 consecutive bytes. We store the most significant byte to the lowest address. */ @@ -198,7 +198,7 @@ ib_uint64_t mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*******************************************************//** The following function is used to store data in 8 consecutive bytes. We store the most significant byte to the lowest address. */ @@ -243,7 +243,7 @@ ib_uint64_t mach_ull_read_compressed( /*=====================*/ const byte* b) /*!< in: pointer to memory from where to read */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************//** Writes a 64-bit integer in a compressed form (1..11 bytes). @return size in bytes */ @@ -270,7 +270,7 @@ ib_uint64_t mach_ull_read_much_compressed( /*==========================*/ const byte* b) /*!< in: pointer to memory from where to read */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************//** Reads a ulint in a compressed form if the log record fully contains it. @return pointer to end of the stored field, NULL if not complete */ diff --git a/storage/xtradb/include/mach0data.ic b/storage/xtradb/include/mach0data.ic index 27b9f62b552..4b9275cc12c 100644 --- a/storage/xtradb/include/mach0data.ic +++ b/storage/xtradb/include/mach0data.ic @@ -52,7 +52,6 @@ mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ { - ut_ad(b); return((ulint)(b[0])); } @@ -132,7 +131,6 @@ mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 16) | ((ulint)(b[1]) << 8) | (ulint)(b[2]) @@ -182,7 +180,6 @@ mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 24) | ((ulint)(b[1]) << 16) | ((ulint)(b[2]) << 8) @@ -261,8 +258,6 @@ mach_read_compressed( { ulint flag; - ut_ad(b); - flag = mach_read_from_1(b); if (flag < 0x80UL) { @@ -339,8 +334,6 @@ mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_3(b), mach_read_from_4(b + 3))); } @@ -370,8 +363,6 @@ mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_2(b), mach_read_from_4(b + 2))); } @@ -419,8 +410,6 @@ mach_ull_read_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - n = (ib_uint64_t) mach_read_compressed(b); size = mach_get_compressed_size((ulint) n); @@ -486,8 +475,6 @@ mach_ull_read_much_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - if (*b != (byte)0xFF) { n = 0; size = 0; diff --git a/storage/xtradb/include/mtr0mtr.h b/storage/xtradb/include/mtr0mtr.h index c00472117dc..f939918a90d 100644 --- a/storage/xtradb/include/mtr0mtr.h +++ b/storage/xtradb/include/mtr0mtr.h @@ -227,8 +227,7 @@ UNIV_INTERN void mtr_commit( /*=======*/ - mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull)); + mtr_t* mtr); /*!< in/out: mini-transaction */ /**********************************************************//** Sets and returns a savepoint in mtr. @return savepoint */ @@ -334,7 +333,7 @@ mtr_memo_contains( mtr_t* mtr, /*!< in: mtr */ const void* object, /*!< in: object to search */ ulint type) /*!< in: type of object */ - MY_ATTRIBUTE((warn_unused_result, nonnull)); + MY_ATTRIBUTE((warn_unused_result)); /**********************************************************//** Checks if memo contains the given page. diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index d2d2be7662f..b8f30977fbe 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -51,16 +52,6 @@ extern ibool os_has_said_disk_full; /** Flag: enable debug printout for asynchronous i/o */ extern ibool os_aio_print_debug; -/** Number of pending os_file_pread() operations */ -extern ulint os_file_n_pending_preads; -/** Number of pending os_file_pwrite() operations */ -extern ulint os_file_n_pending_pwrites; - -/** Number of pending read operations */ -extern ulint os_n_pending_reads; -/** Number of pending write operations */ -extern ulint os_n_pending_writes; - #ifdef __WIN__ /** We define always WIN_ASYNC_IO, and check at run-time whether @@ -552,9 +543,10 @@ os_file_create_simple_no_error_handling_func( ibool* success)/*!< out: TRUE if succeed, FALSE if error */ MY_ATTRIBUTE((nonnull, warn_unused_result)); /****************************************************************//** -Tries to disable OS caching on an opened file descriptor. */ +Tries to disable OS caching on an opened file descriptor. +@return true if operation is success and false otherwise */ UNIV_INTERN -void +bool os_file_set_nocache( /*================*/ os_file_t fd, /*!< in: file descriptor to alter */ @@ -1175,6 +1167,7 @@ UNIV_INTERN void os_aio_simulated_wake_handler_threads(void); /*=======================================*/ +#ifdef _WIN32 /**********************************************************************//** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must @@ -1182,8 +1175,10 @@ call os_aio_simulated_wake_handler_threads later to ensure the threads are not left sleeping! */ UNIV_INTERN void -os_aio_simulated_put_read_threads_to_sleep(void); -/*============================================*/ +os_aio_simulated_put_read_threads_to_sleep(); +#else /* _WIN32 */ +# define os_aio_simulated_put_read_threads_to_sleep() +#endif /* _WIN32 */ #ifdef WIN_ASYNC_IO /**********************************************************************//** diff --git a/storage/xtradb/include/os0sync.h b/storage/xtradb/include/os0sync.h index f6207555f1a..7bc591b2911 100644 --- a/storage/xtradb/include/os0sync.h +++ b/storage/xtradb/include/os0sync.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -38,12 +39,12 @@ Created 9/6/1995 Heikki Tuuri #include "ut0lst.h" #include "sync0types.h" -#if defined __i386__ || defined __x86_64__ || defined _M_IX86 \ - || defined _M_X64 || defined __WIN__ - -#define IB_STRONG_MEMORY_MODEL - -#endif /* __i386__ || __x86_64__ || _M_IX86 || _M_X64 || __WIN__ */ +/** CPU cache line size */ +#ifdef __powerpc__ +#define CACHE_LINE_SIZE 128 +#else +#define CACHE_LINE_SIZE 64 +#endif #ifdef HAVE_WINDOWS_ATOMICS typedef LONG lock_word_t; /*!< On Windows, InterlockedExchange operates @@ -718,10 +719,7 @@ os_atomic_clear(volatile lock_word_t* ptr) # define HAVE_ATOMIC_BUILTINS # define HAVE_ATOMIC_BUILTINS_BYTE - -# ifndef _WIN32 -# define HAVE_ATOMIC_BUILTINS_64 -# endif +# define HAVE_ATOMIC_BUILTINS_64 /**********************************************************//** Atomic compare and exchange of signed integers (both 32 and 64 bit). @@ -943,6 +941,58 @@ for synchronization */ "Memory barrier is not used" #endif + +/** Simple counter aligned to CACHE_LINE_SIZE +@tparam Type the integer type of the counter +@tparam atomic whether to use atomic memory access */ +template <typename Type = ulint, bool atomic = false> +struct MY_ALIGNED(CACHE_LINE_SIZE) simple_counter +{ + /** Increment the counter */ + Type inc() { return add(1); } + /** Decrement the counter */ + Type dec() { return sub(1); } + + /** Add to the counter + @param[in] i amount to be added + @return the value of the counter after adding */ + Type add(Type i) + { + compile_time_assert(!atomic || sizeof(Type) == sizeof(ulint)); + if (atomic) { + /* GCC would perform a type check in this code + also in case the template is instantiated with + simple_counter<Type=not_ulint, atomic=false>. + On Solaris, os_atomic_increment_ulint() maps + to atomic_add_long_nv(), which expects the + parameter to be correctly typed. */ + return os_atomic_increment_ulint( + reinterpret_cast<ulint*>(&m_counter), i); + } else { + return m_counter += i; + } + } + /** Subtract from the counter + @param[in] i amount to be subtracted + @return the value of the counter after adding */ + Type sub(Type i) + { + compile_time_assert(!atomic || sizeof(Type) == sizeof(ulint)); + if (atomic) { + return os_atomic_decrement_ulint(&m_counter, i); + } else { + return m_counter -= i; + } + } + + /** @return the value of the counter (non-atomic access)! */ + operator Type() const { return m_counter; } + +private: + /** The counter */ + Type m_counter; +}; + #ifndef UNIV_NONINL #include "os0sync.ic" #endif diff --git a/storage/xtradb/include/os0thread.h b/storage/xtradb/include/os0thread.h index 671b9b7dc3f..7865358b0f7 100644 --- a/storage/xtradb/include/os0thread.h +++ b/storage/xtradb/include/os0thread.h @@ -131,11 +131,9 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ -/** -Waits until the specified thread completes and joins it. Its return value is -ignored. - -@param thread thread to join */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ UNIV_INTERN void os_thread_join( diff --git a/storage/xtradb/include/page0page.h b/storage/xtradb/include/page0page.h index cb43c937757..eefa0fa4c5b 100644 --- a/storage/xtradb/include/page0page.h +++ b/storage/xtradb/include/page0page.h @@ -235,8 +235,7 @@ ulint page_header_get_offs( /*=================*/ const page_t* page, /*!< in: page */ - ulint field) /*!< in: PAGE_FREE, ... */ - MY_ATTRIBUTE((nonnull, pure)); + ulint field); /*!< in: PAGE_FREE, ... */ /*************************************************************//** Returns the pointer stored in the given header field, or NULL. */ @@ -528,7 +527,7 @@ bool page_is_leaf( /*=========*/ const page_t* page) /*!< in: page */ - MY_ATTRIBUTE((nonnull, pure)); + MY_ATTRIBUTE((warn_unused_result)); /************************************************************//** Determine whether the page is empty. @return true if the page is empty (PAGE_N_RECS = 0) */ @@ -849,8 +848,7 @@ page_copy_rec_list_end( buf_block_t* block, /*!< in: index page containing rec */ rec_t* rec, /*!< in: record on page */ dict_index_t* index, /*!< in: record descriptor */ - mtr_t* mtr) /*!< in: mtr */ - MY_ATTRIBUTE((nonnull)); + mtr_t* mtr); /*!< in: mtr */ /*************************************************************//** Copies records from page to new_page, up to the given record, NOT including that record. Infimum and supremum records are not copied. @@ -871,8 +869,7 @@ page_copy_rec_list_start( buf_block_t* block, /*!< in: index page containing rec */ rec_t* rec, /*!< in: record on page */ dict_index_t* index, /*!< in: record descriptor */ - mtr_t* mtr) /*!< in: mtr */ - MY_ATTRIBUTE((nonnull)); + mtr_t* mtr); /*!< in: mtr */ /*************************************************************//** Deletes records from a page from a given record onward, including that record. The infimum and supremum records are not deleted. */ @@ -921,8 +918,7 @@ page_move_rec_list_end( buf_block_t* block, /*!< in: index page from where to move */ rec_t* split_rec, /*!< in: first record to move */ dict_index_t* index, /*!< in: record descriptor */ - mtr_t* mtr) /*!< in: mtr */ - MY_ATTRIBUTE((nonnull(1, 2, 4, 5))); + mtr_t* mtr); /*!< in: mtr */ /*************************************************************//** Moves record list start to another page. Moved records do not include split_rec. @@ -952,8 +948,7 @@ page_dir_split_slot( page_t* page, /*!< in: index page */ page_zip_des_t* page_zip,/*!< in/out: compressed page whose uncompressed part will be written, or NULL */ - ulint slot_no)/*!< in: the directory slot */ - MY_ATTRIBUTE((nonnull(1))); + ulint slot_no);/*!< in: the directory slot */ /*************************************************************//** Tries to balance the given directory slot with too few records with the upper neighbor, so that there are at least the minimum number @@ -965,8 +960,7 @@ page_dir_balance_slot( /*==================*/ page_t* page, /*!< in/out: index page */ page_zip_des_t* page_zip,/*!< in/out: compressed page, or NULL */ - ulint slot_no)/*!< in: the directory slot */ - MY_ATTRIBUTE((nonnull(1))); + ulint slot_no);/*!< in: the directory slot */ /**********************************************************//** Parses a log record of a record list end or start deletion. @return end of log record or NULL */ diff --git a/storage/xtradb/include/page0page.ic b/storage/xtradb/include/page0page.ic index 4a22a32112f..80939bbb828 100644 --- a/storage/xtradb/include/page0page.ic +++ b/storage/xtradb/include/page0page.ic @@ -154,7 +154,6 @@ page_header_get_offs( { ulint offs; - ut_ad(page); ut_ad((field == PAGE_FREE) || (field == PAGE_LAST_INSERT) || (field == PAGE_HEAP_TOP)); diff --git a/storage/xtradb/include/page0zip.h b/storage/xtradb/include/page0zip.h index 81068e7bd29..adafaa6d8b6 100644 --- a/storage/xtradb/include/page0zip.h +++ b/storage/xtradb/include/page0zip.h @@ -132,7 +132,7 @@ page_zip_compress( dict_index_t* index, /*!< in: index of the B-tree node */ ulint level, /*!< in: compression level */ mtr_t* mtr) /*!< in: mini-transaction, or NULL */ - MY_ATTRIBUTE((nonnull(1,2,3))); + MY_ATTRIBUTE((warn_unused_result)); /**********************************************************************//** Decompress a page. This function should tolerate errors on the compressed @@ -424,8 +424,7 @@ page_zip_reorganize( out: data, n_blobs, m_start, m_end, m_nonempty */ dict_index_t* index, /*!< in: index of the B-tree node */ - mtr_t* mtr) /*!< in: mini-transaction */ - MY_ATTRIBUTE((nonnull)); + mtr_t* mtr); /*!< in: mini-transaction */ #ifndef UNIV_HOTBACKUP /**********************************************************************//** Copy the records of a page byte for byte. Do not copy the page header @@ -458,7 +457,7 @@ page_zip_parse_compress( byte* end_ptr,/*!< in: buffer end */ page_t* page, /*!< out: uncompressed page */ page_zip_des_t* page_zip)/*!< out: compressed page */ - MY_ATTRIBUTE((nonnull(1,2))); + MY_ATTRIBUTE((warn_unused_result)); #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/xtradb/include/rem0rec.h b/storage/xtradb/include/rem0rec.h index 971709fd2c8..cddc7ef63f9 100644 --- a/storage/xtradb/include/rem0rec.h +++ b/storage/xtradb/include/rem0rec.h @@ -747,8 +747,7 @@ rec_copy( /*=====*/ void* buf, /*!< in: buffer */ const rec_t* rec, /*!< in: physical record */ - const ulint* offsets)/*!< in: array returned by rec_get_offsets() */ - MY_ATTRIBUTE((nonnull)); + const ulint* offsets);/*!< in: array returned by rec_get_offsets() */ #ifndef UNIV_HOTBACKUP /**********************************************************//** Determines the size of a data tuple prefix in a temporary file. diff --git a/storage/xtradb/include/row0mysql.h b/storage/xtradb/include/row0mysql.h index 70da84640e5..e1148517d99 100644 --- a/storage/xtradb/include/row0mysql.h +++ b/storage/xtradb/include/row0mysql.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +42,6 @@ struct SysIndexCallback; extern ibool row_rollback_on_timeout; -extern uint srv_compressed_columns_zip_level; -extern ulong srv_compressed_columns_threshold; - struct row_prebuilt_t; /*******************************************************************//** @@ -55,48 +53,6 @@ row_mysql_prebuilt_free_blob_heap( row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a ha_innobase:: table handle */ -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a - ha_innobase:: table handle */ - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -135,21 +91,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len); /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt); - /*<! in: use prebuilt->compress_heap only - here */ /*******************************************************************//** Reads a reference to a BLOB in the MySQL format. @return pointer to BLOB data */ @@ -160,17 +105,8 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len); /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ /**************************************************************//** Pad a column with spaces. */ UNIV_INTERN @@ -218,16 +154,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed */ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ + ulint comp); /*!< in: nonzero=compact format */ /****************************************************************//** Handles user errors and lock waits detected by the database engine. @return true if it was a lock wait and we should continue running the @@ -721,8 +648,6 @@ struct mysql_row_templ_t { ulint is_unsigned; /*!< if a column type is an integer type and this field is != 0, then it is an unsigned integer type */ - bool compressed; /*!< if column format is compressed */ - LEX_CSTRING zip_dict_data; /*!< associated compression dictionary */ }; #define MYSQL_FETCH_CACHE_SIZE 8 @@ -920,8 +845,6 @@ struct row_prebuilt_t { in fetch_cache */ mem_heap_t* blob_heap; /*!< in SELECTS BLOB fields are copied to this heap */ - mem_heap_t* compress_heap; /*!< memory heap used to compress - /decompress blob column*/ mem_heap_t* old_vers_heap; /*!< memory heap where a previous version is built in consistent read */ bool in_fts_query; /*!< Whether we are in a FTS query */ diff --git a/storage/xtradb/include/row0upd.h b/storage/xtradb/include/row0upd.h index e59ec58b63c..4312fcf7339 100644 --- a/storage/xtradb/include/row0upd.h +++ b/storage/xtradb/include/row0upd.h @@ -248,9 +248,8 @@ row_upd_index_replace_new_col_vals_index_pos( /*!< in: if TRUE, limit the replacement to ordering fields of index; note that this does not work for non-clustered indexes. */ - mem_heap_t* heap) /*!< in: memory heap for allocating and + mem_heap_t* heap); /*!< in: memory heap for allocating and copying the new values */ - MY_ATTRIBUTE((nonnull)); /***********************************************************//** Replaces the new column values stored in the update vector to the index entry given. */ @@ -311,7 +310,7 @@ row_upd_changes_ord_field_binary_func( compile time */ const row_ext_t*ext) /*!< NULL, or prefixes of the externally stored columns in the old row */ - MY_ATTRIBUTE((nonnull(1,2), warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); #ifdef UNIV_DEBUG # define row_upd_changes_ord_field_binary(index,update,thr,row,ext) \ row_upd_changes_ord_field_binary_func(index,update,thr,row,ext) diff --git a/storage/xtradb/include/srv0mon.h b/storage/xtradb/include/srv0mon.h index 2d90f47eefe..09af5d4159b 100644 --- a/storage/xtradb/include/srv0mon.h +++ b/storage/xtradb/include/srv0mon.h @@ -2,6 +2,7 @@ Copyright (c) 2010, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -541,22 +542,30 @@ on the counters */ /** Increment a monitor counter under mutex protection. Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be incremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_INC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_INC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ - if (MONITOR_IS_ON(monitor)) { \ + if (enabled) { \ mutex_enter(mutex); \ if (++MONITOR_VALUE(monitor) > MONITOR_MAX_VALUE(monitor)) { \ MONITOR_MAX_VALUE(monitor) = MONITOR_VALUE(monitor); \ } \ mutex_exit(mutex); \ } +/** Increment a monitor counter under mutex protection. +Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be incremented by 1 */ +#define MONITOR_MUTEX_INC(mutex, monitor) \ + MONITOR_MUTEX_INC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) /** Decrement a monitor counter under mutex protection. Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be decremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_DEC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_DEC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ if (MONITOR_IS_ON(monitor)) { \ mutex_enter(mutex); \ @@ -565,13 +574,20 @@ Use MONITOR_DEC if appropriate mutex protection already exists. } \ mutex_exit(mutex); \ } +/** Decrement a monitor counter under mutex protection. +Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be decremented by 1 */ +#define MONITOR_MUTEX_DEC(mutex, monitor) \ + MONITOR_MUTEX_DEC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) #if defined HAVE_ATOMIC_BUILTINS_64 /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_increment_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -584,9 +600,10 @@ Use MONITOR_INC if appropriate mutex protection exists. /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_decrement_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -617,14 +634,29 @@ srv_mon_free(void); /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) MONITOR_MUTEX_INC(&monitor_mutex, monitor) +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + MONITOR_MUTEX_INC_LOW(&monitor_mutex, monitor, enabled) /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) MONITOR_MUTEX_DEC(&monitor_mutex, monitor) +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + MONITOR_MUTEX_DEC_LOW(&monitor_mutex, monitor, enabled) #endif /* HAVE_ATOMIC_BUILTINS_64 */ +/** Atomically increment a monitor counter if it is enabled. +Use MONITOR_INC if appropriate mutex protection exists. +@param monitor monitor to be incremented by 1 */ +#define MONITOR_ATOMIC_INC(monitor) \ + MONITOR_ATOMIC_INC_LOW(monitor, MONITOR_IS_ON(monitor)) +/** Atomically decrement a monitor counter if it is enabled. +Use MONITOR_DEC if appropriate mutex protection exists. +@param monitor monitor to be decremented by 1 */ +#define MONITOR_ATOMIC_DEC(monitor) \ + MONITOR_ATOMIC_DEC_LOW(monitor, MONITOR_IS_ON(monitor)) + #define MONITOR_DEC(monitor) \ if (MONITOR_IS_ON(monitor)) { \ MONITOR_VALUE(monitor)--; \ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 45c9cb80566..1ca8e6de591 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -55,11 +55,10 @@ Created 10/10/1995 Heikki Tuuri /* Global counters used inside InnoDB. */ struct srv_stats_t { - typedef ib_counter_t<lsn_t, 1, single_indexer_t> lsn_ctr_1_t; - typedef ib_counter_t<ulint, 1, single_indexer_t> ulint_ctr_1_t; - typedef ib_counter_t<lint, 1, single_indexer_t> lint_ctr_1_t; typedef ib_counter_t<ulint, 64> ulint_ctr_64_t; - typedef ib_counter_t<ib_int64_t, 1, single_indexer_t> ib_int64_ctr_1_t; + typedef simple_counter<lsn_t> lsn_ctr_1_t; + typedef simple_counter<ulint> ulint_ctr_1_t; + typedef simple_counter<ib_int64_t> ib_int64_ctr_1_t; /** Count the amount of data written in total (in bytes) */ ulint_ctr_1_t data_written; @@ -73,8 +72,9 @@ struct srv_stats_t { /** Amount of data written to the log files in bytes */ lsn_ctr_1_t os_log_written; - /** Number of writes being done to the log files */ - lint_ctr_1_t os_log_pending_writes; + /** Number of writes being done to the log files. + Protected by log_sys->write_mutex. */ + ulint_ctr_1_t os_log_pending_writes; /** We increase this counter, when we don't have enough space in the log buffer and have to flush it */ @@ -113,7 +113,7 @@ struct srv_stats_t { ulint_ctr_1_t n_lock_wait_count; /** Number of threads currently waiting on database locks */ - lint_ctr_1_t n_lock_wait_current_count; + simple_counter<ulint, true> n_lock_wait_current_count; /** Number of rows read. */ ulint_ctr_64_t n_rows_read; @@ -152,13 +152,16 @@ extern const char* srv_main_thread_op_info; /** Prefix used by MySQL to indicate pre-5.1 table name encoding */ extern const char srv_mysql50_table_name_prefix[10]; -/* The monitor thread waits on this event. */ +/** Event to signal srv_monitor_thread. Not protected by a mutex. +Set after setting srv_print_innodb_monitor. */ extern os_event_t srv_monitor_event; -/* The error monitor thread waits on this event. */ +/** Event to signal the shutdown of srv_error_monitor_thread. +Not protected by a mutex. */ extern os_event_t srv_error_event; -/** The buffer pool dump/load thread waits on this event. */ +/** Event for waking up buf_dump_thread. Not protected by a mutex. +Set on shutdown or by buf_dump_start() or buf_load_start(). */ extern os_event_t srv_buf_dump_event; /** The buffer pool dump/load file name */ @@ -419,9 +422,6 @@ extern double srv_adaptive_flushing_lwm; extern ulong srv_flushing_avg_loops; extern ulong srv_force_recovery; -#ifndef DBUG_OFF -extern ulong srv_force_recovery_crash; -#endif /* !DBUG_OFF */ extern ulint srv_fast_shutdown; /*!< If this is 1, do not do a purge and index buffer merge. @@ -436,6 +436,7 @@ extern unsigned long long srv_stats_transient_sample_pages; extern my_bool srv_stats_persistent; extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; +extern my_bool srv_stats_include_delete_marked; extern unsigned long long srv_stats_modified_counter; extern my_bool srv_stats_sample_traditional; @@ -968,17 +969,12 @@ ulint srv_get_task_queue_length(void); /*===========================*/ -/*********************************************************************//** -Releases threads of the type given from suspension in the thread table. -NOTE! The server mutex has to be reserved by the caller! -@return number of threads released: this may be less than n if not -enough threads were suspended at the moment */ -UNIV_INTERN -ulint -srv_release_threads( -/*================*/ - enum srv_thread_type type, /*!< in: thread type */ - ulint n); /*!< in: number of threads to release */ +/** Ensure that a given number of threads of the type given are running +(or are already terminated). +@param[in] type thread type +@param[in] n number of threads that have to run */ +void +srv_release_threads(enum srv_thread_type type, ulint n); /**********************************************************************//** Check whether any background thread are active. If so print which thread @@ -989,12 +985,10 @@ const char* srv_any_background_threads_are_active(void); /*=======================================*/ -/**********************************************************************//** -Wakeup the purge threads. */ +/** Wake up the purge threads. */ UNIV_INTERN void -srv_purge_wakeup(void); -/*==================*/ +srv_purge_wakeup(); /** Status variables to be passed to MySQL */ struct export_var_t{ @@ -1153,4 +1147,12 @@ struct srv_slot_t{ # define srv_file_per_table 1 #endif /* !UNIV_HOTBACKUP */ +#ifndef DBUG_OFF +/** false before InnoDB monitor has been printed at least once, true +afterwards */ +extern bool srv_debug_monitor_printed; +#else +#define srv_debug_monitor_printed false +#endif + #endif diff --git a/storage/xtradb/include/trx0purge.h b/storage/xtradb/include/trx0purge.h index a862523c092..7b9b5dc49cd 100644 --- a/storage/xtradb/include/trx0purge.h +++ b/storage/xtradb/include/trx0purge.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -145,7 +146,10 @@ struct trx_purge_t{ log operation can prevent this by obtaining an s-latch here. It also protects state and running */ - os_event_t event; /*!< State signal event */ + os_event_t event; /*!< State signal event; + os_event_set() and os_event_reset() + are protected by trx_purge_t::latch + X-lock */ ulint n_stop; /*!< Counter to track number stops */ volatile bool running; /*!< true, if purge is active, we check this without the latch too */ diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h index 84817c9645a..44f49d47b4d 100644 --- a/storage/xtradb/include/trx0trx.h +++ b/storage/xtradb/include/trx0trx.h @@ -106,7 +106,7 @@ void trx_free_prepared( /*==============*/ trx_t* trx) /*!< in, own: trx object */ - UNIV_COLD MY_ATTRIBUTE((nonnull)); + UNIV_COLD; /********************************************************************//** Frees a transaction object for MySQL. */ UNIV_INTERN diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 397843d309c..fff5c0a67a1 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -44,10 +45,10 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 34 +#define INNODB_VERSION_BUGFIX 35 #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 79.1 +#define PERCONA_INNODB_VERSION 80.0 #endif /* Enable UNIV_LOG_ARCHIVE in XtraDB */ @@ -183,7 +184,7 @@ command. Not tested on Windows. */ #define UNIV_COMPILE_TEST_FUNCS */ -#if defined(HAVE_valgrind)&& defined(HAVE_VALGRIND_MEMCHECK_H) +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif #if 0 @@ -303,22 +304,12 @@ definitions: */ #endif /* !UNIV_MUST_NOT_INLINE */ -#ifdef _WIN32 -#define UNIV_WORD_SIZE 4 -#elif defined(_WIN64) -#define UNIV_WORD_SIZE 8 -#else -/** MySQL config.h generated by GNU autoconf will define SIZEOF_LONG in Posix */ -#define UNIV_WORD_SIZE SIZEOF_LONG -#endif +#define UNIV_WORD_SIZE SIZEOF_SIZE_T /** The following alignment is used in memory allocations in memory heap management to ensure correct alignment for doubles etc. */ #define UNIV_MEM_ALIGNMENT 8 -/** The following alignment is used in aligning lints etc. */ -#define UNIV_WORD_ALIGNMENT UNIV_WORD_SIZE - /* DATABASE VERSION CONTROL ======================== @@ -447,13 +438,12 @@ the word size of the machine, that is on a 32-bit platform 32 bits, and on a macro ULINTPF. */ -#ifdef __WIN__ +#ifdef _WIN32 /* Use the integer types and formatting strings defined in Visual Studio. */ -# define UINT32PF "%I32u" -# define INT64PF "%I64d" -# define UINT64PF "%I64u" -# define UINT64PFx "%016I64x" -# define DBUG_LSN_PF "%llu" +# define UINT32PF "%u" +# define INT64PF "%lld" +# define UINT64PF "%llu" +# define UINT64PFx "%016llx" typedef __int64 ib_int64_t; typedef unsigned __int64 ib_uint64_t; typedef unsigned __int32 ib_uint32_t; @@ -463,13 +453,12 @@ typedef unsigned __int32 ib_uint32_t; # define INT64PF "%" PRId64 # define UINT64PF "%" PRIu64 # define UINT64PFx "%016" PRIx64 -# define DBUG_LSN_PF UINT64PF typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; -# endif /* __WIN__ */ +#endif -# define IB_ID_FMT UINT64PF +#define IB_ID_FMT UINT64PF #ifdef _WIN64 typedef unsigned __int64 ulint; diff --git a/storage/xtradb/include/ut0counter.h b/storage/xtradb/include/ut0counter.h index 63a133a175d..edc0db3b03d 100644 --- a/storage/xtradb/include/ut0counter.h +++ b/storage/xtradb/include/ut0counter.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,13 +31,7 @@ Created 2012/04/12 by Sunny Bains #include "univ.i" #include <string.h> #include "os0thread.h" - -/** CPU cache line size */ -#ifdef __powerpc__ -#define CACHE_LINE_SIZE 128 -#else -#define CACHE_LINE_SIZE 64 -#endif +#include "os0sync.h" /** Default number of slots to use in ib_counter_t */ #define IB_N_SLOTS 64 @@ -44,8 +39,6 @@ Created 2012/04/12 by Sunny Bains /** Get the offset into the counter array. */ template <typename Type, int N> struct generic_indexer_t { - /** Default constructor/destructor should be OK. */ - /** @return offset within m_counter */ size_t offset(size_t index) const UNIV_NOTHROW { return(((index % N) + 1) * (CACHE_LINE_SIZE / sizeof(Type))); @@ -58,8 +51,6 @@ struct generic_indexer_t { use the thread id. */ template <typename Type, int N> struct get_sched_indexer_t : public generic_indexer_t<Type, N> { - /** Default constructor/destructor should be OK. */ - /* @return result from sched_getcpu(), the thread id if it fails. */ size_t get_rnd_index() const UNIV_NOTHROW { @@ -76,31 +67,17 @@ struct get_sched_indexer_t : public generic_indexer_t<Type, N> { /** Use the thread id to index into the counter array. */ template <typename Type, int N> struct thread_id_indexer_t : public generic_indexer_t<Type, N> { - /** Default constructor/destructor should are OK. */ - /* @return a random number, currently we use the thread id. Where thread id is represented as a pointer, it may not work as effectively. */ size_t get_rnd_index() const UNIV_NOTHROW { return((lint) os_thread_get_curr_id()); } -}; - -/** For counters wher N=1 */ -template <typename Type, int N=1> -struct single_indexer_t { - /** Default constructor/destructor should are OK. */ - - /** @return offset within m_counter */ - size_t offset(size_t index) const UNIV_NOTHROW { - ut_ad(N == 1); - return((CACHE_LINE_SIZE / sizeof(Type))); - } - /* @return 1 */ - size_t get_rnd_index() const UNIV_NOTHROW { - ut_ad(N == 1); - return(1); + /** @return a random offset to the array */ + size_t get_rnd_offset() const UNIV_NOTHROW + { + return(generic_indexer_t<Type, N>::offset(get_rnd_index())); } }; @@ -112,17 +89,11 @@ template < typename Type, int N = IB_N_SLOTS, template<typename, int> class Indexer = thread_id_indexer_t> -class ib_counter_t { -public: - ib_counter_t() { memset(m_counter, 0x0, sizeof(m_counter)); } - +struct MY_ALIGNED(CACHE_LINE_SIZE) ib_counter_t +{ +#ifdef UNIV_DEBUG ~ib_counter_t() { - ut_ad(validate()); - } - - bool validate() UNIV_NOTHROW { -#ifdef UNIV_DEBUG size_t n = (CACHE_LINE_SIZE / sizeof(Type)); /* Check that we aren't writing outside our defined bounds. */ @@ -131,27 +102,23 @@ public: ut_ad(m_counter[i + j] == 0); } } -#endif /* UNIV_DEBUG */ - return(true); } +#endif /* UNIV_DEBUG */ - /** If you can't use a good index id. Increment by 1. */ + /** Increment the counter by 1. */ void inc() UNIV_NOTHROW { add(1); } - /** If you can't use a good index id. - * @param n - is the amount to increment */ - void add(Type n) UNIV_NOTHROW { - size_t i = m_policy.offset(m_policy.get_rnd_index()); - - ut_ad(i < UT_ARR_SIZE(m_counter)); + /** Increment the counter by 1. + @param[in] index a reasonably thread-unique identifier */ + void inc(size_t index) UNIV_NOTHROW { add(index, 1); } - m_counter[i] += n; - } + /** Add to the counter. + @param[in] n amount to be added */ + void add(Type n) UNIV_NOTHROW { add(m_policy.get_rnd_offset(), n); } - /** Use this if you can use a unique indentifier, saves a - call to get_rnd_index(). - @param i - index into a slot - @param n - amount to increment */ + /** Add to the counter. + @param[in] index a reasonably thread-unique identifier + @param[in] n amount to be added */ void add(size_t index, Type n) UNIV_NOTHROW { size_t i = m_policy.offset(index); @@ -160,31 +127,6 @@ public: m_counter[i] += n; } - /** If you can't use a good index id. Decrement by 1. */ - void dec() UNIV_NOTHROW { sub(1); } - - /** If you can't use a good index id. - * @param - n is the amount to decrement */ - void sub(Type n) UNIV_NOTHROW { - size_t i = m_policy.offset(m_policy.get_rnd_index()); - - ut_ad(i < UT_ARR_SIZE(m_counter)); - - m_counter[i] -= n; - } - - /** Use this if you can use a unique indentifier, saves a - call to get_rnd_index(). - @param i - index into a slot - @param n - amount to decrement */ - void sub(size_t index, Type n) UNIV_NOTHROW { - size_t i = m_policy.offset(index); - - ut_ad(i < UT_ARR_SIZE(m_counter)); - - m_counter[i] -= n; - } - /* @return total value - not 100% accurate, since it is not atomic. */ operator Type() const UNIV_NOTHROW { Type total = 0; diff --git a/storage/xtradb/include/ut0wqueue.h b/storage/xtradb/include/ut0wqueue.h index 33385ddf2d4..09b2eb717a7 100644 --- a/storage/xtradb/include/ut0wqueue.h +++ b/storage/xtradb/include/ut0wqueue.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -99,7 +100,9 @@ ib_wqueue_timedwait( struct ib_wqueue_t { ib_mutex_t mutex; /*!< mutex protecting everything */ ib_list_t* items; /*!< work item list */ - os_event_t event; /*!< event we use to signal additions to list */ + os_event_t event; /*!< event we use to signal additions to list; + os_event_set() and os_event_reset() are + protected by ib_wqueue_t::mutex */ }; #endif diff --git a/storage/xtradb/lock/lock0wait.cc b/storage/xtradb/lock/lock0wait.cc index 4ca048a50d5..5b67fbe72b7 100644 --- a/storage/xtradb/lock/lock0wait.cc +++ b/storage/xtradb/lock/lock0wait.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -248,6 +249,9 @@ lock_wait_suspend_thread( slot = lock_wait_table_reserve_slot(thr, lock_wait_timeout); + lock_wait_mutex_exit(); + trx_mutex_exit(trx); + if (thr->lock_state == QUE_THR_LOCK_ROW) { srv_stats.n_lock_wait_count.inc(); srv_stats.n_lock_wait_current_count.inc(); @@ -259,19 +263,21 @@ lock_wait_suspend_thread( } } - lock_wait_mutex_exit(); - trx_mutex_exit(trx); - ulint lock_type = ULINT_UNDEFINED; - lock_mutex_enter(); - + /* The wait_lock can be cleared by another thread when the + lock is released. But the wait can only be initiated by the + current thread which owns the transaction. Only acquire the + mutex if the wait_lock is still active. */ if (const lock_t* wait_lock = trx->lock.wait_lock) { - lock_type = lock_get_type_low(wait_lock); + lock_mutex_enter(); + wait_lock = trx->lock.wait_lock; + if (wait_lock) { + lock_type = lock_get_type_low(wait_lock); + } + lock_mutex_exit(); } - lock_mutex_exit(); - had_dict_lock = trx->dict_operation_lock_mode; switch (had_dict_lock) { diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 15183b9c1cb..5ef9203ce72 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1569,17 +1570,7 @@ log_write_up_to( } loop: -#ifdef UNIV_DEBUG - loop_count++; - - ut_ad(loop_count < 5); - -# if 0 - if (loop_count > 2) { - fprintf(stderr, "Log loop count %lu\n", loop_count); - } -# endif -#endif + ut_ad(++loop_count < 100); mutex_enter(&(log_sys->mutex)); ut_ad(!recv_no_log_write); @@ -1867,7 +1858,7 @@ log_preflush_pool_modified_pages( and we could not make a new checkpoint on the basis of the info on the buffer pool only. */ - recv_apply_hashed_log_recs(TRUE); + recv_apply_hashed_log_recs(true); } if (!buf_page_cleaner_is_active @@ -2238,7 +2229,7 @@ log_checkpoint( ut_ad(!srv_read_only_mode); if (recv_recovery_is_on()) { - recv_apply_hashed_log_recs(TRUE); + recv_apply_hashed_log_recs(true); } if (srv_unix_file_flush_method != SRV_UNIX_NOSYNC && @@ -2577,6 +2568,11 @@ loop: start_lsn += len; buf += len; + if (recv_sys->report(ut_time())) { + ib_logf(IB_LOG_LEVEL_INFO, "Read redo log up to LSN=" LSN_PF, + start_lsn); + } + if (start_lsn != end_lsn) { if (release_mutex) { @@ -3505,7 +3501,6 @@ logs_empty_and_mark_files_at_shutdown(void) lsn_t lsn; lsn_t tracked_lsn; ulint count = 0; - ulint total_trx; ulint pending_io; enum srv_thread_type active_thd; const char* thread_name; @@ -3517,12 +3512,6 @@ logs_empty_and_mark_files_at_shutdown(void) if (log_disable_checkpoint_active) log_enable_checkpoint(); - while (srv_fast_shutdown == 0 && trx_rollback_or_clean_is_active) { - /* we should wait until rollback after recovery end - for slow shutdown */ - os_thread_sleep(100000); - } - /* Wait until the master thread and all other operations are idle: our algorithm only works if the server is idle at shutdown */ @@ -3556,10 +3545,9 @@ loop: shutdown, because the InnoDB layer may have committed or prepared transactions and we don't want to lose them. */ - total_trx = trx_sys_any_active_transactions(); - - if (total_trx > 0) { - + if (ulint total_trx = srv_was_started && !srv_read_only_mode + && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + ? trx_sys_any_active_transactions() : 0) { if (srv_print_verbose_log && count > 600) { ib_logf(IB_LOG_LEVEL_INFO, "Waiting for %lu active transactions to finish", @@ -3575,7 +3563,8 @@ loop: active_thd = srv_get_active_thread_type(); - if (active_thd != SRV_NONE) { + if (active_thd != SRV_NONE + || (srv_fast_shutdown != 2 && trx_rollback_or_clean_is_active)) { if (active_thd == SRV_PURGE) { srv_purge_wakeup(); @@ -3591,11 +3580,9 @@ loop: switch (active_thd) { case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; + thread_type = "rollback of" + " recovered transactions"; + break; case SRV_WORKER: thread_type = "worker threads"; break; diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 209ca743628..764c9e0c52f 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -76,12 +76,14 @@ struct log_bitmap_struct { both the correct type and the tree does not mind its overwrite during rbt_next() tree traversal. */ - ib_mutex_t mutex; /*!< mutex protecting all the fields.*/ }; /* The log parsing and bitmap output struct instance */ static struct log_bitmap_struct* log_bmp_sys; +/* Mutex protecting log_bmp_sys */ +static ib_mutex_t log_bmp_sys_mutex; + /** File name stem for bitmap files. */ static const char* bmp_file_name_stem = "ib_modified_log_"; @@ -173,28 +175,24 @@ log_online_set_page_bit( ulint space, /*!<in: log record space id */ ulint page_no)/*!<in: log record page id */ { - ulint block_start_page; - ulint block_pos; - uint bit_pos; - ib_rbt_bound_t tree_search_pos; - byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; - byte *page_ptr; - - ut_ad(mutex_own(&log_bmp_sys->mutex)); + ut_ad(mutex_own(&log_bmp_sys_mutex)); ut_a(space != ULINT_UNDEFINED); ut_a(page_no != ULINT_UNDEFINED); - block_start_page = page_no / MODIFIED_PAGE_BLOCK_ID_COUNT + ulint block_start_page = page_no / MODIFIED_PAGE_BLOCK_ID_COUNT * MODIFIED_PAGE_BLOCK_ID_COUNT; - block_pos = block_start_page ? (page_no % block_start_page / 8) + ulint block_pos = block_start_page ? (page_no % block_start_page / 8) : (page_no / 8); - bit_pos = page_no % 8; + uint bit_pos = page_no % 8; + byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space); mach_write_to_4(search_page + MODIFIED_PAGE_1ST_PAGE_ID, block_start_page); + byte *page_ptr; + ib_rbt_bound_t tree_search_pos; if (!rbt_search(log_bmp_sys->modified_pages, &tree_search_pos, search_page)) { page_ptr = rbt_value(byte, tree_search_pos.last); @@ -596,12 +594,19 @@ log_online_is_bitmap_file( && (!strcmp(stem, bmp_file_name_stem))); } -/*********************************************************************//** -Initialize the online log following subsytem. */ +/** Initialize the constant part of the log tracking subsystem */ +UNIV_INTERN +void +log_online_init(void) +{ + mutex_create(log_bmp_sys_mutex_key, &log_bmp_sys_mutex, + SYNC_LOG_ONLINE); +} + +/** Initialize the dynamic part of the log tracking subsystem */ UNIV_INTERN void log_online_read_init(void) -/*======================*/ { ibool success; lsn_t tracking_start_lsn @@ -625,9 +630,6 @@ log_online_read_init(void) log_bmp_sys->read_buf = static_cast<byte *> (ut_align(log_bmp_sys->read_buf_ptr, OS_FILE_LOG_BLOCK_SIZE)); - mutex_create(log_bmp_sys_mutex_key, &log_bmp_sys->mutex, - SYNC_LOG_ONLINE); - /* Initialize bitmap file directory from srv_data_home and add a path separator if needed. */ srv_data_home_len = strlen(srv_data_home); @@ -767,13 +769,15 @@ log_online_read_init(void) log_set_tracked_lsn(tracking_start_lsn); } -/*********************************************************************//** -Shut down the online log following subsystem. */ +/** Shut down the dynamic part of the log tracking subsystem */ UNIV_INTERN void log_online_read_shutdown(void) -/*==========================*/ { + mutex_enter(&log_bmp_sys_mutex); + + srv_track_changed_pages = FALSE; + ib_rbt_node_t *free_list_node = log_bmp_sys->page_free_list; if (log_bmp_sys->out.file != os_file_invalid) { @@ -789,10 +793,21 @@ log_online_read_shutdown(void) free_list_node = next; } - mutex_free(&log_bmp_sys->mutex); - ut_free(log_bmp_sys->read_buf_ptr); ut_free(log_bmp_sys); + log_bmp_sys = NULL; + + srv_redo_log_thread_started = false; + + mutex_exit(&log_bmp_sys_mutex); +} + +/** Shut down the constant part of the log tracking subsystem */ +UNIV_INTERN +void +log_online_shutdown(void) +{ + mutex_free(&log_bmp_sys_mutex); } /*********************************************************************//** @@ -838,13 +853,12 @@ void log_online_parse_redo_log(void) /*===========================*/ { + ut_ad(mutex_own(&log_bmp_sys_mutex)); + byte *ptr = log_bmp_sys->parse_buf; byte *end = log_bmp_sys->parse_buf_end; - ulint len = 0; - ut_ad(mutex_own(&log_bmp_sys->mutex)); - while (ptr != end && log_bmp_sys->next_parse_lsn < log_bmp_sys->end_lsn) { @@ -926,6 +940,8 @@ log_online_add_to_parse_buf( ulint skip_len) /*!< in: how much of log data to skip */ { + ut_ad(mutex_own(&log_bmp_sys_mutex)); + ulint start_offset = skip_len ? skip_len : LOG_BLOCK_HDR_SIZE; ulint end_offset = (data_len == OS_FILE_LOG_BLOCK_SIZE) @@ -934,8 +950,6 @@ log_online_add_to_parse_buf( ulint actual_data_len = (end_offset >= start_offset) ? end_offset - start_offset : 0; - ut_ad(mutex_own(&log_bmp_sys->mutex)); - ut_memcpy(log_bmp_sys->parse_buf_end, log_block + start_offset, actual_data_len); @@ -958,11 +972,9 @@ log_online_parse_redo_log_block( log data should be skipped as they were parsed before */ { - ulint block_data_len; + ut_ad(mutex_own(&log_bmp_sys_mutex)); - ut_ad(mutex_own(&log_bmp_sys->mutex)); - - block_data_len = log_block_get_data_len(log_block); + ulint block_data_len = log_block_get_data_len(log_block); ut_ad(block_data_len % OS_FILE_LOG_BLOCK_SIZE == 0 || block_data_len < OS_FILE_LOG_BLOCK_SIZE); @@ -982,14 +994,14 @@ log_online_follow_log_seg( lsn_t block_start_lsn, /*!< in: the LSN to read from */ lsn_t block_end_lsn) /*!< in: the LSN to read to */ { + ut_ad(mutex_own(&log_bmp_sys_mutex)); + /* Pointer to the current OS_FILE_LOG_BLOCK-sized chunk of the read log data to parse */ byte* log_block = log_bmp_sys->read_buf; byte* log_block_end = log_bmp_sys->read_buf + (block_end_lsn - block_start_lsn); - ut_ad(mutex_own(&log_bmp_sys->mutex)); - mutex_enter(&log_sys->mutex); log_group_read_log_seg(LOG_RECOVER, log_bmp_sys->read_buf, group, block_start_lsn, block_end_lsn, TRUE); @@ -1049,11 +1061,11 @@ log_online_follow_log_group( lsn_t contiguous_lsn) /*!< in: the LSN of log block start containing the log_parse_start_lsn */ { + ut_ad(mutex_own(&log_bmp_sys_mutex)); + lsn_t block_start_lsn = contiguous_lsn; lsn_t block_end_lsn; - ut_ad(mutex_own(&log_bmp_sys->mutex)); - log_bmp_sys->next_parse_lsn = log_bmp_sys->start_lsn; log_bmp_sys->parse_buf_end = log_bmp_sys->parse_buf; @@ -1090,20 +1102,28 @@ log_online_write_bitmap_page( /*=========================*/ const byte *block) /*!< in: block to write */ { - ibool success; - - ut_ad(srv_track_changed_pages); - ut_ad(mutex_own(&log_bmp_sys->mutex)); + ut_ad(mutex_own(&log_bmp_sys_mutex)); /* Simulate a write error */ DBUG_EXECUTE_IF("bitmap_page_write_error", - ib_logf(IB_LOG_LEVEL_ERROR, - "simulating bitmap write error in " - "log_online_write_bitmap_page"); - return FALSE;); - - success = os_file_write(log_bmp_sys->out.name, log_bmp_sys->out.file, - block, log_bmp_sys->out.offset, + { + ulint space_id + = mach_read_from_4(block + + MODIFIED_PAGE_SPACE_ID); + if (space_id > 0) { + ib_logf(IB_LOG_LEVEL_ERROR, + "simulating bitmap write " + "error in " + "log_online_write_bitmap_page " + "for space ID %lu", + space_id); + return FALSE; + } + }); + + ibool success = os_file_write(log_bmp_sys->out.name, + log_bmp_sys->out.file, block, + log_bmp_sys->out.offset, MODIFIED_PAGE_BLOCK_SIZE); if (UNIV_UNLIKELY(!success)) { @@ -1143,11 +1163,7 @@ ibool log_online_write_bitmap(void) /*=========================*/ { - ib_rbt_node_t *bmp_tree_node; - const ib_rbt_node_t *last_bmp_tree_node; - ibool success = TRUE; - - ut_ad(mutex_own(&log_bmp_sys->mutex)); + ut_ad(mutex_own(&log_bmp_sys_mutex)); if (log_bmp_sys->out.offset >= srv_max_bitmap_file_size) { if (!log_online_rotate_bitmap_file(log_bmp_sys->start_lsn)) { @@ -1155,9 +1171,12 @@ log_online_write_bitmap(void) } } - bmp_tree_node = (ib_rbt_node_t *) - rbt_first(log_bmp_sys->modified_pages); - last_bmp_tree_node = rbt_last(log_bmp_sys->modified_pages); + ib_rbt_node_t *bmp_tree_node + = (ib_rbt_node_t *)rbt_first(log_bmp_sys->modified_pages); + const ib_rbt_node_t * const last_bmp_tree_node + = rbt_last(log_bmp_sys->modified_pages); + + ibool success = TRUE; while (bmp_tree_node) { @@ -1190,9 +1209,11 @@ log_online_write_bitmap(void) rbt_next(log_bmp_sys->modified_pages, bmp_tree_node); DBUG_EXECUTE_IF("bitmap_page_2_write_error", - ut_ad(bmp_tree_node); /* 2nd page must exist */ - DBUG_SET("+d,bitmap_page_write_error"); - DBUG_SET("-d,bitmap_page_2_write_error");); + if (bmp_tree_node) + { + DBUG_SET("+d,bitmap_page_write_error"); + DBUG_SET("-d,bitmap_page_2_write_error"); + }); } rbt_reset(log_bmp_sys->modified_pages); @@ -1213,10 +1234,19 @@ log_online_follow_redo_log(void) log_group_t* group; ibool result; - ut_ad(srv_track_changed_pages); ut_ad(!srv_read_only_mode); - mutex_enter(&log_bmp_sys->mutex); + if (!srv_track_changed_pages) + return TRUE; + + DEBUG_SYNC_C("log_online_follow_redo_log"); + + mutex_enter(&log_bmp_sys_mutex); + + if (!srv_track_changed_pages) { + mutex_exit(&log_bmp_sys_mutex); + return TRUE; + } /* Grab the LSN of the last checkpoint, we will parse up to it */ mutex_enter(&(log_sys->mutex)); @@ -1224,7 +1254,7 @@ log_online_follow_redo_log(void) mutex_exit(&(log_sys->mutex)); if (log_bmp_sys->end_lsn == log_bmp_sys->start_lsn) { - mutex_exit(&log_bmp_sys->mutex); + mutex_exit(&log_bmp_sys_mutex); return TRUE; } @@ -1247,7 +1277,7 @@ log_online_follow_redo_log(void) log_bmp_sys->start_lsn = log_bmp_sys->end_lsn; log_set_tracked_lsn(log_bmp_sys->start_lsn); - mutex_exit(&log_bmp_sys->mutex); + mutex_exit(&log_bmp_sys_mutex); return result; } @@ -1594,6 +1624,8 @@ log_online_bitmap_iterator_init( { ut_a(i); + i->max_lsn = max_lsn; + if (UNIV_UNLIKELY(min_lsn > max_lsn)) { /* Empty range */ @@ -1702,6 +1734,9 @@ log_online_bitmap_iterator_next( return TRUE; } + if (i->end_lsn >= i->max_lsn && i->last_page_in_run) + return FALSE; + while (!checksum_ok) { while (i->in.size < MODIFIED_PAGE_BLOCK_SIZE @@ -1797,15 +1832,21 @@ log_online_purge_changed_page_bitmaps( lsn = LSN_MAX; } + bool log_bmp_sys_inited = false; if (srv_redo_log_thread_started) { /* User requests might happen with both enabled and disabled tracking */ - mutex_enter(&log_bmp_sys->mutex); + log_bmp_sys_inited = true; + mutex_enter(&log_bmp_sys_mutex); + if (!srv_redo_log_thread_started) { + log_bmp_sys_inited = false; + mutex_exit(&log_bmp_sys_mutex); + } } if (!log_online_setup_bitmap_file_range(&bitmap_files, 0, LSN_MAX)) { - if (srv_redo_log_thread_started) { - mutex_exit(&log_bmp_sys->mutex); + if (log_bmp_sys_inited) { + mutex_exit(&log_bmp_sys_mutex); } return TRUE; } @@ -1843,7 +1884,7 @@ log_online_purge_changed_page_bitmaps( } } - if (srv_redo_log_thread_started) { + if (log_bmp_sys_inited) { if (lsn > log_bmp_sys->end_lsn) { lsn_t new_file_lsn; if (lsn == LSN_MAX) { @@ -1859,7 +1900,7 @@ log_online_purge_changed_page_bitmaps( } } - mutex_exit(&log_bmp_sys->mutex); + mutex_exit(&log_bmp_sys_mutex); } free(bitmap_files.files); diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 9e42fb5cc1c..4edfb1e4347 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -79,7 +80,7 @@ this must be less than UNIV_PAGE_SIZE as it is stored in the buffer pool */ #define RECV_READ_AHEAD_AREA 32 /** The recovery system */ -UNIV_INTERN recv_sys_t* recv_sys = NULL; +UNIV_INTERN recv_sys_t* recv_sys; /** TRUE when applying redo log records during crash recovery; FALSE otherwise. Note that this is FALSE while a background thread is rolling back incomplete transactions. */ @@ -131,9 +132,6 @@ UNIV_INTERN ibool recv_is_making_a_backup = FALSE; UNIV_INTERN ibool recv_is_from_backup = FALSE; # define buf_pool_get_curr_size() (5 * 1024 * 1024) #endif /* !UNIV_HOTBACKUP */ -/** The following counter is used to decide when to print info on -log scan */ -static ulint recv_scan_print_counter; /** The type of the previous parsed redo log record */ static ulint recv_previous_parsed_rec_type; @@ -176,7 +174,7 @@ UNIV_INTERN mysql_pfs_key_t recv_writer_mutex_key; # endif /* UNIV_PFS_MUTEX */ /** Flag indicating if recv_writer thread is active. */ -UNIV_INTERN bool recv_writer_thread_active = false; +static volatile bool recv_writer_thread_active; UNIV_INTERN os_thread_t recv_writer_thread_handle = 0; #endif /* !UNIV_HOTBACKUP */ @@ -304,8 +302,6 @@ recv_sys_var_init(void) recv_no_ibuf_operations = FALSE; - recv_scan_print_counter = 0; - recv_previous_parsed_rec_type = 999999; recv_previous_parsed_rec_offset = 0; @@ -342,8 +338,6 @@ DECLARE_THREAD(recv_writer_thread)( os_thread_pf(os_thread_get_curr_id())); #endif /* UNIV_DEBUG_THREAD_CREATION */ - recv_writer_thread_active = true; - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { os_thread_sleep(100000); @@ -418,6 +412,7 @@ recv_sys_init( recv_sys->last_block_buf_start, OS_FILE_LOG_BLOCK_SIZE)); recv_sys->found_corrupt_log = FALSE; + recv_sys->progress_time = ut_time(); recv_max_page_lsn = 0; @@ -1676,6 +1671,7 @@ recv_recover_page_func( ibool success; #endif /* !UNIV_HOTBACKUP */ mtr_t mtr; + ib_time_t time; mutex_enter(&(recv_sys->mutex)); @@ -1796,7 +1792,7 @@ recv_recover_page_func( } DBUG_PRINT("ib_log", - ("apply " DBUG_LSN_PF ": %u len %u " + ("apply " LSN_PF ": %u len %u " "page %u:%u", recv->start_lsn, (unsigned) recv->type, (unsigned) recv->len, @@ -1853,6 +1849,8 @@ recv_recover_page_func( mtr_commit(&mtr); + time = ut_time(); + mutex_enter(&(recv_sys->mutex)); if (recv_max_page_lsn < page_lsn) { @@ -1861,11 +1859,16 @@ recv_recover_page_func( recv_addr->state = RECV_PROCESSED; - ut_a(recv_sys->n_addrs); - recv_sys->n_addrs--; - - mutex_exit(&(recv_sys->mutex)); + ut_a(recv_sys->n_addrs > 0); + if (--recv_sys->n_addrs && recv_sys->progress_time - time >= 15) { + recv_sys->progress_time = time; + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: To recover: " ULINTPF " pages from log\n", + recv_sys->n_addrs); + } + mutex_exit(&recv_sys->mutex); } #ifndef UNIV_HOTBACKUP @@ -1911,59 +1914,48 @@ recv_read_in_area( } buf_read_recv_pages(FALSE, space, zip_size, page_nos, n); - /* - fprintf(stderr, "Recv pages at %lu n %lu\n", page_nos[0], n); - */ return(n); } -/*******************************************************************//** -Empties the hash table of stored log records, applying them to appropriate -pages. */ +/** Apply the hash table of stored log records to persistent data pages. +@param[in] last_batch whether the change buffer merge will be + performed as part of the operation */ UNIV_INTERN void -recv_apply_hashed_log_recs( -/*=======================*/ - ibool allow_ibuf) /*!< in: if TRUE, also ibuf operations are - allowed during the application; if FALSE, - no ibuf operations are allowed, and after - the application all file pages are flushed to - disk and invalidated in buffer pool: this - alternative means that no new log records - can be generated during the application; - the caller must in this case own the log - mutex */ +recv_apply_hashed_log_recs(bool last_batch) { - recv_addr_t* recv_addr; - ulint i; - ibool has_printed = FALSE; - mtr_t mtr; -loop: - mutex_enter(&(recv_sys->mutex)); - - if (recv_sys->apply_batch_on) { + for (;;) { + mutex_enter(&recv_sys->mutex); - mutex_exit(&(recv_sys->mutex)); + if (!recv_sys->apply_batch_on) { + break; + } + mutex_exit(&recv_sys->mutex); os_thread_sleep(500000); - - goto loop; } - ut_ad((allow_ibuf == 0) == (mutex_own(&log_sys->mutex) != 0)); + ut_ad(!last_batch == mutex_own(&log_sys->mutex)); - if (!allow_ibuf) { + if (!last_batch) { recv_no_ibuf_operations = TRUE; } + if (ulint n = recv_sys->n_addrs) { + const char* msg = last_batch + ? "Starting final batch to recover " + : "Starting a batch to recover "; + ib_logf(IB_LOG_LEVEL_INFO, + "%s" ULINTPF " pages from redo log", msg, n); + } + recv_sys->apply_log_recs = TRUE; recv_sys->apply_batch_on = TRUE; - for (i = 0; i < hash_get_n_cells(recv_sys->addr_hash); i++) { - - for (recv_addr = static_cast<recv_addr_t*>( - HASH_GET_FIRST(recv_sys->addr_hash, i)); - recv_addr != 0; + for (ulint i = 0; i < hash_get_n_cells(recv_sys->addr_hash); i++) { + for (recv_addr_t* recv_addr = static_cast<recv_addr_t*>( + HASH_GET_FIRST(recv_sys->addr_hash, i)); + recv_addr; recv_addr = static_cast<recv_addr_t*>( HASH_GET_NEXT(addr_hash, recv_addr))) { @@ -1972,24 +1964,12 @@ loop: ulint page_no = recv_addr->page_no; if (recv_addr->state == RECV_NOT_PROCESSED) { - if (!has_printed) { - ib_logf(IB_LOG_LEVEL_INFO, - "Starting an apply batch" - " of log records" - " to the database..."); - fputs("InnoDB: Progress in percent: ", - stderr); - has_printed = TRUE; - } - - mutex_exit(&(recv_sys->mutex)); + mutex_exit(&recv_sys->mutex); if (buf_page_peek(space, page_no)) { - buf_block_t* block; - + mtr_t mtr; mtr_start(&mtr); - - block = buf_page_get( + buf_block_t* block = buf_page_get( space, zip_size, page_no, RW_X_LATCH, &mtr); buf_block_dbg_add_level( @@ -2002,19 +1982,9 @@ loop: page_no); } - mutex_enter(&(recv_sys->mutex)); + mutex_enter(&recv_sys->mutex); } } - - if (has_printed - && (i * 100) / hash_get_n_cells(recv_sys->addr_hash) - != ((i + 1) * 100) - / hash_get_n_cells(recv_sys->addr_hash)) { - - fprintf(stderr, "%lu ", (ulong) - ((i * 100) - / hash_get_n_cells(recv_sys->addr_hash))); - } } /* Wait until all the pages have been processed */ @@ -2028,12 +1998,7 @@ loop: mutex_enter(&(recv_sys->mutex)); } - if (has_printed) { - - fprintf(stderr, "\n"); - } - - if (!allow_ibuf) { + if (!last_batch) { bool success; /* Flush all the file pages to disk and invalidate them in @@ -2073,11 +2038,7 @@ loop: recv_sys_empty_hash(); - if (has_printed) { - fprintf(stderr, "InnoDB: Apply batch completed\n"); - } - - mutex_exit(&(recv_sys->mutex)); + mutex_exit(&recv_sys->mutex); } #else /* !UNIV_HOTBACKUP */ /*******************************************************************//** @@ -2100,11 +2061,6 @@ recv_apply_log_recs_for_backup(void) block = back_block1; - ib_logf(IB_LOG_LEVEL_INFO, - "Starting an apply batch of log records to the database..."); - - fputs("InnoDB: Progress in percent: ", stderr); - n_hash_cells = hash_get_n_cells(recv_sys->addr_hash); for (i = 0; i < n_hash_cells; i++) { @@ -2216,13 +2172,6 @@ recv_apply_log_recs_for_backup(void) skip_this_recv_addr: recv_addr = HASH_GET_NEXT(addr_hash, recv_addr); } - - if ((100 * i) / n_hash_cells - != (100 * (i + 1)) / n_hash_cells) { - fprintf(stderr, "%lu ", - (ulong) ((100 * i) / n_hash_cells)); - fflush(stderr); - } } recv_sys_empty_hash(); @@ -2491,7 +2440,7 @@ loop: recv_sys->recovered_lsn = new_recovered_lsn; DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": log rec %u len %u " + ("scan " LSN_PF ": log rec %u len %u " "page %u:%u", old_lsn, (unsigned) type, (unsigned) len, (unsigned) space, (unsigned) page_no)); @@ -2582,7 +2531,7 @@ loop: #endif /* UNIV_LOG_DEBUG */ DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": multi-log rec %u " + ("scan " LSN_PF ": multi-log rec %u " "len %u page %u:%u", recv_sys->recovered_lsn, (unsigned) type, (unsigned) len, @@ -2886,20 +2835,18 @@ recv_scan_log_recs( #ifndef UNIV_HOTBACKUP if (recv_log_scan_is_startup_type && !recv_needed_recovery) { - if (!srv_read_only_mode) { ib_logf(IB_LOG_LEVEL_INFO, - "Log scan progressed past the " - "checkpoint lsn " LSN_PF "", + "Starting crash recovery from " + "checkpoint LSN=" LSN_PF, recv_sys->scanned_lsn); recv_init_crash_recovery(); } else { - - ib_logf(IB_LOG_LEVEL_WARN, - "Recovery skipped, " - "--innodb-read-only set!"); - + ib_logf(IB_LOG_LEVEL_ERROR, + "innodb_read_only prevents" + " crash recovery"); + recv_needed_recovery = TRUE; return(TRUE); } } @@ -2950,19 +2897,6 @@ recv_scan_log_recs( *group_scanned_lsn = scanned_lsn; - if (recv_needed_recovery - || (recv_is_from_backup && !recv_is_making_a_backup)) { - recv_scan_print_counter++; - - if (finished || (recv_scan_print_counter % 80 == 0)) { - - fprintf(stderr, - "InnoDB: Doing recovery: scanned up to" - " log sequence number " LSN_PF "\n", - *group_scanned_lsn); - } - } - if (more_data && !recv_sys->found_corrupt_log) { /* Try to parse more log records */ @@ -2978,7 +2912,7 @@ recv_scan_log_recs( log yet: they would be produced by ibuf operations */ - recv_apply_hashed_log_recs(FALSE); + recv_apply_hashed_log_recs(false); } #endif /* !UNIV_HOTBACKUP */ @@ -3054,11 +2988,6 @@ recv_init_crash_recovery(void) recv_needed_recovery = TRUE; - ib_logf(IB_LOG_LEVEL_INFO, "Database was not shutdown normally!"); - ib_logf(IB_LOG_LEVEL_INFO, "Starting crash recovery."); - ib_logf(IB_LOG_LEVEL_INFO, - "Reading tablespace information from the .ibd files..."); - fil_load_single_table_tablespaces(); /* If we are using the doublewrite method, we will @@ -3069,15 +2998,14 @@ recv_init_crash_recovery(void) if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { ib_logf(IB_LOG_LEVEL_INFO, - "Restoring possible half-written data pages "); - - ib_logf(IB_LOG_LEVEL_INFO, + "Restoring possible half-written data pages " "from the doublewrite buffer..."); buf_dblwr_process(); /* Spawn the background thread to flush dirty pages from the buffer pools. */ + recv_writer_thread_active = true; recv_writer_thread_handle = os_thread_create( recv_writer_thread, 0, 0); } @@ -3323,6 +3251,11 @@ recv_recovery_from_checkpoint_start_func( /* Done with startup scan. Clear the flag. */ recv_log_scan_is_startup_type = FALSE; + + if (srv_read_only_mode && recv_needed_recovery) { + return(DB_READ_ONLY); + } + if (TYPE_CHECKPOINT) { /* NOTE: we always do a 'recovery' at startup, but only if there is something wrong we will print a message to the @@ -3473,15 +3406,6 @@ void recv_recovery_from_checkpoint_finish(void) /*======================================*/ { - /* Apply the hashed log records to the respective file pages */ - - if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - - recv_apply_hashed_log_recs(TRUE); - } - - DBUG_PRINT("ib_log", ("apply completed")); - if (recv_needed_recovery) { trx_sys_print_mysql_master_log_pos(); trx_sys_print_mysql_binlog_offset(); diff --git a/storage/xtradb/mach/mach0data.cc b/storage/xtradb/mach/mach0data.cc index 206434dc5ab..feeedb01609 100644 --- a/storage/xtradb/mach/mach0data.cc +++ b/storage/xtradb/mach/mach0data.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,7 +55,6 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); - } /* Workaround GCC bug @@ -64,7 +63,11 @@ mach_parse_compressed( function, causing and out-of-bounds read if we are reading a short integer close to the end of buffer. */ #if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) - asm volatile("": : :"memory"); +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); #endif if (flag < 0xC0UL) { @@ -75,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } - } else if (flag < 0xE0UL) { +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -84,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -92,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); + + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } diff --git a/storage/xtradb/mtr/mtr0mtr.cc b/storage/xtradb/mtr/mtr0mtr.cc index 5ed0803727f..ed44fba9762 100644 --- a/storage/xtradb/mtr/mtr0mtr.cc +++ b/storage/xtradb/mtr/mtr0mtr.cc @@ -312,7 +312,6 @@ mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index baaac6ee321..9632bb77df9 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -209,11 +210,15 @@ struct os_aio_array_t{ os_event_t not_full; /*!< The event which is set to the signaled state when there is space in - the aio outside the ibuf segment */ + the aio outside the ibuf segment; + os_event_set() and os_event_reset() + are protected by os_aio_array_t::mutex */ os_event_t is_empty; /*!< The event which is set to the signaled state when there are no - pending i/os in this array */ + pending i/os in this array; + os_event_set() and os_event_reset() + are protected by os_aio_array_t::mutex */ ulint n_slots;/*!< Total number of slots in the aio array. This must be divisible by n_threads. */ @@ -255,8 +260,8 @@ struct os_aio_array_t{ #define OS_AIO_IO_SETUP_RETRY_ATTEMPTS 5 #endif -/** Array of events used in simulated aio */ -static os_event_t* os_aio_segment_wait_events = NULL; +/** Array of events used in simulated aio. */ +static os_event_t* os_aio_segment_wait_events; /** The aio arrays for non-ibuf i/o and ibuf i/o, as well as sync aio. These are NULL when the module has not yet been initialized. @{ */ @@ -286,21 +291,6 @@ UNIV_INTERN time_t os_last_printout; UNIV_INTERN ibool os_has_said_disk_full = FALSE; -#if !defined(UNIV_HOTBACKUP) \ - && (!defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8) -/** The mutex protecting the following counts of pending I/O operations */ -static os_ib_mutex_t os_file_count_mutex; -#endif /* !UNIV_HOTBACKUP && (!HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8) */ - -/** Number of pending os_file_pread() operations */ -UNIV_INTERN ulint os_file_n_pending_preads = 0; -/** Number of pending os_file_pwrite() operations */ -UNIV_INTERN ulint os_file_n_pending_pwrites = 0; -/** Number of pending write operations */ -UNIV_INTERN ulint os_n_pending_writes = 0; -/** Number of pending read operations */ -UNIV_INTERN ulint os_n_pending_reads = 0; - #ifdef UNIV_DEBUG # ifndef UNIV_HOTBACKUP /**********************************************************************//** @@ -882,10 +872,6 @@ void os_io_init_simple(void) /*===================*/ { -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_file_count_mutex = os_mutex_create(); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_file_seek_mutexes[i] = os_mutex_create(); } @@ -1095,50 +1081,15 @@ next_file: char* full_path; int ret; struct stat statinfo; -#ifdef HAVE_READDIR_R - char dirent_buf[sizeof(struct dirent) - + _POSIX_PATH_MAX + 100]; - /* In /mysys/my_lib.c, _POSIX_PATH_MAX + 1 is used as - the max file name len; but in most standards, the - length is NAME_MAX; we add 100 to be even safer */ -#endif next_file: -#ifdef HAVE_READDIR_R - ret = readdir_r(dir, (struct dirent*) dirent_buf, &ent); - - if (ret != 0 -#ifdef UNIV_AIX - /* On AIX, only if we got non-NULL 'ent' (result) value and - a non-zero 'ret' (return) value, it indicates a failed - readdir_r() call. An NULL 'ent' with an non-zero 'ret' - would indicate the "end of the directory" is reached. */ - && ent != NULL -#endif - ) { - fprintf(stderr, - "InnoDB: cannot read directory %s, error %lu\n", - dirname, (ulong) ret); - - return(-1); - } - - if (ent == NULL) { - /* End of directory */ - - return(1); - } - - ut_a(strlen(ent->d_name) < _POSIX_PATH_MAX + 100 - 1); -#else ent = readdir(dir); if (ent == NULL) { return(1); } -#endif ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH); if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { @@ -1462,7 +1413,11 @@ os_file_set_nocache_if_needed(os_file_t file, const char* name, && (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT || (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC)))) - os_file_set_nocache(file, name, mode_str); + /* Do fsync() on log files when setting O_DIRECT fails. + See log_io_complete() */ + if (!os_file_set_nocache(file, name, mode_str) + && srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT) + srv_unix_file_flush_method = SRV_UNIX_O_DIRECT; } /****************************************************************//** @@ -1631,9 +1586,10 @@ os_file_create_simple_no_error_handling_func( } /****************************************************************//** -Tries to disable OS caching on an opened file descriptor. */ +Tries to disable OS caching on an opened file descriptor. +@return TRUE if operation is success and FALSE otherwise */ UNIV_INTERN -void +bool os_file_set_nocache( /*================*/ os_file_t fd /*!< in: file descriptor to alter */ @@ -1654,6 +1610,7 @@ os_file_set_nocache( "Failed to set DIRECTIO_ON on file %s: %s: %s, " "continuing anyway.", file_name, operation_name, strerror(errno_save)); + return false; } #elif defined(O_DIRECT) if (fcntl(fd, F_SETFL, O_DIRECT) == -1) { @@ -1684,8 +1641,10 @@ short_warning: "continuing anyway.", file_name, operation_name, strerror(errno_save)); } + return false; } #endif /* defined(UNIV_SOLARIS) && defined(DIRECTIO_ON) */ + return true; } @@ -2336,48 +2295,52 @@ os_file_set_size( os_file_t file, /*!< in: handle to a file */ os_offset_t size) /*!< in: file size */ { - os_offset_t current_size; ibool ret; byte* buf; byte* buf2; ulint buf_size; - current_size = 0; - #ifdef HAVE_POSIX_FALLOCATE if (srv_use_posix_fallocate) { + int err; + do { + err = posix_fallocate(file, 0, size); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); - if (posix_fallocate(file, current_size, size) == -1) { - - ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " - "space for file \'%s\' failed. Current size " - INT64PF ", desired size " INT64PF, - name, current_size, size); - os_file_handle_error_no_exit (name, "posix_fallocate", - FALSE); - return(FALSE); + if (err) { + ib_logf(IB_LOG_LEVEL_ERROR, + "preallocating " INT64PF " bytes for" + "file %s failed with error %d", + size, name, err); } - return(TRUE); + return(!err); } #endif +#ifdef _WIN32 + /* Write 1 page of zeroes at the desired end. */ + buf_size = UNIV_PAGE_SIZE; + os_offset_t current_size = size - buf_size; +#else /* Write up to 1 megabyte at a time. */ buf_size = ut_min(64, (ulint) (size / UNIV_PAGE_SIZE)) * UNIV_PAGE_SIZE; - buf2 = static_cast<byte*>(ut_malloc(buf_size + UNIV_PAGE_SIZE)); + os_offset_t current_size = 0; +#endif + buf2 = static_cast<byte*>(calloc(1, buf_size + UNIV_PAGE_SIZE)); + + if (!buf2) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot allocate " ULINTPF " bytes to extend file\n", + buf_size + UNIV_PAGE_SIZE); + return(FALSE); + } /* Align the buffer for possible raw i/o */ buf = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); - /* Write buffer full of zeros */ - memset(buf, 0, buf_size); - - if (size >= (os_offset_t) 100 << 20) { - - fprintf(stderr, "InnoDB: Progress in MB:"); - } - - while (current_size < size) { + do { ulint n_bytes; if (size - current_size < (os_offset_t) buf_size) { @@ -2388,37 +2351,15 @@ os_file_set_size( ret = os_file_write(name, file, buf, current_size, n_bytes); if (!ret) { - ut_free(buf2); - goto error_handling; - } - - /* Print about progress for each 100 MB written */ - if ((current_size + n_bytes) / (100 << 20) - != current_size / (100 << 20)) { - - fprintf(stderr, " %lu00", - (ulong) ((current_size + n_bytes) - / (100 << 20))); + break; } current_size += n_bytes; - } - - if (size >= (os_offset_t) 100 << 20) { + } while (current_size < size); - fprintf(stderr, "\n"); - } - - ut_free(buf2); - - ret = os_file_flush(file); - - if (ret) { - return(TRUE); - } + free(buf2); -error_handling: - return(FALSE); + return(ret && os_file_flush(file)); } /***********************************************************************//** @@ -2620,10 +2561,6 @@ os_file_pread( trx_t* trx) { off_t offs; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) - ssize_t n_bytes; - ssize_t n_read; -#endif /* HAVE_PREAD && !HAVE_BROKEN_PREAD */ ulint sec; ulint ms; ib_uint64_t start_time; @@ -2653,22 +2590,16 @@ os_file_pread( } else { start_time = 0; } -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads++; - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); +#ifdef HAVE_PREAD + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); + + ssize_t n_bytes; /* Handle partial reads and signal interruptions correctly */ for (n_bytes = 0; n_bytes < (ssize_t) n; ) { - n_read = pread(file, buf, (ssize_t)n - n_bytes, offs); + ssize_t n_read = pread(file, buf, (ssize_t)n - n_bytes, offs); if (n_read > 0) { n_bytes += n_read; offs += n_read; @@ -2680,17 +2611,7 @@ os_file_pread( } } -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads--; - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD == 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (UNIV_UNLIKELY(start_time != 0)) { @@ -2709,15 +2630,7 @@ os_file_pread( ulint i; #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; @@ -2747,15 +2660,7 @@ os_file_pread( os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - MONITOR_ATOIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD_SIZE == 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (UNIV_UNLIKELY(start_time != 0) { @@ -2801,18 +2706,9 @@ os_file_pwrite( os_n_file_writes++; -#if defined(HAVE_PWRITE) && !defined(HAVE_BROKEN_PREAD) -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites++; - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_increment_ulint(&os_n_pending_writes, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); +#ifdef HAVE_PWRITE + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); /* Handle partial writes and signal interruptions correctly */ for (ret = 0; ret < (ssize_t) n; ) { @@ -2831,17 +2727,7 @@ os_file_pwrite( } } -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites--; - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_decrement_ulint(&os_n_pending_writes, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); #else @@ -2851,10 +2737,7 @@ os_file_pwrite( ulint i; # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); # ifndef UNIV_HOTBACKUP /* Protect the seek / write operation with a mutex */ @@ -2888,14 +2771,10 @@ func_exit: os_mutex_exit(os_file_seek_mutexes[i]); # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); } -#endif /* !UNIV_HOTBACKUP */ +#endif /* HAVE_PWRITE */ } #endif @@ -2927,15 +2806,13 @@ os_file_read_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); ut_ad(n > 0); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); memset (&overlapped, 0, sizeof (overlapped)); overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); @@ -2948,10 +2825,7 @@ try_again: else if(GetLastError() == ERROR_IO_PENDING) { ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); } - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -3035,15 +2909,13 @@ os_file_read_no_error_handling_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); ut_ad(n > 0); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); memset (&overlapped, 0, sizeof (overlapped)); overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); @@ -3056,10 +2928,7 @@ try_again: else if(GetLastError() == ERROR_IO_PENDING) { ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); } - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -3153,12 +3022,10 @@ os_file_write_func( ut_ad(buf); ut_ad(n > 0); + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); retry: - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); memset (&overlapped, 0, sizeof (overlapped)); overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); @@ -3173,10 +3040,7 @@ retry: ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); } - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); if (ret && len == n) { @@ -4216,13 +4080,6 @@ os_aio_init( os_aio_validate(); - os_aio_segment_wait_events = static_cast<os_event_t*>( - ut_malloc(n_segments * sizeof *os_aio_segment_wait_events)); - - for (ulint i = 0; i < n_segments; ++i) { - os_aio_segment_wait_events[i] = os_event_create(); - } - os_last_printout = ut_time(); #ifdef _WIN32 @@ -4232,8 +4089,18 @@ os_aio_init( ut_a(completion_port && read_completion_port); #endif - return(TRUE); + if (srv_use_native_aio) { + return(TRUE); + } + + os_aio_segment_wait_events = static_cast<os_event_t*>( + ut_malloc(n_segments * sizeof *os_aio_segment_wait_events)); + + for (ulint i = 0; i < n_segments; ++i) { + os_aio_segment_wait_events[i] = os_event_create(); + } + return(TRUE); } /*********************************************************************** @@ -4261,14 +4128,12 @@ os_aio_free(void) os_aio_array_free(os_aio_read_array); - for (ulint i = 0; i < os_aio_n_segments; i++) { - os_event_free(os_aio_segment_wait_events[i]); + if (!srv_use_native_aio) { + for (ulint i = 0; i < os_aio_n_segments; i++) { + os_event_free(os_aio_segment_wait_events[i]); + } } -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_free(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_mutex_free(os_file_seek_mutexes[i]); } @@ -4322,22 +4187,17 @@ os_aio_wake_all_threads_at_shutdown(void) if (os_aio_log_array != 0) { os_aio_array_wake_win_aio_at_shutdown(os_aio_log_array); } - #elif defined(LINUX_NATIVE_AIO) - /* When using native AIO interface the io helper threads wait on io_getevents with a timeout value of 500ms. At each wake up these threads check the server status. No need to do anything to wake them up. */ +#endif /* !WIN_ASYNC_AIO */ if (srv_use_native_aio) { return; } - /* Fall through to simulated AIO handler wakeup if we are - not using native AIO. */ -#endif /* !WIN_ASYNC_AIO */ - /* This loop wakes up all simulated ai/o threads */ for (ulint i = 0; i < os_aio_n_segments; i++) { @@ -4705,6 +4565,7 @@ os_aio_simulated_wake_handler_threads(void) } } +#ifdef _WIN32 /**********************************************************************//** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must @@ -4712,15 +4573,14 @@ call os_aio_simulated_wake_handler_threads later to ensure the threads are not left sleeping! */ UNIV_INTERN void -os_aio_simulated_put_read_threads_to_sleep(void) -/*============================================*/ +os_aio_simulated_put_read_threads_to_sleep() { /* The idea of putting background IO threads to sleep is only for Windows when using simulated AIO. Windows XP seems to schedule background threads too eagerly to allow for coalescing during readahead requests. */ -#ifdef __WIN__ + os_aio_array_t* array; if (srv_use_native_aio) { @@ -4739,8 +4599,8 @@ readahead requests. */ os_event_reset(os_aio_segment_wait_events[i]); } } -#endif /* __WIN__ */ } +#endif /* _WIN32 */ #if defined(LINUX_NATIVE_AIO) /*******************************************************************//** @@ -5910,11 +5770,12 @@ os_aio_print( srv_io_thread_op_info[i], srv_io_thread_function[i]); -#ifndef __WIN__ - if (os_aio_segment_wait_events[i]->is_set()) { +#ifndef _WIN32 + if (!srv_use_native_aio + && os_aio_segment_wait_events[i]->is_set()) { fprintf(file, " ev set"); } -#endif /* __WIN__ */ +#endif /* _WIN32 */ fprintf(file, "\n"); } @@ -5948,19 +5809,24 @@ os_aio_print( time_elapsed = 0.001 + difftime(current_time, os_last_printout); fprintf(file, - "Pending flushes (fsync) log: %lu; buffer pool: %lu\n" - "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, - (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { + "Pending flushes (fsync) log: " ULINTPF + "; buffer pool: " ULINTPF "\n" + ULINTPF " OS file reads, " + ULINTPF " OS file writes, " + ULINTPF " OS fsyncs\n", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + const ulint n_reads = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); + const ulint n_writes = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); + + if (n_reads != 0 || n_writes != 0) { fprintf(file, - "%lu pending preads, %lu pending pwrites\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + ULINTPF " pending reads, " ULINTPF " pending writes\n", + n_reads, n_writes); } if (os_n_file_reads == os_n_file_reads_old) { diff --git a/storage/xtradb/os/os0thread.cc b/storage/xtradb/os/os0thread.cc index af826027efc..cb45929cab6 100644 --- a/storage/xtradb/os/os0thread.cc +++ b/storage/xtradb/os/os0thread.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -210,29 +210,32 @@ os_thread_create_func( #endif } -/** -Waits until the specified thread completes and joins it. Its return value is -ignored. - -@param thread thread to join */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ UNIV_INTERN void os_thread_join( os_thread_t thread) { - /*This function is currently only used to workaround glibc bug + /* This function is currently only used to workaround glibc bug described in http://bugs.mysql.com/bug.php?id=82886 On Windows, no workarounds are necessary, all threads are "detached" upon thread exit (handle is closed), so we do nothing. */ -#ifndef _WIN32 - int ret MY_ATTRIBUTE((unused)) = pthread_join(thread, NULL); +#ifdef __WIN__ + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret MY_ATTRIBUTE((unused)) = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); - /* Waiting on already-quit threads is allowed */ + /* Waiting on already-quit threads is allowed. */ ut_ad(ret == 0 || ret == ESRCH); -#endif +#endif /* __WIN__ */ } /*****************************************************************//** @@ -261,8 +264,9 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - if (detach) + if (detach) { pthread_detach(pthread_self()); + } pthread_exit(exit_value); #endif } diff --git a/storage/xtradb/page/page0page.cc b/storage/xtradb/page/page0page.cc index 48c4b53aaa4..1a8271de1e4 100644 --- a/storage/xtradb/page/page0page.cc +++ b/storage/xtradb/page/page0page.cc @@ -1447,7 +1447,6 @@ page_dir_split_slot( ulint i; ulint n_owned; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); @@ -1509,7 +1508,6 @@ page_dir_balance_slot( rec_t* old_rec; rec_t* new_rec; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); diff --git a/storage/xtradb/page/page0zip.cc b/storage/xtradb/page/page0zip.cc index 21008ff0ffe..0d2bb7fb986 100644 --- a/storage/xtradb/page/page0zip.cc +++ b/storage/xtradb/page/page0zip.cc @@ -4814,8 +4814,6 @@ page_zip_parse_compress( ulint size; ulint trailer_size; - ut_ad(ptr != NULL); - ut_ad(end_ptr != NULL); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { diff --git a/storage/xtradb/rem/rem0rec.cc b/storage/xtradb/rem/rem0rec.cc index 09cd810cd7b..c6d9ba935e1 100644 --- a/storage/xtradb/rem/rem0rec.cc +++ b/storage/xtradb/rem/rem0rec.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -320,8 +321,7 @@ rec_init_offsets_comp_ordinary( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (UNIV_UNLIKELY(col->len > 255 - - prtype_get_compression_extra(col->prtype)) + if (UNIV_UNLIKELY(col->len > 255) || UNIV_UNLIKELY(col->mtype == DATA_BLOB)) { if (len & 0x80) { @@ -786,7 +786,7 @@ rec_get_nth_field_offs_old( /**********************************************************//** Determines the size of a data tuple prefix in ROW_FORMAT=COMPACT. @return total size */ -UNIV_INLINE MY_ATTRIBUTE((warn_unused_result, nonnull(1,2))) +UNIV_INLINE MY_ATTRIBUTE((warn_unused_result)) ulint rec_get_converted_size_comp_prefix_low( /*===================================*/ @@ -846,8 +846,7 @@ rec_get_converted_size_comp_prefix_low( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = field->fixed_len; if (temp && fixed_len @@ -879,8 +878,7 @@ rec_get_converted_size_comp_prefix_low( ut_ad(col->len >= 256 || col->mtype == DATA_BLOB); extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { @@ -1276,16 +1274,12 @@ rec_convert_dtuple_to_rec_comp( *lens-- = (byte) (len >> 8) | 0xc0; *lens-- = (byte) len; } else { - ut_ad(len <= dtype_get_len(type) + - prtype_get_compression_extra( - dtype_get_prtype(type)) + ut_ad(len <= dtype_get_len(type) || dtype_get_mtype(type) == DATA_BLOB || !strcmp(index->name, FTS_INDEX_TABLE_IND_NAME)); if (len < 128 - || (dtype_get_len(type) < 256 - - prtype_get_compression_extra( - dtype_get_prtype(type)) + || (dtype_get_len(type) < 256 && dtype_get_mtype(type) != DATA_BLOB)) { *lens-- = (byte) len; diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index cb47d605623..13877b8fe08 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -58,6 +58,8 @@ tokenized doc string. The index has three "fields": integer value) 3) Word's position in original doc. +@see fts_create_one_index_table() + @return dict_index_t structure for the fts sort index */ UNIV_INTERN dict_index_t* @@ -99,16 +101,12 @@ row_merge_create_fts_sort_index( field->prefix_len = 0; field->col = static_cast<dict_col_t*>( mem_heap_alloc(new_index->heap, sizeof(dict_col_t))); - field->col->len = FTS_MAX_WORD_LEN; - - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - field->col->mtype = DATA_VARCHAR; - } else { - field->col->mtype = DATA_VARMYSQL; - } - field->col->prtype = idx_field->col->prtype | DATA_NOT_NULL; + field->col->mtype = charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL; field->col->mbminmaxlen = idx_field->col->mbminmaxlen; + field->col->len = HA_FT_MAXCHARLEN * DATA_MBMAXLEN(field->col->mbminmaxlen); + field->fixed_len = 0; /* Doc ID */ @@ -362,6 +360,7 @@ row_fts_free_pll_merge_buf( /*********************************************************************//** Tokenize incoming text data and add to the sort buffer. +@see row_merge_buf_encode() @return TRUE if the record passed, FALSE if out of space */ static ibool @@ -370,8 +369,6 @@ row_merge_fts_doc_tokenize( row_merge_buf_t** sort_buf, /*!< in/out: sort buffer */ doc_id_t doc_id, /*!< in: Doc ID */ fts_doc_t* doc, /*!< in: Doc to be tokenized */ - dtype_t* word_dtype, /*!< in: data structure for - word col */ merge_file_t** merge_file, /*!< in/out: merge file */ ibool opt_doc_id_size,/*!< in: whether to use 4 bytes instead of 8 bytes integer to @@ -403,7 +400,7 @@ row_merge_fts_doc_tokenize( ulint idx = 0; ib_uint32_t position; ulint offset = 0; - ulint cur_len = 0; + ulint cur_len; doc_id_t write_doc_id; inc = innobase_mysql_fts_get_token( @@ -457,14 +454,34 @@ row_merge_fts_doc_tokenize( dfield_set_data(field, t_str.f_str, t_str.f_len); len = dfield_get_len(field); - field->type.mtype = word_dtype->mtype; - field->type.prtype = word_dtype->prtype | DATA_NOT_NULL; + dict_col_copy_type(dict_index_get_nth_col(buf->index, 0), &field->type); + field->type.prtype |= DATA_NOT_NULL; + ut_ad(len <= field->type.len); - /* Variable length field, set to max size. */ - field->type.len = FTS_MAX_WORD_LEN; - field->type.mbminmaxlen = word_dtype->mbminmaxlen; + /* For the temporary file, row_merge_buf_encode() uses + 1 byte for representing the number of extra_size bytes. + This number will always be 1, because for this 3-field index + consisting of one variable-size column, extra_size will always + be 1 or 2, which can be encoded in one byte. + + The extra_size is 1 byte if the length of the + variable-length column is less than 128 bytes or the + maximum length is less than 256 bytes. */ + + /* One variable length column, word with its lenght less than + fts_max_token_size, add one extra size and one extra byte. + + Since the max length for FTS token now is larger than 255, + so we will need to signify length byte itself, so only 1 to 128 + bytes can be used for 1 bytes, larger than that 2 bytes. */ + if (len < 128 || field->type.len < 256) { + /* Extra size is one byte. */ + cur_len = 2 + len; + } else { + /* Extra size is two bytes. */ + cur_len = 3 + len; + } - cur_len += len; dfield_dup(field, buf->heap); field++; @@ -514,20 +531,6 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* One variable length column, word with its lenght less than - fts_max_token_size, add one extra size and one extra byte. - - Since the max length for FTS token now is larger than 255, - so we will need to signify length byte itself, so only 1 to 128 - bytes can be used for 1 bytes, larger than that 2 bytes. */ - if (t_str.f_len < 128) { - /* Extra size is one byte. */ - cur_len += 2; - } else { - /* Extra size is two bytes. */ - cur_len += 3; - } - /* Reserve one byte for the end marker of row_merge_block_t. */ if (buf->total_size + data_size[idx] + cur_len >= srv_sort_buf_size - 1) { @@ -620,8 +623,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dtype_t word_dtype; - dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; @@ -643,13 +644,6 @@ fts_parallel_tokenization( doc.charset = fts_index_get_charset( psort_info->psort_common->dup->index); - idx_field = dict_index_get_nth_field( - psort_info->psort_common->dup->index, 0); - word_dtype.prtype = idx_field->col->prtype; - word_dtype.mbminmaxlen = idx_field->col->mbminmaxlen; - word_dtype.mtype = (strcmp(doc.charset->name, "latin1_swedish_ci") == 0) - ? DATA_VARCHAR : DATA_VARMYSQL; - block = psort_info->merge_block; zip_size = dict_table_zip_size(table); @@ -699,7 +693,6 @@ loop: processed = row_merge_fts_doc_tokenize( buf, doc_item->doc_id, &doc, - &word_dtype, merge_file, psort_info->psort_common->opt_doc_id_size, &t_ctx); @@ -1258,10 +1251,9 @@ row_fts_build_sel_tree_level( int child_left; int child_right; ulint i; - ulint num_item; + ulint num_item = ulint(1) << level; - start = static_cast<ulint>((1 << level) - 1); - num_item = static_cast<ulint>(1 << level); + start = num_item - 1; for (i = 0; i < num_item; i++) { child_left = sel_tree[(start + i) * 2 + 1]; @@ -1330,7 +1322,7 @@ row_fts_build_sel_tree( treelevel++; } - start = (1 << treelevel) - 1; + start = (ulint(1) << treelevel) - 1; for (i = 0; i < (int) fts_sort_pll_degree; i++) { sel_tree[i + start] = i; diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index e1fef3d3716..23b9ba60269 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -573,8 +574,8 @@ AbstractCallback::init( } else if (!is_compressed_table() && m_page_size != UNIV_PAGE_SIZE) { ib_logf(IB_LOG_LEVEL_ERROR, - "Page size %lu of ibd file is not the same " - "as the server page size %lu", + "Page size " ULINTPF " of ibd file is not the same " + "as the server page size " ULINTPF, m_page_size, UNIV_PAGE_SIZE); return(DB_CORRUPTION); @@ -583,8 +584,8 @@ AbstractCallback::init( ib_logf(IB_LOG_LEVEL_ERROR, "File size " UINT64PF " is not a multiple " - "of the page size %lu", - (ib_uint64_t) file_size, (ulong) m_page_size); + "of the page size " ULINTPF, + (ib_uint64_t) file_size, m_page_size); return(DB_CORRUPTION); } @@ -725,8 +726,8 @@ FetchIndexRootPages::operator() ( if (block->page.offset * m_page_size != offset) { ib_logf(IB_LOG_LEVEL_ERROR, "Page offset doesn't match file offset: " - "page offset: %lu, file offset: %lu", - (ulint) block->page.offset, + "page offset: %u, file offset: " ULINTPF, + block->page.offset, (ulint) (offset / m_page_size)); err = DB_CORRUPTION; @@ -1159,10 +1160,9 @@ row_import::match_index_columns( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Index field count %lu doesn't match" - " tablespace metadata file value %lu", - (ulong) index->n_fields, - (ulong) cfg_index->m_n_fields); + "Index field count %u doesn't match" + " tablespace metadata file value " ULINTPF, + index->n_fields, cfg_index->m_n_fields); return(DB_ERROR); } @@ -1179,34 +1179,31 @@ row_import::match_index_columns( ER_TABLE_SCHEMA_MISMATCH, "Index field name %s doesn't match" " tablespace metadata field name %s" - " for field position %lu", - field->name, cfg_field->name, (ulong) i); + " for field position " ULINTPF, + field->name, cfg_field->name, i); err = DB_ERROR; } if (cfg_field->prefix_len != field->prefix_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s prefix len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->prefix_len, - (ulong) cfg_field->prefix_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s prefix len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->prefix_len, cfg_field->prefix_len); err = DB_ERROR; } if (cfg_field->fixed_len != field->fixed_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s fixed len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->fixed_len, - (ulong) cfg_field->fixed_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s fixed len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->fixed_len, + cfg_field->fixed_len); err = DB_ERROR; } @@ -1248,12 +1245,11 @@ row_import::match_table_columns( } else if (cfg_col_index != col->ind) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Column %s ordinal value mismatch, it's at " - "%lu in the table and %lu in the tablespace " - "meta-data file", - col_name, - (ulong) col->ind, (ulong) cfg_col_index); + ER_TABLE_SCHEMA_MISMATCH, + "Column %s ordinal value mismatch, it's at %u" + " in the table and " ULINTPF + " in the tablespace meta-data file", + col_name, col->ind, cfg_col_index); err = DB_ERROR; } else { @@ -1335,23 +1331,20 @@ row_import::match_schema( THD* thd) UNIV_NOTHROW { /* Do some simple checks. */ - const unsigned relevant_flags = m_flags & ~DICT_TF_MASK_DATA_DIR; - const unsigned relevant_table_flags - = m_table->flags & ~DICT_TF_MASK_DATA_DIR; - if (relevant_flags != relevant_table_flags) { + if ((m_table->flags ^ m_flags) & ~DICT_TF_MASK_DATA_DIR) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Table flags don't match, server table has 0x%x " - "and the meta-data file has 0x%x", - relevant_table_flags, relevant_flags); + "Table flags don't match, server table has 0x%x" + " and the meta-data file has 0x%lx", + m_table->flags, ulong(m_flags)); return(DB_ERROR); } else if (m_table->n_cols != m_n_cols) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of columns don't match, table has %lu " - "columns but the tablespace meta-data file has " - "%lu columns", - (ulong) m_table->n_cols, (ulong) m_n_cols); + "Number of columns don't match, table has %u " + "columns but the tablespace meta-data file has " + ULINTPF " columns", + m_table->n_cols, m_n_cols); return(DB_ERROR); } else if (UT_LIST_GET_LEN(m_table->indexes) != m_n_indexes) { @@ -1361,11 +1354,10 @@ row_import::match_schema( table matching the IMPORT definition. */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of indexes don't match, table has %lu " - "indexes but the tablespace meta-data file has " - "%lu indexes", - (ulong) UT_LIST_GET_LEN(m_table->indexes), - (ulong) m_n_indexes); + "Number of indexes don't match, table has " ULINTPF + " indexes but the tablespace meta-data file has " + ULINTPF " indexes", + UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); return(DB_ERROR); } @@ -1441,8 +1433,8 @@ row_import::set_root_by_heuristic() UNIV_NOTHROW table_name, sizeof(table_name), m_table->name, FALSE); ib_logf(IB_LOG_LEVEL_WARN, - "Table %s should have %lu indexes but the tablespace " - "has %lu indexes", + "Table %s should have " ULINTPF + " indexes but the tablespace has " ULINTPF " indexes", table_name, UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); @@ -1680,9 +1672,10 @@ PageConverter::adjust_cluster_index_blob_column( ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Externally stored column(%lu) has a reference " - "length of %lu in the cluster index %s", - (ulong) i, (ulong) len, index_name); + "Externally stored column(" ULINTPF + ") has a reference length of " ULINTPF + " in the cluster index %s", + i, len, index_name); return(DB_CORRUPTION); } @@ -2043,7 +2036,8 @@ PageConverter::update_page( return(err); } - ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (%lu)", page_type); + ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (" ULINTPF ")", + page_type); return(DB_CORRUPTION); } @@ -2077,7 +2071,8 @@ PageConverter::validate( if (checksum != 0) { /* Checksum check passed in buf_page_is_corrupted(). */ ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu checksum %lu should be zero.", + "%s: Page %lu checksum " ULINTPF + " should be zero.", m_filepath, (ulong) (offset / m_page_size), checksum); } @@ -2391,11 +2386,10 @@ row_import_adjust_root_pages_of_secondary_indexes( ib_errf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_INNODB_INDEX_CORRUPT, - "Index '%s' contains %lu entries, " - "should be %lu, you should recreate " + "Index '%s' contains " ULINTPF " entries, " + "should be " ULINTPF ", you should recreate " "this index.", index_name, - (ulong) purge.get_n_rows(), - (ulong) n_rows_in_table); + purge.get_n_rows(), n_rows_in_table); index->type |= DICT_CORRUPT; @@ -2746,7 +2740,7 @@ row_import_read_index_data( if (len > OS_FILE_MAX_PATH) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Index name length (%lu) is too long, " + "Index name length (" ULINTPF ") is too long, " "the meta-data is corrupt", len); return(DB_CORRUPTION); @@ -2827,8 +2821,8 @@ row_import_read_indexes( } else if (cfg->m_n_indexes > 1024) { // FIXME: What is the upper limit? */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Number of indexes in meta-data file is too high: %lu", - (ulong) cfg->m_n_indexes); + "Number of indexes in meta-data file is too high: " + ULINTPF, cfg->m_n_indexes); cfg->m_n_indexes = 0; return(DB_CORRUPTION); @@ -2926,8 +2920,8 @@ row_import_read_columns( if (len == 0 || len > 128) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Column name length %lu, is invalid", - (ulong) len); + "Column name length " ULINTPF ", is invalid", + len); return(DB_CORRUPTION); } @@ -3098,8 +3092,9 @@ row_import_read_v1( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, "Tablespace to be imported has a different " "page size than this server. Server page size " - "is %lu, whereas tablespace page size is %lu", - UNIV_PAGE_SIZE, (ulong) cfg->m_page_size); + "is " ULINTPF ", whereas tablespace page size is " + ULINTPF, + UNIV_PAGE_SIZE, cfg->m_page_size); return(DB_ERROR); } @@ -3164,8 +3159,8 @@ row_import_read_meta_data( return(row_import_read_v1(file, thd, &cfg)); default: ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Unsupported meta-data version number (%lu), " - "file ignored", (ulong) cfg.m_version); + "Unsupported meta-data version number (" ULINTPF "), " + "file ignored", cfg.m_version); } return(DB_ERROR); diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index e397053949e..5082aae15cf 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -95,7 +96,8 @@ row_merge_tuple_print( } ut_print_buf(f, dfield_get_data(field), len); if (len != field_len) { - fprintf(f, " (total %lu bytes)", field_len); + fprintf(f, " (total " ULINTPF " bytes)", + field_len); } } } @@ -529,8 +531,7 @@ row_merge_buf_add( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = ifield->fixed_len; if (fixed_len && !dict_table_is_comp(index->table) @@ -559,8 +560,7 @@ row_merge_buf_add( } else if (dfield_is_ext(field)) { extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { @@ -790,9 +790,9 @@ row_merge_buf_write( ut_ad(b < &block[srv_sort_buf_size]); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu %lu", - (void*) b, of->fd, (ulong) of->offset, - (ulong) i); + fprintf(stderr, "row_merge_buf_write %p,%d," + ULINTPF " " ULINTPF, + (void*) b, of->fd, of->offset, i); row_merge_tuple_print(stderr, entry, n_fields); } #endif /* UNIV_DEBUG */ @@ -809,8 +809,8 @@ row_merge_buf_write( #endif /* UNIV_DEBUG_VALGRIND */ #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu EOF\n", - (void*) b, of->fd, (ulong) of->offset); + fprintf(stderr, "row_merge_buf_write %p,%d," ULINTPF " EOF\n", + (void*) b, of->fd, of->offset); } #endif /* UNIV_DEBUG */ } @@ -866,15 +866,8 @@ row_merge_read( #ifdef UNIV_DEBUG if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); - } -#endif /* UNIV_DEBUG */ - -#ifdef UNIV_DEBUG - if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_read fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -917,8 +910,8 @@ row_merge_write( #ifdef UNIV_DEBUG if (row_merge_print_block_write) { - fprintf(stderr, "row_merge_write fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_write fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -953,14 +946,8 @@ row_merge_read_rec( ulint data_size; ulint avail_size; - ut_ad(block); - ut_ad(buf); ut_ad(b >= &block[0]); ut_ad(b < &block[srv_sort_buf_size]); - ut_ad(index); - ut_ad(foffs); - ut_ad(mrec); - ut_ad(offsets); ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); @@ -972,9 +959,10 @@ row_merge_read_rec( *mrec = NULL; #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu EOF\n", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF + " EOF\n", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); } #endif /* UNIV_DEBUG */ return(NULL); @@ -1089,9 +1077,9 @@ err_exit: func_exit: #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu ", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF " ", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); rec_print_comp(stderr, *mrec, offsets); putc('\n', stderr); } @@ -1125,8 +1113,8 @@ row_merge_write_rec_low( ut_ad(e == rec_offs_extra_size(offsets) + 1); if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%d,%lu ", - (void*) b, fd, (ulong) foffs); + fprintf(stderr, "row_merge_write %p,%d," ULINTPF " ", + (void*) b, fd, foffs); rec_print_comp(stderr, mrec, offsets); putc('\n', stderr); } @@ -1228,8 +1216,8 @@ row_merge_write_eof( ut_ad(foffs); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%p,%d,%lu EOF\n", - (void*) b, (void*) block, fd, (ulong) *foffs); + fprintf(stderr, "row_merge_write %p,%p,%d," ULINTPF " EOF\n", + (void*) b, (void*) block, fd, *foffs); } #endif /* UNIV_DEBUG */ @@ -2074,11 +2062,12 @@ row_merge_blocks( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks fd=%d ofs=%lu + fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) *foffs0, - file->fd, (ulong) *foffs1, - of->fd, (ulong) of->offset); + "row_merge_blocks fd=%d ofs=" ULINTPF + " + fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + file->fd, *foffs1, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ @@ -2177,10 +2166,10 @@ row_merge_blocks_copy( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks_copy fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) foffs0, - of->fd, (ulong) of->offset); + "row_merge_blocks_copy fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ @@ -3834,8 +3823,8 @@ wait_again: for (j = 0; j < FTS_NUM_AUX_INDEX; j++) { - os_thread_join(merge_info[j] - .thread_hdl); + os_thread_join(merge_info[j] + .thread_hdl); } } } else { diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 46bf523750c..c0c2b6fbabe 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -69,48 +70,6 @@ Created 9/17/2000 Heikki Tuuri /** Provide optional 4.x backwards compatibility for 5.0 and above */ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; -/** -Z_NO_COMPRESSION = 0 -Z_BEST_SPEED = 1 -Z_BEST_COMPRESSION = 9 -Z_DEFAULT_COMPRESSION = -1 -Compression level to be used by zlib for compressed-blob columns. -Settable by user. -*/ -UNIV_INTERN uint srv_compressed_columns_zip_level = DEFAULT_COMPRESSION_LEVEL; -/** -(Z_FILTERED | Z_HUFFMAN_ONLY | Z_RLE | Z_FIXED | Z_DEFAULT_STRATEGY) - -The strategy parameter is used to tune the compression algorithm. Use the -value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a -filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only -(no string match), or Z_RLE to limit match distances to one -(run-length encoding). Filtered data consists mostly of small values with a -somewhat random distribution. In this case, the compression algorithm is -tuned to compress them better. -The effect of Z_FILTERED is to force more Huffman coding and less string -matching; it is somewhat intermediate between Z_DEFAULT_STRATEGY and -Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as Z_HUFFMAN_ONLY, -but give better compression for PNG image data. The strategy parameter only -affects the compression ratio but not the correctness of the compressed -output even if it is not set appropriately. Z_FIXED prevents the use of -dynamic Huffman codes, allowing for a simpler decoder for special -applications. -*/ -const uint srv_compressed_columns_zlib_strategy = Z_DEFAULT_STRATEGY; -/** Compress the column if the data length exceeds this value. */ -UNIV_INTERN ulong srv_compressed_columns_threshold = 96; -/** -Determine if zlib needs to compute adler32 value for the compressed data. -This variables is similar to page_zip_zlib_wrap, but only used by -compressed blob columns. -*/ -const bool srv_compressed_columns_zlib_wrap = true; -/** -Determine if zlib will use custom memory allocation functions based on -InnoDB memory heap routines (mem_heap_t*). -*/ -const bool srv_compressed_columns_zlib_use_heap = false; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ char* table_name; /*!< table name */ @@ -214,17 +173,6 @@ row_mysql_prebuilt_free_blob_heap( prebuilt->blob_heap = NULL; } -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt) /*!< in: prebuilt struct of a - ha_innobase:: table handle */ -{ - mem_heap_free(prebuilt->compress_heap); - prebuilt->compress_heap = NULL; -} - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -281,425 +229,6 @@ row_mysql_read_true_varchar( return(field + 1); } -/** - Compressed BLOB header format: - --------------------------------------------------------------- - | reserved | wrap | algorithm | len-len | compressed | unused | - | [1] | [1] | [5] | [3] | [1] | [5] | - --------------------------------------------------------------- - | 0 0 | 1 1 | 2 6 | 7 9 | 10 10 | 11 15 | - --------------------------------------------------------------- - * 'reserved' bit is planned to be used in future versions of the BLOB - header. In this version it must always be - 'default_zip_column_reserved_value' (0). - * 'wrap' identifies if compression algorithm calculated a checksum - (adler32 in case of zlib) and appended it to the compressed data. - * 'algorithm' identifies which algoritm was used to compress this BLOB. - Currently, the only value 'default_zip_column_algorithm_value' (0) is - supported. - * 'len-len' field identifies the length of the column length data portion - followed by this header (see below). - * If 'compressed' bit is set to 1, then this header is immediately followed - by 1..8 bytes (depending on the value of 'len-len' bitfield) which - determine original (uncompressed) block size. These 'len-len' bytes are - followed by compressed representation of the original data. - * If 'compressed' bit is set to 0, every other bitfield ('wrap', - 'algorithm' and 'le-len') must be ignored. In this case the header is - immediately followed by uncompressed (original) data. -*/ - -/** - Currently the only supported value for the 'reserved' field is - false (0). -*/ -static const bool default_zip_column_reserved_value = false; - -/** - Currently the only supported value for the 'algorithm' field is 0, which - means 'zlib'. -*/ -static const uint default_zip_column_algorithm_value = 0; - -static const size_t zip_column_prefix_max_length = - ZIP_COLUMN_HEADER_LENGTH + 8; -static const size_t zip_column_header_length = ZIP_COLUMN_HEADER_LENGTH; - -/* 'reserved', bit 0 */ -static const uint zip_column_reserved = 0; -/* 0000 0000 0000 0001 */ -static const uint zip_column_reserved_mask = 0x0001; - -/* 'wrap', bit 1 */ -static const uint zip_column_wrap = 1; -/* 0000 0000 0000 0010 */ -static const uint zip_column_wrap_mask = 0x0002; - -/* 'algorithm', bit 2,3,4,5,6 */ -static const uint zip_column_algorithm = 2; -/* 0000 0000 0111 1100 */ -static const uint zip_column_algorithm_mask = 0x007C; - -/* 'len-len', bit 7,8,9 */ -static const uint zip_column_data_length = 7; -/* 0000 0011 1000 0000 */ -static const uint zip_column_data_length_mask = 0x0380; - -/* 'compressed', bit 10 */ -static const uint zip_column_compressed = 10; -/* 0000 0100 0000 0000 */ -static const uint zip_column_compressed_mask = 0x0400; - -/** Updates compressed block header with the given components */ -static void -column_set_compress_header( - byte* data, - bool compressed, - ulint lenlen, - uint alg, - bool wrap, - bool reserved) -{ - ulint header = 0; - header |= (compressed << zip_column_compressed); - header |= (lenlen << zip_column_data_length); - header |= (alg << zip_column_algorithm); - header |= (wrap << zip_column_wrap); - header |= (reserved << zip_column_reserved); - mach_write_to_2(data, header); -} - -/** Parse compressed block header into components */ -static void -column_get_compress_header( - const byte* data, - bool* compressed, - ulint* lenlen, - uint* alg, - bool* wrap, - bool* reserved -) -{ - ulint header = mach_read_from_2(data); - *compressed = ((header & zip_column_compressed_mask) >> - zip_column_compressed); - *lenlen = ((header & zip_column_data_length_mask) >> - zip_column_data_length); - *alg = ((header & zip_column_algorithm_mask) >> - zip_column_algorithm); - *wrap = ((header & zip_column_wrap_mask) >> - zip_column_wrap); - *reserved = ((header & zip_column_reserved_mask) >> - zip_column_reserved); -} - -/** Allocate memory for zlib. */ -static -void* -column_zip_zalloc( - void* opaque, /*!< in/out: memory heap */ - uInt items, /*!< in: number of items to allocate */ - uInt size) /*!< in: size of an item in bytes */ -{ - return(mem_heap_zalloc(static_cast<mem_heap_t*>(opaque), - items * size)); -} - -/** Deallocate memory for zlib. */ -static -void -column_zip_free( - void* opaque MY_ATTRIBUTE((unused)), /*!< in: memory heap */ - void* address MY_ATTRIBUTE((unused))) /*!< in: object to free */ -{ -} - -/** Configure the zlib allocator to use the given memory heap. */ -UNIV_INTERN -void -column_zip_set_alloc( - void* stream, /*!< in/out: zlib stream */ - mem_heap_t* heap) /*!< in: memory heap to use */ -{ - z_stream* strm = static_cast<z_stream*>(stream); - - if (srv_compressed_columns_zlib_use_heap) { - strm->zalloc = column_zip_zalloc; - strm->zfree = column_zip_free; - strm->opaque = heap; - } else { - strm->zalloc = (alloc_func)0; - strm->zfree = (free_func)0; - strm->opaque = (voidpf)0; - } -} - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - int err = 0; - ulint comp_len = *len; - ulint buf_len = *len + zip_column_prefix_max_length; - byte* buf; - byte* ptr; - z_stream c_stream; - bool wrap = srv_compressed_columns_zlib_wrap; - - int window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, buf_len)); - } - - buf = static_cast<byte*>(mem_heap_zalloc( - prebuilt->compress_heap,buf_len)); - - if (*len < srv_compressed_columns_threshold || - srv_compressed_columns_zip_level == Z_NO_COMPRESSION) - goto do_not_compress; - - ptr = buf + zip_column_header_length + lenlen; - - /*init deflate object*/ - c_stream.next_in = const_cast<Bytef*>(data); - c_stream.avail_in = *len; - c_stream.next_out = ptr; - c_stream.avail_out = comp_len; - - column_zip_set_alloc(&c_stream, prebuilt->compress_heap); - - err = deflateInit2(&c_stream, srv_compressed_columns_zip_level, - Z_DEFLATED, window_bits, MAX_MEM_LEVEL, - srv_compressed_columns_zlib_strategy); - ut_a(err == Z_OK); - - if (dict_data != 0 && dict_data_len != 0) { - err = deflateSetDictionary(&c_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - } - - err = deflate(&c_stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&c_stream); - if (err == Z_OK) - err = Z_BUF_ERROR; - } else { - comp_len = c_stream.total_out; - err = deflateEnd(&c_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - /* data after compress is larger than uncompressed data*/ - break; - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "failed to compress the column, error: %d\n", err); - } - - /* make sure the compressed data size is smaller than - uncompressed data */ - if (err == Z_OK && - *len > (comp_len + zip_column_header_length + lenlen)) { - column_set_compress_header(buf, true, lenlen - 1, - default_zip_column_algorithm_value, wrap, - default_zip_column_reserved_value); - ptr = buf + zip_column_header_length; - /*store the uncompressed data length*/ - switch (lenlen) { - case 1: - mach_write_to_1(ptr, *len); - break; - case 2: - mach_write_to_2(ptr, *len); - break; - case 3: - mach_write_to_3(ptr, *len); - break; - case 4: - mach_write_to_4(ptr, *len); - break; - default: - ut_error; - } - - *len = comp_len + zip_column_header_length + lenlen; - return buf; - } - -do_not_compress: - ptr = buf; - column_set_compress_header(ptr, false, 0, - default_zip_column_algorithm_value, false, - default_zip_column_reserved_value); - ptr += zip_column_header_length; - memcpy(ptr, data, *len); - *len += zip_column_header_length; - return buf; -} - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - ulint buf_len = 0; - byte* buf; - int err = 0; - int window_bits = 0; - z_stream d_stream; - bool is_compressed = false; - bool wrap = false; - bool reserved = false; - ulint lenlen = 0; - uint alg = 0; - - ut_ad(*len != ULINT_UNDEFINED); - ut_ad(*len >= zip_column_header_length); - - column_get_compress_header(data, &is_compressed, &lenlen, &alg, - &wrap, &reserved); - - if (reserved != default_zip_column_reserved_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported compressed BLOB header format\n"); - } - - if (alg != default_zip_column_algorithm_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported 'algorithm' value in the" - " compressed BLOB header\n"); - } - - ut_a(lenlen < 4); - - data += zip_column_header_length; - if (!is_compressed) { /* column not compressed */ - *len -= zip_column_header_length; - return data; - } - - lenlen++; - - ulint comp_len = *len - zip_column_header_length - lenlen; - - ulint uncomp_len = 0; - switch (lenlen) { - case 1: - uncomp_len = mach_read_from_1(data); - break; - case 2: - uncomp_len = mach_read_from_2(data); - break; - case 3: - uncomp_len = mach_read_from_3(data); - break; - case 4: - uncomp_len = mach_read_from_4(data); - break; - default: - ut_error; - } - - data += lenlen; - - /* data is compressed, decompress it*/ - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, uncomp_len)); - } - - buf_len = uncomp_len; - buf = static_cast<byte*>(mem_heap_zalloc( - prebuilt->compress_heap, buf_len)); - - /* init d_stream */ - d_stream.next_in = const_cast<Bytef*>(data); - d_stream.avail_in = comp_len; - d_stream.next_out = buf; - d_stream.avail_out = buf_len; - - column_zip_set_alloc(&d_stream, prebuilt->compress_heap); - - window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - err = inflateInit2(&d_stream, window_bits); - ut_a(err == Z_OK); - - err = inflate(&d_stream, Z_FINISH); - if (err == Z_NEED_DICT) { - ut_a(dict_data != 0 && dict_data_len != 0); - err = inflateSetDictionary(&d_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - err = inflate(&d_stream, Z_FINISH); - } - - if (err != Z_STREAM_END) { - inflateEnd(&d_stream); - if (err == Z_BUF_ERROR && d_stream.avail_in == 0) - err = Z_DATA_ERROR; - } else { - buf_len = d_stream.total_out; - err = inflateEnd(&d_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - ib_logf(IB_LOG_LEVEL_FATAL, - "zlib buf error, this shouldn't happen\n"); - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress column, error: %d\n", err); - } - - if (err == Z_OK) { - if (buf_len != uncomp_len) { - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress blob column, may" - " be corrupted\n"); - } - *len = buf_len; - return buf; - } - - *len -= (zip_column_header_length + lenlen); - return data; -} - - /*******************************************************************//** Stores a reference to a BLOB in the MySQL format. */ UNIV_INTERN @@ -713,21 +242,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len) /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt) - /*<! in: use prebuilt->compress_heap only - here */ { /* MySQL might assume the field is set to zero except the length and the pointer fields */ @@ -739,27 +257,11 @@ row_mysql_store_blob_ref( In 32-bit architectures we only use the first 4 bytes of the pointer slot. */ - ut_a(col_len - 8 > 1 || - len < 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 2 || - len < 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 3 || - len < 256 * 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - - const byte *ptr = NULL; - - if (need_decompression) - ptr = row_decompress_column((const byte*)data, &len, - dict_data, dict_data_len, prebuilt); - - if (ptr) - memcpy(dest + col_len - 8, &ptr, sizeof ptr); - else - memcpy(dest + col_len - 8, &data, sizeof data); + ut_a(col_len - 8 > 1 || len < 256); + ut_a(col_len - 8 > 2 || len < 256 * 256); + ut_a(col_len - 8 > 3 || len < 256 * 256 * 256); + memcpy(dest + col_len - 8, &data, sizeof data); mach_write_to_n_little_endian(dest, col_len - 8, len); } @@ -773,32 +275,15 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len) /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ { byte* data = NULL; - byte* ptr = NULL; *len = mach_read_from_n_little_endian(ref, col_len - 8); memcpy(&data, ref + col_len - 8, sizeof data); - if (need_compression) { - ptr = row_compress_column(data, len, col_len - 8, dict_data, - dict_data_len, prebuilt); - if (ptr) - data = ptr; - } - return(data); } @@ -881,16 +366,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ + ulint comp) /*!< in: nonzero=compact format */ { const byte* ptr = mysql_data; const dtype_t* dtype; @@ -943,14 +419,8 @@ row_mysql_store_col_in_innobase_format( lenlen = 2; } - const byte* tmp_ptr = row_mysql_read_true_varchar( + ptr = row_mysql_read_true_varchar( &col_len, mysql_data, lenlen); - if (need_compression) - ptr = row_compress_column(tmp_ptr, &col_len, - lenlen, dict_data, dict_data_len, - prebuilt); - else - ptr = tmp_ptr; } else { /* Remove trailing spaces from old style VARCHAR columns. */ @@ -1032,9 +502,7 @@ row_mysql_store_col_in_innobase_format( } } else if (type == DATA_BLOB && row_format_col) { - ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len, - need_compression, dict_data, dict_data_len, - prebuilt); + ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len); } dfield_set_data(dfield, ptr, col_len); @@ -1092,11 +560,7 @@ row_mysql_convert_row_to_innobase( TRUE, /* MySQL row format data */ mysql_rec + templ->mysql_col_offset, templ->mysql_col_len, - dict_table_is_comp(prebuilt->table), - templ->compressed, - reinterpret_cast<const byte*>( - templ->zip_dict_data.str), - templ->zip_dict_data.length, prebuilt); + dict_table_is_comp(prebuilt->table)); next_column: ; } @@ -1442,10 +906,6 @@ row_prebuilt_free( mem_heap_free(prebuilt->blob_heap); } - if (prebuilt->compress_heap) { - mem_heap_free(prebuilt->compress_heap); - } - if (prebuilt->old_vers_heap) { mem_heap_free(prebuilt->old_vers_heap); } @@ -1875,9 +1335,6 @@ row_insert_for_mysql( return(DB_READ_ONLY); } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - trx->op_info = "inserting"; row_mysql_delay_if_needed(); @@ -1908,6 +1365,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { @@ -1993,11 +1452,10 @@ error_exit: que_thr_stop_for_mysql_no_error(thr, trx); if (UNIV_LIKELY(!(trx->fake_changes))) { - if (table->is_system_db) { - srv_stats.n_system_rows_inserted.add((size_t)trx->id, 1); + srv_stats.n_system_rows_inserted.inc(size_t(trx->id)); } else { - srv_stats.n_rows_inserted.add((size_t)trx->id, 1); + srv_stats.n_rows_inserted.inc(size_t(trx->id)); } if (prebuilt->clust_index_was_generated) { @@ -2399,17 +1857,15 @@ run_again: dict_table_n_rows_dec(prebuilt->table); if (table->is_system_db) { - srv_stats.n_system_rows_deleted.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_deleted.inc(size_t(trx->id)); } else { - srv_stats.n_rows_deleted.add((size_t)trx->id, 1); + srv_stats.n_rows_deleted.inc(size_t(trx->id)); } } else { if (table->is_system_db) { - srv_stats.n_system_rows_updated.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_updated.inc(size_t(trx->id)); } else { - srv_stats.n_rows_updated.add((size_t)trx->id, 1); + srv_stats.n_rows_updated.inc(size_t(trx->id)); } } @@ -2647,17 +2103,15 @@ run_again: dict_table_n_rows_dec(table); if (table->is_system_db) { - srv_stats.n_system_rows_deleted.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_deleted.inc(size_t(trx->id)); } else { - srv_stats.n_rows_deleted.add((size_t)trx->id, 1); + srv_stats.n_rows_deleted.inc(size_t(trx->id)); } } else { if (table->is_system_db) { - srv_stats.n_system_rows_updated.add( - (size_t)trx->id, 1); + srv_stats.n_system_rows_updated.inc(size_t(trx->id)); } else { - srv_stats.n_rows_updated.add((size_t)trx->id, 1); + srv_stats.n_rows_updated.inc(size_t(trx->id)); } } @@ -3443,8 +2897,6 @@ row_mysql_table_id_reassign( " WHERE TABLE_ID = :old_id;\n" "UPDATE SYS_INDEXES SET TABLE_ID = :new_id\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n", FALSE, trx); return(err); @@ -4228,9 +3680,6 @@ next_rec: "UPDATE SYS_INDEXES" " SET TABLE_ID = :new_id, SPACE = :new_space\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS\n" - " SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n" , FALSE, trx); @@ -4858,19 +4307,6 @@ row_drop_table_for_mysql( filepath = fil_make_ibd_name(tablename, false); } - /* Remove all compression dictionary references for the - table */ - err = dict_create_remove_zip_dict_references_for_table( - table->id, trx); - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, "Error: (%s) not " - "able to remove compression dictionary " - "references for table %s", ut_strerr(err), - tablename); - - goto funct_exit; - } - if (dict_table_has_fts_index(table) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) { ut_ad(table->n_ref_count == 0); diff --git a/storage/xtradb/row/row0purge.cc b/storage/xtradb/row/row0purge.cc index bc2e0b0e1cb..35b3520749b 100644 --- a/storage/xtradb/row/row0purge.cc +++ b/storage/xtradb/row/row0purge.cc @@ -897,7 +897,7 @@ row_purge_record_func( Fetches an undo log record and does the purge for the recorded operation. If none left, or the current purge completed, returns the control to the parent node, which is always a query thread node. */ -static MY_ATTRIBUTE((nonnull)) +static void row_purge( /*======*/ diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index ad1e9e2bf9d..fd23d83e0e5 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2459,8 +2460,7 @@ row_sel_convert_mysql_key_to_innobase( /* MySQL key value format col */ FALSE, key_ptr + data_offset, data_len, - dict_table_is_comp(index->table), - false, 0, 0 ,0); + dict_table_is_comp(index->table)); ut_a(buf <= original_buf + buf_len); } @@ -2554,15 +2554,15 @@ row_sel_store_row_id_to_prebuilt( #ifdef UNIV_DEBUG /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,idx,field,src,len, prebuilt) + (dest,templ,idx,field,src,len) #else /* UNIV_DEBUG */ /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,src,len, prebuilt) + (dest,templ,src,len) #endif /* UNIV_DEBUG */ /**************************************************************//** @@ -2592,10 +2592,7 @@ row_sel_field_store_in_mysql_format_func( templ->icp_rec_field_no */ #endif /* UNIV_DEBUG */ const byte* data, /*!< in: data to store */ - ulint len, /*!< in: length of the data */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap - only here */ + ulint len) /*!< in: length of the data */ { byte* ptr; #ifdef UNIV_DEBUG @@ -2639,15 +2636,6 @@ row_sel_field_store_in_mysql_format_func( field_end = dest + templ->mysql_col_len; if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) { - /* If this is a compressed column, - decompress it first */ - if (templ->compressed) - data = row_decompress_column(data, &len, - reinterpret_cast<const byte*>( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2698,11 +2686,7 @@ row_sel_field_store_in_mysql_format_func( already copied to the buffer in row_sel_store_mysql_rec */ row_mysql_store_blob_ref(dest, templ->mysql_col_len, data, - len, templ->compressed, - reinterpret_cast<const byte*>( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); + len); break; case DATA_MYSQL: @@ -2855,7 +2839,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); if (heap != prebuilt->blob_heap) { mem_heap_free(heap); @@ -2905,7 +2889,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); } ut_ad(len != UNIV_SQL_NULL); @@ -2953,9 +2937,6 @@ row_sel_store_mysql_rec( prebuilt->blob_heap = NULL; } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - for (i = 0; i < prebuilt->n_template; i++) { const mysql_row_templ_t*templ = &prebuilt->mysql_template[i]; const ulint field_no diff --git a/storage/xtradb/row/row0upd.cc b/storage/xtradb/row/row0upd.cc index f346adde850..66fe55bbcce 100644 --- a/storage/xtradb/row/row0upd.cc +++ b/storage/xtradb/row/row0upd.cc @@ -1116,8 +1116,6 @@ row_upd_index_replace_new_col_vals_index_pos( ulint n_fields; const ulint zip_size = dict_table_zip_size(index->table); - ut_ad(index); - dtuple_set_info_bits(entry, update->info_bits); if (order_only) { @@ -1302,8 +1300,6 @@ row_upd_changes_ord_field_binary_func( ulint i; const dict_index_t* clust_index; - ut_ad(index); - ut_ad(update); ut_ad(thr); ut_ad(thr->graph); ut_ad(thr->graph->trx); diff --git a/storage/xtradb/srv/srv0conc.cc b/storage/xtradb/srv/srv0conc.cc index 63268e3a266..eac1b52a64f 100644 --- a/storage/xtradb/srv/srv0conc.cc +++ b/storage/xtradb/srv/srv0conc.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -78,7 +79,9 @@ typedef UT_LIST_NODE_T(struct srv_conc_slot_t) srv_conc_node_t; /** Slot for a thread waiting in the concurrency control queue. */ struct srv_conc_slot_t{ - os_event_t event; /*!< event to wait */ + os_event_t event; /*!< event to wait for; + os_event_set() and os_event_reset() + are protected by srv_conc_mutex */ ibool reserved; /*!< TRUE if slot reserved */ ibool wait_ended; /*!< TRUE when another thread has @@ -345,11 +348,11 @@ srv_conc_exit_innodb_without_atomics( } } - os_fast_mutex_unlock(&srv_conc_mutex); - if (slot != NULL) { os_event_set(slot->event); } + + os_fast_mutex_unlock(&srv_conc_mutex); } /*********************************************************************//** diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index 4a709160ea6..ea21a4c1454 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -2,6 +2,7 @@ Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -643,11 +644,11 @@ static monitor_info_t innodb_counter_info[] = MONITOR_DEFAULT_START, MONITOR_OVLD_OS_FSYNC}, {"os_pending_reads", "os", "Number of reads pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_READS}, {"os_pending_writes", "os", "Number of writes pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_WRITES}, {"os_log_bytes_written", "os", diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 5d043e4a57e..5b3f86d8641 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -74,12 +74,6 @@ Created 10/8/1995 Heikki Tuuri #include "mysql/plugin.h" #include "mysql/service_thd_wait.h" -/* prototypes of new functions added to ha_innodb.cc for kill_idle_transaction */ -ibool innobase_thd_is_idle(const void* thd); -ib_int64_t innobase_thd_get_start_time(const void* thd); -void innobase_thd_kill(ulong thd_id); -ulong innobase_thd_get_thread_id(const void* thd); - /* prototypes for new functions added to ha_innodb.cc */ ibool innobase_get_slow_log(); @@ -426,11 +420,6 @@ starting from SRV_FORCE_IGNORE_CORRUPT, so that data can be recovered by SELECT or mysqldump. When this is nonzero, we do not allow any user modifications to the data. */ UNIV_INTERN ulong srv_force_recovery; -#ifndef DBUG_OFF -/** Inject a crash at different steps of the recovery process. -This is for testing and debugging only. */ -UNIV_INTERN ulong srv_force_recovery_crash; -#endif /* !DBUG_OFF */ /** Print all user-level transactions deadlocks to mysqld stderr */ @@ -459,6 +448,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ UNIV_INTERN unsigned long long srv_stats_transient_sample_pages = 8; UNIV_INTERN my_bool srv_stats_persistent = TRUE; +UNIV_INTERN my_bool srv_stats_include_delete_marked = FALSE; UNIV_INTERN unsigned long long srv_stats_persistent_sample_pages = 20; UNIV_INTERN my_bool srv_stats_auto_recalc = TRUE; @@ -553,7 +543,7 @@ UNIV_INTERN const char* srv_io_thread_function[SRV_MAX_N_IO_THREADS]; UNIV_INTERN time_t srv_last_monitor_time; -UNIV_INTERN ib_mutex_t srv_innodb_monitor_mutex; +static ib_mutex_t srv_innodb_monitor_mutex; /* Mutex for locking srv_monitor_file. Not created if srv_read_only_mode */ UNIV_INTERN ib_mutex_t srv_monitor_file_mutex; @@ -733,7 +723,11 @@ struct srv_sys_t{ ulint n_sys_threads; /*!< size of the sys_threads array */ - srv_slot_t* sys_threads; /*!< server thread table */ + srv_slot_t* sys_threads; /*!< server thread table; + os_event_set() and + os_event_reset() on + sys_threads[]->event are + covered by srv_sys_t::mutex */ ulint n_threads_active[SRV_MASTER + 1]; /*!< number of threads active @@ -755,13 +749,16 @@ UNIV_INTERN ib_mutex_t server_mutex; static srv_sys_t* srv_sys = NULL; -/** Event to signal the monitor thread. */ +/** Event to signal srv_monitor_thread. Not protected by a mutex. +Set after setting srv_print_innodb_monitor. */ UNIV_INTERN os_event_t srv_monitor_event; -/** Event to signal the error thread */ +/** Event to signal the shutdown of srv_error_monitor_thread. +Not protected by a mutex. */ UNIV_INTERN os_event_t srv_error_event; -/** Event to signal the buffer pool dump/load thread */ +/** Event for waking up buf_dump_thread. Not protected by a mutex. +Set on shutdown or by buf_dump_start() or buf_load_start(). */ UNIV_INTERN os_event_t srv_buf_dump_event; /** The buffer pool dump/load file name */ @@ -931,7 +928,6 @@ srv_suspend_thread_low( /*===================*/ srv_slot_t* slot) /*!< in/out: thread slot */ { - ut_ad(!srv_read_only_mode); ut_ad(srv_sys_mutex_own()); @@ -989,34 +985,71 @@ srv_suspend_thread( return(sig_count); } -/*********************************************************************//** -Releases threads of the type given from suspension in the thread table. -NOTE! The server mutex has to be reserved by the caller! -@return number of threads released: this may be less than n if not - enough threads were suspended at the moment. */ -UNIV_INTERN -ulint -srv_release_threads( -/*================*/ - srv_thread_type type, /*!< in: thread type */ - ulint n) /*!< in: number of threads to release */ +/** Resume the calling thread. +@param[in,out] slot thread slot +@param[in] sig_count signal count (if wait) +@param[in] wait whether to wait for the event +@param[in] timeout_usec timeout in microseconds (0=infinite) +@return whether the wait timed out */ +static +bool +srv_resume_thread(srv_slot_t* slot, ib_int64_t sig_count = 0, bool wait = true, + ulint timeout_usec = 0) +{ + bool timeout; + + ut_ad(!srv_read_only_mode); + ut_ad(slot->in_use); + ut_ad(slot->suspended); + + if (!wait) { + timeout = false; + } else if (timeout_usec) { + timeout = OS_SYNC_TIME_EXCEEDED == os_event_wait_time_low( + slot->event, timeout_usec, sig_count); + } else { + timeout = false; + os_event_wait_low(slot->event, sig_count); + } + + srv_sys_mutex_enter(); + ut_ad(slot->in_use); + ut_ad(slot->suspended); + + slot->suspended = FALSE; + ++srv_sys->n_threads_active[slot->type]; + srv_sys_mutex_exit(); + return(timeout); +} + +/** Ensure that a given number of threads of the type given are running +(or are already terminated). +@param[in] type thread type +@param[in] n number of threads that have to run */ +void +srv_release_threads(enum srv_thread_type type, ulint n) { - ulint i; - ulint count = 0; + ulint running; ut_ad(srv_thread_type_validate(type)); ut_ad(n > 0); - srv_sys_mutex_enter(); + do { + running = 0; - for (i = 0; i < srv_sys->n_sys_threads; i++) { - srv_slot_t* slot; + srv_sys_mutex_enter(); - slot = &srv_sys->sys_threads[i]; + for (ulint i = 0; i < srv_sys->n_sys_threads; i++) { + srv_slot_t* slot = &srv_sys->sys_threads[i]; - if (slot->in_use - && srv_slot_get_type(slot) == type - && slot->suspended) { + if (!slot->in_use || srv_slot_get_type(slot) != type) { + continue; + } else if (!slot->suspended) { + if (++running >= n) { + break; + } + continue; + } switch (type) { case SRV_NONE: @@ -1046,21 +1079,11 @@ srv_release_threads( break; } - slot->suspended = FALSE; - - ++srv_sys->n_threads_active[type]; - os_event_set(slot->event); - - if (++count == n) { - break; - } } - } - srv_sys_mutex_exit(); - - return(count); + srv_sys_mutex_exit(); + } while (running && running < n); } /*********************************************************************//** @@ -1073,11 +1096,8 @@ srv_free_slot( { srv_sys_mutex_enter(); - if (!slot->suspended) { - /* Mark the thread as inactive. */ - srv_suspend_thread_low(slot); - } - + /* Mark the thread as inactive. */ + srv_suspend_thread_low(slot); /* Free the slot for reuse. */ ut_ad(slot->in_use); slot->in_use = FALSE; @@ -1189,10 +1209,15 @@ srv_free(void) os_event_free(srv_sys->sys_threads[i].event); os_event_free(srv_error_event); + srv_error_event = NULL; os_event_free(srv_monitor_event); + srv_monitor_event = NULL; os_event_free(srv_buf_dump_event); + srv_buf_dump_event = NULL; os_event_free(srv_checkpoint_completed_event); + srv_checkpoint_completed_event = NULL; os_event_free(srv_redo_log_tracked_event); + srv_redo_log_tracked_event = NULL; mutex_free(&srv_sys->mutex); mutex_free(&srv_sys->tasks_mutex); } @@ -1376,22 +1401,26 @@ srv_printf_innodb_monitor( low level 135. Therefore we can reserve the latter mutex here without a danger of a deadlock of threads. */ - mutex_enter(&dict_foreign_err_mutex); + if (!recv_recovery_on) { - if (!srv_read_only_mode && ftell(dict_foreign_err_file) != 0L) { - fputs("------------------------\n" - "LATEST FOREIGN KEY ERROR\n" - "------------------------\n", file); - ut_copy_file(file, dict_foreign_err_file); - } + mutex_enter(&dict_foreign_err_mutex); - mutex_exit(&dict_foreign_err_mutex); + if (!srv_read_only_mode + && ftell(dict_foreign_err_file) != 0L) { + fputs("------------------------\n" + "LATEST FOREIGN KEY ERROR\n" + "------------------------\n", file); + ut_copy_file(file, dict_foreign_err_file); + } + + mutex_exit(&dict_foreign_err_mutex); + } /* Only if lock_print_info_summary proceeds correctly, before we call the lock_print_info_all_transactions to print all the lock information. IMPORTANT NOTE: This function acquires the lock mutex on success. */ - ret = lock_print_info_summary(file, nowait); + ret = recv_recovery_on ? FALSE : lock_print_info_summary(file, nowait); if (ret) { if (trx_start_pos) { @@ -1424,10 +1453,13 @@ srv_printf_innodb_monitor( "--------\n", file); os_aio_print(file); - fputs("-------------------------------------\n" - "INSERT BUFFER AND ADAPTIVE HASH INDEX\n" - "-------------------------------------\n", file); - ibuf_print(file); + if (!recv_recovery_on) { + + fputs("-------------------------------------\n" + "INSERT BUFFER AND ADAPTIVE HASH INDEX\n" + "-------------------------------------\n", file); + ibuf_print(file); + } fprintf(file, @@ -1439,10 +1471,13 @@ srv_printf_innodb_monitor( btr_cur_n_sea_old = btr_cur_n_sea; btr_cur_n_non_sea_old = btr_cur_n_non_sea; - fputs("---\n" - "LOG\n" - "---\n", file); - log_print(file); + if (!recv_recovery_on) { + + fputs("---\n" + "LOG\n" + "---\n", file); + log_print(file); + } fputs("----------------------\n" "BUFFER POOL AND MEMORY\n" @@ -1537,8 +1572,9 @@ srv_printf_innodb_monitor( ? (recv_sys->addr_hash->n_cells * sizeof(hash_cell_t)) : 0), recv_sys_subtotal); + fprintf(file, "Dictionary memory allocated " ULINTPF "\n", - dict_sys->size); + dict_sys ? dict_sys->size : 0); buf_print_io(file); @@ -1646,6 +1682,10 @@ srv_printf_innodb_monitor( mutex_exit(&srv_innodb_monitor_mutex); fflush(file); +#ifndef DBUG_OFF + srv_debug_monitor_printed = true; +#endif + return(ret); } @@ -1695,10 +1735,10 @@ srv_export_innodb_status(void) mutex_enter(&srv_innodb_monitor_mutex); export_vars.innodb_data_pending_reads = - os_n_pending_reads; + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); export_vars.innodb_data_pending_writes = - os_n_pending_writes; + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); export_vars.innodb_data_pending_fsyncs = fil_n_pending_log_flushes @@ -1956,6 +1996,12 @@ srv_export_innodb_status(void) mutex_exit(&srv_innodb_monitor_mutex); } +#ifndef DBUG_OFF +/** false before InnoDB monitor has been printed at least once, true +afterwards */ +bool srv_debug_monitor_printed = false; +#endif + /*********************************************************************//** A thread which prints the info output by various InnoDB monitors. @return a dummy parameter */ @@ -2227,36 +2273,6 @@ loop: old_sema = sema; } - if (srv_kill_idle_transaction && trx_sys) { - trx_t* trx; - time_t now; -rescan_idle: - now = time(NULL); - mutex_enter(&trx_sys->mutex); - trx = UT_LIST_GET_FIRST(trx_sys->mysql_trx_list); - while (trx) { - if (trx->state == TRX_STATE_ACTIVE - && trx->mysql_thd - && innobase_thd_is_idle(trx->mysql_thd)) { - ib_int64_t start_time = innobase_thd_get_start_time(trx->mysql_thd); - ulong thd_id = innobase_thd_get_thread_id(trx->mysql_thd); - - if (trx->last_stmt_start != start_time) { - trx->idle_start = now; - trx->last_stmt_start = start_time; - } else if (difftime(now, trx->idle_start) - > srv_kill_idle_transaction) { - /* kill the session */ - mutex_exit(&trx_sys->mutex); - innobase_thd_kill(thd_id); - goto rescan_idle; - } - } - trx = UT_LIST_GET_NEXT(mysql_trx_list, trx); - } - mutex_exit(&trx_sys->mutex); - } - /* Flush stderr so that a database user gets the output to possible MySQL error file */ @@ -2412,10 +2428,8 @@ DECLARE_THREAD(srv_redo_log_follow_thread)( } while (srv_shutdown_state < SRV_SHUTDOWN_LAST_PHASE); - srv_track_changed_pages = FALSE; log_online_read_shutdown(); os_event_set(srv_redo_log_tracked_event); - srv_redo_log_thread_started = false; /* Defensive, not required */ my_thread_end(); os_thread_exit(NULL); @@ -2581,15 +2595,7 @@ srv_active_wake_master_thread(void) if (slot->in_use) { ut_a(srv_slot_get_type(slot) == SRV_MASTER); - - if (slot->suspended) { - - slot->suspended = FALSE; - - ++srv_sys->n_threads_active[SRV_MASTER]; - - os_event_set(slot->event); - } + os_event_set(slot->event); } srv_sys_mutex_exit(); @@ -3110,7 +3116,7 @@ suspend_thread: manual also mentions this string in several places. */ srv_main_thread_op_info = "waiting for server activity"; - os_event_wait(slot->event); + srv_resume_thread(slot); if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) { os_thread_exit(NULL); @@ -3232,8 +3238,7 @@ DECLARE_THREAD(srv_worker_thread)( do { srv_suspend_thread(slot); - - os_event_wait(slot->event); + srv_resume_thread(slot); srv_current_thread_priority = srv_purge_thread_priority; @@ -3373,8 +3378,6 @@ srv_purge_coordinator_suspend( ib_int64_t sig_count = srv_suspend_thread(slot); do { - ulint ret; - rw_lock_x_lock(&purge_sys->latch); purge_sys->running = false; @@ -3383,32 +3386,11 @@ srv_purge_coordinator_suspend( /* We don't wait right away on the the non-timed wait because we want to signal the thread that wants to suspend purge. */ - - if (stop) { - os_event_wait_low(slot->event, sig_count); - ret = 0; - } else if (rseg_history_len <= trx_sys->rseg_history_len) { - ret = os_event_wait_time_low( - slot->event, SRV_PURGE_MAX_TIMEOUT, sig_count); - } else { - /* We don't want to waste time waiting, if the - history list increased by the time we got here, - unless purge has been stopped. */ - ret = 0; - } - - srv_sys_mutex_enter(); - - /* The thread can be in state !suspended after the timeout - but before this check if another thread sent a wakeup signal. */ - - if (slot->suspended) { - slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; - ut_a(srv_sys->n_threads_active[slot->type] == 1); - } - - srv_sys_mutex_exit(); + const bool wait = stop + || rseg_history_len <= trx_sys->rseg_history_len; + const bool timeout = srv_resume_thread( + slot, sig_count, wait, + stop ? 0 : SRV_PURGE_MAX_TIMEOUT); sig_count = srv_suspend_thread(slot); @@ -3420,6 +3402,19 @@ srv_purge_coordinator_suspend( if (!stop) { ut_a(purge_sys->n_stop == 0); purge_sys->running = true; + + if (timeout + && rseg_history_len == trx_sys->rseg_history_len + && trx_sys->rseg_history_len < 5000) { + /* No new records were added since the + wait started. Simply wait for new + records. The magic number 5000 is an + approximation for the case where we + have cached UNDO log records which + prevent truncate of the UNDO + segments. */ + stop = true; + } } else { ut_a(purge_sys->n_stop > 0); @@ -3428,33 +3423,9 @@ srv_purge_coordinator_suspend( } rw_lock_x_unlock(&purge_sys->latch); - - if (ret == OS_SYNC_TIME_EXCEEDED) { - - /* No new records added since wait started then simply - wait for new records. The magic number 5000 is an - approximation for the case where we have cached UNDO - log records which prevent truncate of the UNDO - segments. */ - - if (rseg_history_len == trx_sys->rseg_history_len - && trx_sys->rseg_history_len < 5000) { - - stop = true; - } - } - } while (stop); - srv_sys_mutex_enter(); - - if (slot->suspended) { - slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; - ut_a(srv_sys->n_threads_active[slot->type] == 1); - } - - srv_sys_mutex_exit(); + srv_resume_thread(slot, 0, false); } /*********************************************************************//** @@ -3510,8 +3481,9 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( srv_purge_coordinator_suspend(slot, rseg_history_len); } + ut_ad(!slot->suspended); + if (srv_purge_should_exit(n_total_purged)) { - ut_a(!slot->suspended); break; } @@ -3626,12 +3598,10 @@ srv_get_task_queue_length(void) return(n_tasks); } -/**********************************************************************//** -Wakeup the purge threads. */ +/** Wake up the purge threads. */ UNIV_INTERN void -srv_purge_wakeup(void) -/*==================*/ +srv_purge_wakeup() { ut_ad(!srv_read_only_mode); @@ -3646,4 +3616,3 @@ srv_purge_wakeup(void) } } } - diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index cae3b71c2bb..d94dfe87783 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,6 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -608,19 +609,6 @@ create_log_file( /** Initial number of the first redo log file */ #define INIT_LOG_FILE0 (SRV_N_LOG_FILES_MAX + 1) -#ifdef DBUG_OFF -# define RECOVERY_CRASH(x) do {} while(0) -#else -# define RECOVERY_CRASH(x) do { \ - if (srv_force_recovery_crash == x) { \ - fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ - srv_force_recovery_crash); \ - fflush(stderr); \ - exit(3); \ - } \ -} while (0) -#endif - /*********************************************************************//** Creates all log files. @return DB_SUCCESS or error code */ @@ -661,13 +649,14 @@ create_log_files( file should be recoverable. The buffer pool was clean, and we can simply create all log files from the scratch. */ - RECOVERY_CRASH(6); + DBUG_EXECUTE_IF("innodb_log_abort_6", + return(DB_ERROR);); } } ut_ad(!buf_pool_check_no_pending_io()); - RECOVERY_CRASH(7); + DBUG_EXECUTE_IF("innodb_log_abort_7", return(DB_ERROR);); for (unsigned i = 0; i < srv_n_log_files; i++) { sprintf(logfilename + dirnamelen, @@ -680,7 +669,7 @@ create_log_files( } } - RECOVERY_CRASH(8); + DBUG_EXECUTE_IF("innodb_log_abort_8", return(DB_ERROR);); /* We did not create the first log file initially as ib_logfile0, so that crash recovery cannot find it until it @@ -735,10 +724,16 @@ create_log_files( return(DB_SUCCESS); } -/*********************************************************************//** -Renames the first log file. */ +/** Rename the first redo log file. +@param[in,out] logfilename buffer for the log file name +@param[in] dirnamelen length of the directory path +@param[in] lsn FIL_PAGE_FILE_FLUSH_LSN value +@param[in,out] logfile0 name of the first log file +@return error code +@retval DB_SUCCESS on successful operation */ +MY_ATTRIBUTE((warn_unused_result, nonnull)) static -void +dberr_t create_log_files_rename( /*====================*/ char* logfilename, /*!< in/out: buffer for log file name */ @@ -749,6 +744,9 @@ create_log_files_rename( /* If innodb_flush_method=O_DSYNC, we need to explicitly flush the log buffers. */ fil_flush(SRV_LOG_SPACE_FIRST_ID); + + DBUG_EXECUTE_IF("innodb_log_abort_9", return(DB_ERROR);); + /* Close the log files, so that we can rename the first one. */ fil_close_log_files(false); @@ -757,26 +755,28 @@ create_log_files_rename( checkpoint has been created. */ sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); - RECOVERY_CRASH(9); - ib_logf(IB_LOG_LEVEL_INFO, "Renaming log file %s to %s", logfile0, logfilename); mutex_enter(&log_sys->mutex); ut_ad(strlen(logfile0) == 2 + strlen(logfilename)); - ibool success = os_file_rename( - innodb_file_log_key, logfile0, logfilename); - ut_a(success); - - RECOVERY_CRASH(10); + dberr_t err = os_file_rename( + innodb_file_log_key, logfile0, logfilename) + ? DB_SUCCESS : DB_ERROR; /* Replace the first file with ib_logfile0. */ strcpy(logfile0, logfilename); mutex_exit(&log_sys->mutex); - fil_open_log_and_system_tablespace_files(); + DBUG_EXECUTE_IF("innodb_log_abort_10", err = DB_ERROR;); - ib_logf(IB_LOG_LEVEL_WARN, "New log files created, LSN=" LSN_PF, lsn); + if (err == DB_SUCCESS) { + fil_open_log_and_system_tablespace_files(); + ib_logf(IB_LOG_LEVEL_WARN, + "New log files created, LSN=" LSN_PF, lsn); + } + + return(err); } /*********************************************************************//** @@ -1608,8 +1608,6 @@ innobase_start_or_create_for_mysql(void) lsn_t max_arch_log_no = LSN_MAX; #endif /* UNIV_LOG_ARCHIVE */ ulint sum_of_new_sizes; - ulint sum_of_data_file_sizes; - ulint tablespace_size_in_header; dberr_t err; unsigned i; ulint srv_n_log_files_found = srv_n_log_files; @@ -1622,6 +1620,19 @@ innobase_start_or_create_for_mysql(void) size_t dirnamelen; bool sys_datafiles_created = false; + /* Check that os_fast_mutexes work as expected */ + os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); + + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); + + os_fast_mutex_unlock(&srv_os_test_mutex); + + os_fast_mutex_lock(&srv_os_test_mutex); + + os_fast_mutex_unlock(&srv_os_test_mutex); + + os_fast_mutex_free(&srv_os_test_mutex); + high_level_read_only = srv_read_only_mode || srv_force_recovery > SRV_FORCE_NO_TRX_UNDO; @@ -1664,22 +1675,7 @@ innobase_start_or_create_for_mysql(void) #endif /* PAGE_ATOMIC_REF_COUNT */ ); - - if (sizeof(ulint) != sizeof(void*)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: size of InnoDB's ulint is %lu, " - "but size of void*\n", (ulong) sizeof(ulint)); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: is %lu. The sizes should be the same " - "so that on a 64-bit\n", - (ulong) sizeof(void*)); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: platforms you can allocate more than 4 GB " - "of memory.\n"); - } + compile_time_assert(sizeof(ulint) == sizeof(void*)); /* If stacktrace is used we set up signal handler for SIGUSR2 signal here. If signal handler set fails we report that and disable @@ -2103,6 +2099,7 @@ innobase_start_or_create_for_mysql(void) fsp_init(); log_init(); + log_online_init(); lock_sys_create(srv_lock_table_size); @@ -2277,14 +2274,18 @@ innobase_start_or_create_for_mysql(void) dirnamelen, max_flushed_lsn, logfile0); + if (err == DB_SUCCESS) { + err = create_log_files_rename( + logfilename, + dirnamelen, + max_flushed_lsn, + logfile0); + } + if (err != DB_SUCCESS) { return(err); } - create_log_files_rename( - logfilename, dirnamelen, - max_flushed_lsn, logfile0); - /* Suppress the message about crash recovery. */ max_flushed_lsn = min_flushed_lsn @@ -2410,7 +2411,6 @@ files_checked: trx_sys_create(); if (create_new_db) { - ut_a(!srv_read_only_mode); init_log_online(); @@ -2454,8 +2454,12 @@ files_checked: fil_flush_file_spaces(FIL_TABLESPACE); - create_log_files_rename(logfilename, dirnamelen, - max_flushed_lsn, logfile0); + err = create_log_files_rename(logfilename, dirnamelen, + max_flushed_lsn, logfile0); + + if (err != DB_SUCCESS) { + return(err); + } } else { /* Check if we support the max format that is stamped @@ -2484,6 +2488,22 @@ files_checked: and there must be no page in the buf_flush list. */ buf_pool_invalidate(); + /* Start monitor thread early enough so that e.g. crash + recovery failing to find free pages in the buffer pool is + diagnosed. */ + if (!srv_read_only_mode) + { + /* Create the thread which prints InnoDB monitor + info */ + thread_handles[4 + SRV_MAX_N_IO_THREADS] = + os_thread_create( + srv_monitor_thread, + NULL, + thread_ids + 4 + SRV_MAX_N_IO_THREADS); + + thread_started[4 + SRV_MAX_N_IO_THREADS] = true; + } + /* We always try to do a recovery, even if the database had been shut down normally: this is the normal startup path */ @@ -2492,25 +2512,92 @@ files_checked: min_flushed_lsn, max_flushed_lsn); if (err != DB_SUCCESS) { - - return(DB_ERROR); + return(err); } init_log_online(); - /* Since the insert buffer init is in dict_boot, and the - insert buffer is needed in any disk i/o, first we call - dict_boot(). Note that trx_sys_init_at_db_start() only needs - to access space 0, and the insert buffer at this stage already - works for space 0. */ - + /* Initialize the change buffer. */ err = dict_boot(); if (err != DB_SUCCESS) { return(err); } + if (!srv_read_only_mode) { + if (sum_of_new_sizes > 0) { + /* New data file(s) were added */ + mtr_start(&mtr); + fsp_header_inc_size(0, sum_of_new_sizes, &mtr); + mtr_commit(&mtr); + /* Immediately write the log record about + increased tablespace size to disk, so that it + is durable even if mysqld would crash + quickly */ + log_buffer_flush_to_disk(); + } + } + + const ulint tablespace_size_in_header + = fsp_header_get_tablespace_size(); + +#ifdef UNIV_DEBUG + /* buf_debug_prints = TRUE; */ +#endif /* UNIV_DEBUG */ + ulint sum_of_data_file_sizes = 0; + + for (ulint d = 0; d < srv_n_data_files; d++) { + sum_of_data_file_sizes += srv_data_file_sizes[d]; + } + + /* Compare the system tablespace file size to what is + stored in FSP_SIZE. In open_or_create_data_files() + we already checked that the file sizes match the + innodb_data_file_path specification. */ + if (srv_read_only_mode + || sum_of_data_file_sizes == tablespace_size_in_header) { + /* Do not complain about the size. */ + } else if (!srv_auto_extend_last_data_file + || sum_of_data_file_sizes + < tablespace_size_in_header) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Tablespace size stored in header is " ULINTPF + " pages, but the sum of data file sizes is " + ULINTPF " pages", + tablespace_size_in_header, + sum_of_data_file_sizes); + + if (srv_force_recovery == 0 + && sum_of_data_file_sizes + < tablespace_size_in_header) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot start InnoDB. The tail of" + " the system tablespace is" + " missing. Have you edited" + " innodb_data_file_path in my.cnf" + " in an inappropriate way, removing" + " data files from there?" + " You can set innodb_force_recovery=1" + " in my.cnf to force" + " a startup if you are trying to" + " recover a badly corrupt database."); + + return(DB_ERROR); + } + } + + /* This must precede recv_apply_hashed_log_recs(true). */ ib_bh = trx_sys_init_at_db_start(); + + if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { + /* Apply the hashed log records to the + respective file pages, for the last batch of + recv_group_scan_log_recs(). */ + + recv_apply_hashed_log_recs(true); + DBUG_PRINT("ib_log", ("apply completed")); + } + n_recovered_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list); /* The purge system needs to create the purge view and @@ -2579,7 +2666,8 @@ files_checked: ULINT_MAX, LSN_MAX, NULL); ut_a(success); - RECOVERY_CRASH(1); + DBUG_EXECUTE_IF("innodb_log_abort_1", + return(DB_ERROR);); min_flushed_lsn = max_flushed_lsn = log_get_lsn(); @@ -2594,8 +2682,6 @@ files_checked: buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST); - RECOVERY_CRASH(2); - /* Flush the old log files. */ log_buffer_flush_to_disk(); /* If innodb_flush_method=O_DSYNC, @@ -2610,21 +2696,27 @@ files_checked: ut_d(recv_no_log_write = TRUE); ut_ad(!buf_pool_check_no_pending_io()); - RECOVERY_CRASH(3); + DBUG_EXECUTE_IF("innodb_log_abort_3", + return(DB_ERROR);); /* Stamp the LSN to the data files. */ fil_write_flushed_lsn_to_data_files( max_flushed_lsn, 0); - fil_flush_file_spaces(FIL_TABLESPACE); + DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;); - RECOVERY_CRASH(4); + if (err != DB_SUCCESS) { + return(err); + } + + fil_flush_file_spaces(FIL_TABLESPACE); /* Close and free the redo log files, so that we can replace them. */ fil_close_log_files(true); - RECOVERY_CRASH(5); + DBUG_EXECUTE_IF("innodb_log_abort_5", + return(DB_ERROR);); /* Free the old log file space. */ log_group_close_all(); @@ -2648,8 +2740,11 @@ files_checked: fil_write_flushed_lsn_to_data_files(min_flushed_lsn, 0); fil_flush_file_spaces(FIL_TABLESPACE); - create_log_files_rename(logfilename, dirnamelen, - log_get_lsn(), logfile0); + err = create_log_files_rename(logfilename, dirnamelen, + log_get_lsn(), logfile0); + if (err != DB_SUCCESS) { + return(err); + } } srv_startup_is_before_trx_rollback_phase = FALSE; @@ -2663,20 +2758,8 @@ files_checked: trx_sys_file_format_tag_init(); } - if (!create_new_db && sum_of_new_sizes > 0) { - /* New data file(s) were added */ - mtr_start(&mtr); - - fsp_header_inc_size(0, sum_of_new_sizes, &mtr); - - mtr_commit(&mtr); - - /* Immediately write the log record about increased tablespace - size to disk, so that it is durable even if mysqld would crash - quickly */ - - log_buffer_flush_to_disk(); - } + ut_ad(err == DB_SUCCESS); + ut_a(sum_of_new_sizes != ULINT_UNDEFINED); #ifdef UNIV_LOG_ARCHIVE if (!srv_read_only_mode) { @@ -2755,10 +2838,13 @@ files_checked: thread_started[3 + SRV_MAX_N_IO_THREADS] = true; /* Create the thread which prints InnoDB monitor info */ - thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( - srv_monitor_thread, - NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); - thread_started[4 + SRV_MAX_N_IO_THREADS] = true; + if (!thread_started[4 + SRV_MAX_N_IO_THREADS]) { + /* srv_monitor_thread not yet started */ + thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( + srv_monitor_thread, + NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); + thread_started[4 + SRV_MAX_N_IO_THREADS] = true; + } } /* Create the SYS_FOREIGN and SYS_FOREIGN_COLS system tables */ @@ -2776,12 +2862,6 @@ files_checked: } } - /* Create the SYS_ZIP_DICT system table */ - err = dict_create_or_check_sys_zip_dict(); - if (err != DB_SUCCESS) { - return(err); - } - srv_is_being_started = FALSE; ut_a(trx_purge_state() == PURGE_STATE_INIT); @@ -2831,125 +2911,6 @@ files_checked: buf_flush_lru_manager_thread_handle = os_thread_create(buf_flush_lru_manager_thread, NULL, NULL); buf_flush_lru_manager_thread_started = true; -#ifdef UNIV_DEBUG - /* buf_debug_prints = TRUE; */ -#endif /* UNIV_DEBUG */ - sum_of_data_file_sizes = 0; - - for (i = 0; i < srv_n_data_files; i++) { - sum_of_data_file_sizes += srv_data_file_sizes[i]; - } - - tablespace_size_in_header = fsp_header_get_tablespace_size(); - - if (!srv_read_only_mode - && !srv_auto_extend_last_data_file - && sum_of_data_file_sizes != tablespace_size_in_header) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: tablespace size" - " stored in header is %lu pages, but\n", - (ulong) tablespace_size_in_header); - ut_print_timestamp(stderr); - fprintf(stderr, - "InnoDB: the sum of data file sizes is %lu pages\n", - (ulong) sum_of_data_file_sizes); - - if (srv_force_recovery == 0 - && sum_of_data_file_sizes < tablespace_size_in_header) { - /* This is a fatal error, the tail of a tablespace is - missing */ - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot start InnoDB." - " The tail of the system tablespace is\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: missing. Have you edited" - " innodb_data_file_path in my.cnf in an\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: inappropriate way, removing" - " ibdata files from there?\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: You can set innodb_force_recovery=1" - " in my.cnf to force\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: a startup if you are trying" - " to recover a badly corrupt database.\n"); - - return(DB_ERROR); - } - } - - if (!srv_read_only_mode - && srv_auto_extend_last_data_file - && sum_of_data_file_sizes < tablespace_size_in_header) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: tablespace size stored in header" - " is %lu pages, but\n", - (ulong) tablespace_size_in_header); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: the sum of data file sizes" - " is only %lu pages\n", - (ulong) sum_of_data_file_sizes); - - if (srv_force_recovery == 0) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot start InnoDB. The tail of" - " the system tablespace is\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: missing. Have you edited" - " innodb_data_file_path in my.cnf in an\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: inappropriate way, removing" - " ibdata files from there?\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: You can set innodb_force_recovery=1" - " in my.cnf to force\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: a startup if you are trying to" - " recover a badly corrupt database.\n"); - - return(DB_ERROR); - } - } - - /* Check that os_fast_mutexes work as expected */ - os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - - if (0 != os_fast_mutex_trylock(&srv_os_test_mutex)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: pthread_mutex_trylock returns" - " an unexpected value on\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: success! Cannot continue.\n"); - exit(1); - } - - os_fast_mutex_unlock(&srv_os_test_mutex); - - os_fast_mutex_lock(&srv_os_test_mutex); - - os_fast_mutex_unlock(&srv_os_test_mutex); - - os_fast_mutex_free(&srv_os_test_mutex); - if (!srv_file_per_table && srv_pass_corrupt_table) { fprintf(stderr, "InnoDB: Warning:" " The option innodb_file_per_table is disabled," @@ -3190,6 +3151,7 @@ innobase_shutdown_for_mysql(void) btr_search_disable(); ibuf_close(); + log_online_shutdown(); log_shutdown(); trx_sys_file_format_close(); trx_sys_close(); diff --git a/storage/xtradb/sync/sync0arr.cc b/storage/xtradb/sync/sync0arr.cc index e16eddae80c..7325cb49145 100644 --- a/storage/xtradb/sync/sync0arr.cc +++ b/storage/xtradb/sync/sync0arr.cc @@ -1161,9 +1161,10 @@ sync_array_print_long_waits( now the values of pending calls of these. */ fprintf(stderr, - "InnoDB: Pending preads %lu, pwrites %lu\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + "InnoDB: Pending reads " UINT64PF + ", writes " UINT64PF "\n", + MONITOR_VALUE(MONITOR_OS_PENDING_READS), + MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); srv_print_innodb_monitor = TRUE; os_event_set(srv_monitor_event); diff --git a/storage/xtradb/sync/sync0sync.cc b/storage/xtradb/sync/sync0sync.cc index e705ed4f587..c270b73e5a0 100644 --- a/storage/xtradb/sync/sync0sync.cc +++ b/storage/xtradb/sync/sync0sync.cc @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -223,9 +224,6 @@ UNIV_INTERN mysql_pfs_key_t mutex_list_mutex_key; /** Latching order checks start when this is set TRUE */ UNIV_INTERN ibool sync_order_checks_on = FALSE; -/** Number of slots reserved for each OS thread in the sync level array */ -static const ulint SYNC_THREAD_N_LEVELS = 10000; - /** Array for tracking sync levels per thread. */ typedef std::vector<sync_level_t> sync_arr_t; @@ -1349,7 +1347,7 @@ sync_thread_add_level( case SYNC_TRX_UNDO_PAGE: /* Purge is allowed to read in as many UNDO pages as it likes, there was a bogus rule here earlier that forced the caller to - acquire the purge_sys_t::mutex. The purge mutex did not really + acquire the trx_purge_t::mutex. The purge mutex did not really protect anything because it was only ever acquired by the single purge thread. The purge thread can read the UNDO pages without any covering mutex. */ diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index d9e40c5d6f5..6b7b7df4cd8 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -172,13 +173,9 @@ trx_purge_sys_close(void) sess_close(purge_sys->sess); - purge_sys->sess = NULL; - read_view_free(purge_sys->prebuilt_view); read_view_free(purge_sys->prebuilt_clone); - purge_sys->view = NULL; - rw_lock_free(&purge_sys->latch); mutex_free(&purge_sys->bh_mutex); @@ -187,9 +184,6 @@ trx_purge_sys_close(void) ib_bh_free(purge_sys->ib_bh); os_event_free(purge_sys->event); - - purge_sys->event = NULL; - mem_free(purge_sys); purge_sys = NULL; @@ -1306,20 +1300,16 @@ void trx_purge_stop(void) /*================*/ { - purge_state_t state; - ib_int64_t sig_count = os_event_reset(purge_sys->event); - ut_a(srv_n_purge_threads > 0); rw_lock_x_lock(&purge_sys->latch); - ut_a(purge_sys->state != PURGE_STATE_INIT); - ut_a(purge_sys->state != PURGE_STATE_EXIT); - ut_a(purge_sys->state != PURGE_STATE_DISABLED); + const ib_int64_t sig_count = os_event_reset(purge_sys->event); + const purge_state_t state = purge_sys->state; - ++purge_sys->n_stop; + ut_a(state == PURGE_STATE_RUN || state == PURGE_STATE_STOP); - state = purge_sys->state; + ++purge_sys->n_stop; if (state == PURGE_STATE_RUN) { ib_logf(IB_LOG_LEVEL_INFO, "Stopping purge"); diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index c65d95a9817..8ab9511befc 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -739,9 +740,9 @@ trx_rollback_or_clean_recovered( } if (all) { - fprintf(stderr, - "InnoDB: Starting in background the rollback" - " of uncommitted transactions\n"); + ib_logf(IB_LOG_LEVEL_INFO, + "Starting in background the rollback" + " of recovered transactions"); } /* Note: For XA recovered transactions, we rely on MySQL to @@ -761,6 +762,12 @@ trx_rollback_or_clean_recovered( assert_trx_in_rw_list(trx); + if (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown != 0) { + all = FALSE; + break; + } + /* If this function does a cleanup or rollback then it will release the trx_sys->mutex, therefore we need to reacquire it before retrying the loop. */ @@ -778,10 +785,8 @@ trx_rollback_or_clean_recovered( } while (trx != NULL); if (all) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Rollback of non-prepared" - " transactions completed\n"); + ib_logf(IB_LOG_LEVEL_INFO, + "Rollback of non-prepared transactions completed"); } } diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc index 5baea052b7b..1e00fa97299 100644 --- a/storage/xtradb/trx/trx0sys.cc +++ b/storage/xtradb/trx/trx0sys.cc @@ -1199,7 +1199,9 @@ trx_sys_close(void) ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); /* Only prepared transactions may be left in the system. Free them. */ - ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx); + ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx + || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); while ((trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list)) != NULL) { trx_free_prepared(trx); @@ -1245,6 +1247,33 @@ trx_sys_close(void) trx_sys = NULL; } +/** @brief Convert an undo log to TRX_UNDO_PREPARED state on shutdown. + +If any prepared ACTIVE transactions exist, and their rollback was +prevented by innodb_force_recovery, we convert these transactions to +XA PREPARE state in the main-memory data structures, so that shutdown +will proceed normally. These transactions will again recover as ACTIVE +on the next restart, and they will be rolled back unless +innodb_force_recovery prevents it again. + +@param[in] trx transaction +@param[in,out] undo undo log to convert to TRX_UNDO_PREPARED */ +static +void +trx_undo_fake_prepared( + const trx_t* trx, + trx_undo_t* undo) +{ + ut_ad(srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)); + ut_ad(trx->is_recovered); + + if (undo != NULL) { + ut_ad(undo->state == TRX_UNDO_ACTIVE); + undo->state = TRX_UNDO_PREPARED; + } +} + /********************************************************************* Check if there are any active (non-prepared) transactions. @return total number of active transactions or 0 if none */ @@ -1253,15 +1282,42 @@ ulint trx_sys_any_active_transactions(void) /*=================================*/ { - ulint total_trx = 0; - mutex_enter(&trx_sys->mutex); - total_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list) - + UT_LIST_GET_LEN(trx_sys->mysql_trx_list); + ulint total_trx = UT_LIST_GET_LEN(trx_sys->mysql_trx_list); + + if (total_trx == 0) { + total_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list); + ut_a(total_trx >= trx_sys->n_prepared_trx); + + if (total_trx > trx_sys->n_prepared_trx + && srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) { + for (trx_t* trx = UT_LIST_GET_FIRST( + trx_sys->rw_trx_list); + trx != NULL; + trx = UT_LIST_GET_NEXT(trx_list, trx)) { + if (!trx_state_eq(trx, TRX_STATE_ACTIVE) + || !trx->is_recovered) { + continue; + } + /* This was a recovered transaction + whose rollback was disabled by + the innodb_force_recovery setting. + Pretend that it is in XA PREPARE + state so that shutdown will work. */ + trx_undo_fake_prepared( + trx, trx->insert_undo); + trx_undo_fake_prepared( + trx, trx->update_undo); + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + } + } - ut_a(total_trx >= trx_sys->n_prepared_trx); - total_trx -= trx_sys->n_prepared_trx; + ut_a(total_trx >= trx_sys->n_prepared_trx); + total_trx -= trx_sys->n_prepared_trx; + } mutex_exit(&trx_sys->mutex); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index eda04848291..fd620ae6659 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -473,7 +473,11 @@ trx_free_prepared( /*==============*/ trx_t* trx) /*!< in, own: trx object */ { - ut_a(trx_state_eq(trx, TRX_STATE_PREPARED)); + ut_a(trx_state_eq(trx, TRX_STATE_PREPARED) + || (trx_state_eq(trx, TRX_STATE_ACTIVE) + && trx->is_recovered + && (srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO))); ut_a(trx->magic_n == TRX_MAGIC_N); lock_trx_release_locks(trx); diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc index cdd23726f2e..220589dd9ff 100644 --- a/storage/xtradb/trx/trx0undo.cc +++ b/storage/xtradb/trx/trx0undo.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2011,13 +2012,37 @@ trx_undo_free_prepared( ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS); if (trx->update_undo) { - ut_a(trx->update_undo->state == TRX_UNDO_PREPARED); + switch (trx->update_undo->state) { + case TRX_UNDO_PREPARED: + break; + case TRX_UNDO_ACTIVE: + /* lock_trx_release_locks() assigns + trx->is_recovered=false */ + ut_a(srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + break; + default: + ut_error; + } + UT_LIST_REMOVE(undo_list, trx->rseg->update_undo_list, trx->update_undo); trx_undo_mem_free(trx->update_undo); } if (trx->insert_undo) { - ut_a(trx->insert_undo->state == TRX_UNDO_PREPARED); + switch (trx->insert_undo->state) { + case TRX_UNDO_PREPARED: + break; + case TRX_UNDO_ACTIVE: + /* lock_trx_release_locks() assigns + trx->is_recovered=false */ + ut_a(srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + break; + default: + ut_error; + } + UT_LIST_REMOVE(undo_list, trx->rseg->insert_undo_list, trx->insert_undo); trx_undo_mem_free(trx->insert_undo); diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index 4677bd59415..5e9fd081a09 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -1,5 +1,5 @@ -# Copyright (c) 2006, 2014, Oracle and/or its affiliates. -# Copyright (c) 2012, 2014, SkySQL Ab. +# Copyright (c) 2006, 2016, Oracle and/or its affiliates. +# Copyright (c) 2012, 2017, MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ IF(UNIX) ENDFOREACH() IF(INSTALL_SUPPORTFILESDIR) INSTALL(FILES magic DESTINATION ${inst_location} COMPONENT SupportFiles) - INSTALL(DIRECTORY RHEL4-SElinux/ DESTINATION ${inst_location}/SELinux/RHEL4 COMPONENT SupportFiles) + ADD_SUBDIRECTORY(SELinux) ENDIF() INSTALL(FILES mysql.m4 DESTINATION ${INSTALL_SHAREDIR}/aclocal COMPONENT Development) diff --git a/support-files/SELinux/CMakeLists.txt b/support-files/SELinux/CMakeLists.txt new file mode 100644 index 00000000000..996d56d6c05 --- /dev/null +++ b/support-files/SELinux/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright (c) 2017, MariaDB +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +FIND_PROGRAM(CHECKMODULE checkmodule) +FIND_PROGRAM(SEMODULE_PACKAGE semodule_package) +MARK_AS_ADVANCED(CHECKMODULE SEMODULE_PACKAGE) + +SET(params DESTINATION ${INSTALL_SUPPORTFILESDIR}/SELinux COMPONENT SupportFiles) + +IF(CHECKMODULE AND SEMODULE_PACKAGE) + FOREACH(pol mariadb) + SET(src ${CMAKE_CURRENT_SOURCE_DIR}/${pol}.te) + SET(mod ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${pol}-pp.dir/${pol}.mod) + SET(out ${CMAKE_CURRENT_BINARY_DIR}/${pol}.pp) + ADD_CUSTOM_COMMAND(OUTPUT ${out} + COMMAND ${CHECKMODULE} -M -m ${src} -o ${mod} + COMMAND ${SEMODULE_PACKAGE} -m ${mod} -o ${out} + DEPENDS ${src}) + ADD_CUSTOM_TARGET(${pol}-pp ALL DEPENDS ${out}) + INSTALL(FILES ${out} ${params}) + ENDFOREACH() +ENDIF() +INSTALL(FILES mariadb.te rhel4-mysql.fc rhel4-mysql.te ${params}) diff --git a/support-files/SELinux/mariadb.te b/support-files/SELinux/mariadb.te new file mode 100644 index 00000000000..1d3de52c700 --- /dev/null +++ b/support-files/SELinux/mariadb.te @@ -0,0 +1,9 @@ +module mariadb 1.0; + +require { + type mysqld_safe_t; + class capability { setuid setgid }; +} + +#============= mysqld_safe_t ============== +allow mysqld_safe_t self:capability { setuid setgid }; diff --git a/support-files/RHEL4-SElinux/mysql.fc b/support-files/SELinux/rhel4-mysql.fc index aa0fced4bbc..aa0fced4bbc 100644 --- a/support-files/RHEL4-SElinux/mysql.fc +++ b/support-files/SELinux/rhel4-mysql.fc diff --git a/support-files/RHEL4-SElinux/mysql.te b/support-files/SELinux/rhel4-mysql.te index d2609244640..d2609244640 100644 --- a/support-files/RHEL4-SElinux/mysql.te +++ b/support-files/SELinux/rhel4-mysql.te diff --git a/support-files/build-tags b/support-files/build-tags index 4b76e01907a..03b243ee8cc 100755 --- a/support-files/build-tags +++ b/support-files/build-tags @@ -1,9 +1,16 @@ #! /bin/sh rm -f TAGS -filter='\.cpp$\|\.cc$\|\.c$\|\.h$\|sql_yacc\.yy$\|\.hpp$\|\.ic$\|errmsg-utf8\.txt$' -list="find . -type f" -git rev-parse >/dev/null 2>/dev/null && list="git ls-files" - -$list |grep $filter | xargs etags -o TAGS --append +if git rev-parse HEAD >/dev/null 2>&1 +then + cd `git rev-parse --show-toplevel` + echo client storage dbug libmysql sql-common \ + sql extra mysys mysys_ssl strings regex pcre vio include \ + tools unittest plugin libmysqld | \ + xargs -n1 git ls-files | grep -v '\.jar$' | \ + xargs etags -o TAGS --append +else + find . -type f ! -name "*.jar" | + xargs etags -o TAGS --append +fi diff --git a/support-files/mysql-log-rotate.sh b/support-files/mysql-log-rotate.sh index 0ee78b0f7ca..5d1b30b208e 100644 --- a/support-files/mysql-log-rotate.sh +++ b/support-files/mysql-log-rotate.sh @@ -30,7 +30,8 @@ if test -x @bindir@/mysqladmin && \ @bindir@/mysqladmin ping &>/dev/null then - @bindir@/mysqladmin flush-logs + @bindir@/mysqladmin --local flush-error-log \ + flush-engine-log flush-general-log flush-slow-log fi endscript } diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index 5e1ad5bfd99..bff270ee61f 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -151,15 +151,8 @@ parse_server_arguments() { # Get arguments from the my.cnf file, # the only group, which is read from now on is [mysqld] -if test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x $bindir/my_print_defaults -then +if test -x "$bindir/my_print_defaults"; then print_defaults="$bindir/my_print_defaults" -elif test -x $bindir/mysql_print_defaults -then - print_defaults="$bindir/mysql_print_defaults" else # Try to find basedir in /etc/my.cnf conf=/etc/my.cnf @@ -176,11 +169,6 @@ else print_defaults="$d/bin/my_print_defaults" break fi - if test -x "$d/bin/mysql_print_defaults" - then - print_defaults="$d/bin/mysql_print_defaults" - break - fi done fi @@ -382,7 +370,7 @@ case "$mode" in fi else # Try to find appropriate mysqld process - mysqld_pid=`pidof $libexecdir/mysqld` + mysqld_pid=`pgrep $libexecdir/mysqld` # test if multiple pids exist pid_count=`echo $mysqld_pid | wc -w` diff --git a/support-files/rpm/server-postin.sh b/support-files/rpm/server-postin.sh index cd2aec4d84a..daf68f00c0c 100644 --- a/support-files/rpm/server-postin.sh +++ b/support-files/rpm/server-postin.sh @@ -31,6 +31,11 @@ if [ $1 = 1 ] ; then # The user may already exist, make sure it has the proper group nevertheless (BUG#12823) usermod --gid %{mysqld_group} %{mysqld_user} 2> /dev/null || true + # Temporary Workaround for MDEV-11386 - will be corrected in Advance Toolchain 10.0-3 and 8.0-8 + for ldconfig in /opt/at*/sbin/ldconfig; do + test -x $ldconfig && $ldconfig + done + # Change permissions so that the user that will run the MySQL daemon # owns all database files. chown -R %{mysqld_user}:%{mysqld_group} $datadir @@ -55,31 +60,9 @@ fi SETARGETDIR=/etc/selinux/targeted/src/policy SEDOMPROG=$SETARGETDIR/domains/program SECONPROG=$SETARGETDIR/file_contexts/program -if [ -f /etc/redhat-release ] ; then - if grep '\(Red Hat Enterprise Linux ..\|CentOS\) release 4' \ - /etc/redhat-release >/dev/null 2>&1; then - echo - echo - echo 'Notes regarding SELinux on this platform:' - echo '=========================================' - echo - echo 'The default policy might cause server startup to fail because it is ' - echo 'not allowed to access critical files. In this case, please update ' - echo 'your installation. ' - echo - echo 'The default policy might also cause inavailability of SSL related ' - echo 'features because the server is not allowed to access /dev/random ' - echo 'and /dev/urandom. If this is a problem, please do the following: ' - echo - echo ' 1) install selinux-policy-targeted-sources from your OS vendor' - echo ' 2) add the following two lines to '$SEDOMPROG/mysqld.te':' - echo ' allow mysqld_t random_device_t:chr_file read;' - echo ' allow mysqld_t urandom_device_t:chr_file read;' - echo ' 3) cd to '$SETARGETDIR' and issue the following command:' - echo ' make load' - echo - echo - fi + +if [ -x /usr/sbin/semodule ] ; then + /usr/sbin/semodule -i /usr/share/mysql/SELinux/mariadb.pp fi if [ -x sbin/restorecon ] ; then diff --git a/tests/fork_big.pl b/tests/fork_big.pl index 6e78e779d11..ec1f131d453 100755 --- a/tests/fork_big.pl +++ b/tests/fork_big.pl @@ -1,4 +1,5 @@ #!/usr/bin/perl -w +use strict; # Copyright (c) 2001, 2006 MySQL AB # Use is subject to license terms @@ -21,7 +22,7 @@ # # Tested a lot with: --threads=30 -$opt_loop_count=500000; # Change this to make test harder/easier +my $opt_loop_count=500000; # Change this to make test harder/easier ##################### Standard benchmark inits ############################## @@ -31,6 +32,13 @@ use Benchmark; package main; +our ($opt_skip_create,$opt_skip_in,$opt_verbose,$opt_fast_insert); +our ($opt_lock_tables,$opt_debug,$opt_skip_delete,$opt_fast,$opt_force); +our ($opt_threads); +our ($opt_host,$opt_user,$opt_password,$opt_db); +my (@testtables, $abort_table, $numtables, $start_time, $end_time); +my ($dbh); + $opt_skip_create=$opt_skip_in=$opt_verbose=$opt_fast_insert= $opt_lock_tables=$opt_debug=$opt_skip_delete=$opt_fast=$opt_force=0; $opt_threads=5; @@ -94,6 +102,8 @@ $|= 1; # Autoflush #### Start the tests #### +my ($i, $pid, %work); + for ($i=0 ; $i < $opt_threads ; $i ++) { test_insert() if (($pid=fork()) == 0); $work{$pid}="insert"; @@ -118,10 +128,13 @@ test_alter() if (($pid=fork()) == 0); $work{$pid}="alter"; print "Started " . ($opt_threads*2+4) . " threads\n"; +my ($errors, $running_insert_threads); + $errors=0; $running_insert_threads=$opt_threads+$numtables; while (($pid=wait()) != -1) { + my ($ret); $ret=$?/256; print "thread '" . $work{$pid} . "' finished with exit code $ret\n"; if ($work{$pid} =~ /^insert/) @@ -203,7 +216,7 @@ sub test_insert sub test_select { - my ($dbh, $i, $j, $count, $loop); + my ($dbh, $i, $j, $count, $loop, $count_query, $row_counts); $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", $opt_user, $opt_password, @@ -270,7 +283,7 @@ sub test_select_count sub test_join { - my ($dbh, $i, $j, $count, $loop); + my ($dbh, $i, $j, $count, $loop, $count_query, $row_counts); $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", $opt_user, $opt_password, @@ -389,7 +402,7 @@ sub test_update sub test_check { - my ($dbh, $row, $i, $j, $type, $table); + my ($dbh, $sth, $row, $i, $j, $type, $table); $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", $opt_user, $opt_password, { PrintError => 0}) || die $DBI::errstr; @@ -397,7 +410,7 @@ sub test_check $type= "check"; for ($i=$j=0 ; !test_if_abort($dbh) ; $i++) { - sleep(1000); + sleep(200); $table=$testtables[$j]->[0]; $sth=$dbh->prepare("$type table $table") || die "Got error on prepare: $DBI::errstr\n"; $sth->execute || die $DBI::errstr; @@ -426,7 +439,7 @@ sub test_check sub test_repair { - my ($dbh, $row, $i, $type, $table); + my ($dbh, $sth, $row, $i, $type, $table); $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", $opt_user, $opt_password, { PrintError => 0}) || die $DBI::errstr; @@ -434,7 +447,7 @@ sub test_repair $type= "repair"; for ($i=0 ; !test_if_abort($dbh) ; $i++) { - sleep(2000); + sleep(100); $table=$testtables[0]->[0]; $sth=$dbh->prepare("$type table $table") || die "Got error on prepare: $DBI::errstr\n"; $sth->execute || die $DBI::errstr; @@ -470,7 +483,7 @@ sub test_flush $count=0; while (!test_if_abort($dbh)) { - sleep(3000); + sleep(300); $dbh->do("flush tables $tables") || die "Got error on flush $DBI::errstr\n"; $count++; @@ -488,7 +501,7 @@ sub test_flush sub test_database { my ($database) = @_; - my ($dbh, $row, $i, $type, $tables); + my ($dbh, $sth, $row, $i, $type, $tables); $dbh = DBI->connect("DBI:mysql:$database:$opt_host", $opt_user, $opt_password, { PrintError => 0}) || die $DBI::errstr; @@ -521,7 +534,7 @@ sub test_database sub test_alter { - my ($dbh, $row, $i, $type, $table); + my ($dbh, $sth, $row, $i, $type, $table); $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", $opt_user, $opt_password, { PrintError => 0}) || die $DBI::errstr; @@ -559,6 +572,7 @@ sub signal_abort sub test_if_abort() { my ($dbh)=@_; + my ($row); $row=simple_query($dbh,"select * from $opt_db.$abort_table"); return (defined($row) && defined($row->[0]) != 0) ? 1 : 0; } @@ -567,7 +581,7 @@ sub test_if_abort() sub make_count_query { my ($table_count)= @_; - my ($tables, $count_query, $i, $tables_def); + my ($tables, $count_query, $i, $table_def); $tables=""; $count_query="select high_priority "; $table_count--; diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 2acec60166e..4e4768559d4 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -1,5 +1,5 @@ -/* Copyright (c) 2002, 2012, Oracle and/or its affiliates. - Copyright (c) 2008, 2013, Monty Program Ab +/* Copyright (c) 2002, 2014, Oracle and/or its affiliates. + Copyright (c) 2008, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19274,6 +19274,49 @@ static void test_mdev4326() } +/** + BUG#17512527: LIST HANDLING INCORRECT IN MYSQL_PRUNE_STMT_LIST() +*/ +static void test_bug17512527() +{ + MYSQL *conn; + MYSQL_STMT *stmt1, *stmt2; + unsigned long thread_id; + char query[MAX_TEST_QUERY_LENGTH]; + int rc; + + conn= client_connect(0, MYSQL_PROTOCOL_SOCKET, 1); + + stmt1 = mysql_stmt_init(conn); + check_stmt(stmt1); + rc= mysql_stmt_prepare(stmt1, STRING_WITH_LEN("SELECT 1")); + check_execute(stmt1, rc); + + stmt2 = mysql_stmt_init(conn); + check_stmt(stmt2); + + thread_id= mysql_thread_id(conn); + sprintf(query, "KILL %lu", thread_id); + if (thread_query(query)) + exit(1); + + rc= mysql_stmt_prepare(stmt2, STRING_WITH_LEN("SELECT 2")); + check_execute(stmt2, rc); + + rc= mysql_stmt_execute(stmt1); + check_execute_r(stmt1, rc); + + rc= mysql_stmt_execute(stmt2); + check_execute(stmt2, rc); + + mysql_close(conn); + + mysql_stmt_close(stmt2); + mysql_stmt_close(stmt1); +} + + + /* Check compressed protocol */ @@ -19645,6 +19688,9 @@ static struct my_tests_st my_tests[]= { { "test_bug13001491", test_bug13001491 }, { "test_mdev4326", test_mdev4326 }, { "test_ps_sp_out_params", test_ps_sp_out_params }, +#ifndef _WIN32 + { "test_bug17512527", test_bug17512527}, +#endif { "test_compressed_protocol", test_compressed_protocol }, { "test_big_packet", test_big_packet }, { 0, 0 } diff --git a/win/create_def_file.js b/win/create_def_file.js index 5fb28ef0bee..25bbbb4eb3d 100644 --- a/win/create_def_file.js +++ b/win/create_def_file.js @@ -54,6 +54,22 @@ var is64 = args.Item(0).toLowerCase() == "x64"; var shell = new ActiveXObject("WScript.Shell"); var fso = new ActiveXObject("Scripting.FileSystemObject"); +/* + If .def file is used with together with lib.exe + the name mangling for stdcall is slightly different. + + Undescore prefix for stdcall function name must be removed for + lib.exe but not link.exe (see ScrubSymbol()) + + This difference is not documented anywhere and could + be a bug in compiler tools. + + We use a parameter /forLib, if the resulting .def file is used + with lib.exe . +*/ +var forLib = false; + + OutputSymbols(CollectSymbols()); @@ -62,8 +78,8 @@ function OutputSymbols(symbols) { var out = WScript.StdOut; out.WriteLine("EXPORTS"); - for (var sym in symbols) - out.WriteLine(sym); + for (var i= 0; i < symbols.length; i++) + out.WriteLine(symbols[i]); } function echo(message) @@ -72,9 +88,10 @@ function echo(message) } // Extract global symbol names and type from objects +// Returns string array with symbol names function CollectSymbols() { - var uniqueSymbols = new Array(); + var uniqueSymbols = new Object(); try { @@ -146,7 +163,19 @@ function CollectSymbols() uniqueSymbols[symbol] = 1; } fso.DeleteFile(rspfilename); - return uniqueSymbols; + // Sort symbols names + var keys=[]; + var sorted = {}; + for (key in uniqueSymbols) + { + if (uniqueSymbols.hasOwnProperty(key)) + { + keys.push(key); + } + } + keys.sort(); + + return keys; } // performs necessary cleanup on the symbol name @@ -156,6 +185,9 @@ function ScrubSymbol(symbol) if (symbol.charAt(0) != "_") return symbol; + if (forLib) + return symbol.substring(1, symbol.length); + var atSign = symbol.indexOf("@"); if (atSign != -1) { @@ -189,7 +221,11 @@ function CreateResponseFile(filename) var index = 1; for (; index < args.length; index++) { - addToResponseFile(args.Item(index),responseFile); + var param = args.Item(index); + if (param == "/forLib") + forLib = true; + else + addToResponseFile(args.Item(index),responseFile); } responseFile.Close(); } diff --git a/win/packaging/CMakeLists.txt b/win/packaging/CMakeLists.txt index 1682bae6986..d33516044f6 100644 --- a/win/packaging/CMakeLists.txt +++ b/win/packaging/CMakeLists.txt @@ -51,7 +51,7 @@ IF(MSVC_VERSION EQUAL 1600 OR MSVC_VERSION EQUAL 1700 ) SET(WIX_MSVC_SUFFIX "VS2010") ELSEIF(MSVC_VERSION EQUAL 1800) SET(WIX_MSVC_SUFFIX "VS2013") -ELSEIF (MSVC_VERSION EQUAL 1900) +ELSE() SET(WIX_MSVC_SUFFIX "VS2015") ENDIF() |