MDEV-29258 Failing assertion for name length on RENAME TABLE

trx_undo_page_report_rename(): Use the correct maximum length of
a table name. Both the database name and the table name can be up to
NAME_CHAR_LEN (64 characters) times 5 bytes per character in the
my_charset_filename encoding. They are not encoded in UTF-8!

fil_op_write_log(): Reserve the correct amount of log buffer for
a rename operation. The file name will be appended by
mlog_catenate_string().

rename_file_ext(): Reserve a large enough buffer for the file names.

5 files changed, 31 insertions, 8 deletions

diff --git a/mysql-test/suite/innodb/r/foreign_key_not_windows.result b/mysql-test/suite/innodb/r/foreign_key_not_windows.result
index 764ba911214..0dcc3d46bb1 100644
--- a/mysql-test/suite/innodb/r/foreign_key_not_windows.result
+++ b/mysql-test/suite/innodb/r/foreign_key_not_windows.result
@@ -11,6 +11,13 @@ CREATE TABLE `d255`.`_##################################################`
 ERROR HY000: Long database name and identifier for object resulted in path length exceeding 512 characters. Path: './@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023/_@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023@0023
 CREATE TABLE `d255`.`##################################################`
 (a INT PRIMARY KEY, FOREIGN KEY(a) REFERENCES test.t(a)) ENGINE=InnoDB;
+#
+# MDEV-29258 Failing assertion for name length on RENAME TABLE
+#
+CREATE TABLE `d255`.`d245` (x INT) ENGINE=InnoDB;
+DROP TABLE `d255`.`d250`;
+RENAME TABLE `d250#`.`d245` TO `d250#`.`d250`;
+RENAME TABLE `d255`.`d250` TO a;
 DROP DATABASE `d255`;
-DROP TABLE t;
+DROP TABLE a,t;
 # End of 10.3 tests
diff --git a/mysql-test/suite/innodb/t/foreign_key_not_windows.test b/mysql-test/suite/innodb/t/foreign_key_not_windows.test
index 7ad3723d5de..0e1e25d64b3 100644
--- a/mysql-test/suite/innodb/t/foreign_key_not_windows.test
+++ b/mysql-test/suite/innodb/t/foreign_key_not_windows.test
@@ -38,8 +38,23 @@ eval CREATE TABLE `$d255`.`_$d250`
 --replace_result $d255 d255
 eval CREATE TABLE `$d255`.`$d250`
 (a INT PRIMARY KEY, FOREIGN KEY(a) REFERENCES test.t(a)) ENGINE=InnoDB;
+
+--echo #
+--echo # MDEV-29258 Failing assertion for name length on RENAME TABLE
+--echo #
+
+let $d245=-------------------------------------------------;
+--replace_result $d245 d245 $d255 d255
+eval CREATE TABLE `$d255`.`$d245` (x INT) ENGINE=InnoDB;
+--replace_result $d250 d250 $d255 d255
+eval DROP TABLE `$d255`.`$d250`;
+
+--replace_result $d245 d245 $d250 d250 d255 d255
+eval RENAME TABLE `$d255`.`$d245` TO `$d255`.`$d250`;
+--replace_result $d250 d250 $d255 d255
+eval RENAME TABLE `$d255`.`$d250` TO a;
 --replace_result $d255 d255
 eval DROP DATABASE `$d255`;
-DROP TABLE t;
+DROP TABLE a,t;
 
 --echo # End of 10.3 tests
diff --git a/sql/table.cc b/sql/table.cc
index 64fb3150a39..506195127b2 100644
--- a/sql/table.cc
+++ b/sql/table.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2017, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2021, MariaDB
+   Copyright (c) 2008, 2022, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -4197,7 +4197,8 @@ void update_create_info_from_table(HA_CREATE_INFO *create_info, TABLE *table)
 int
 rename_file_ext(const char * from,const char * to,const char * ext)
 {
-  char from_b[FN_REFLEN],to_b[FN_REFLEN];
+  /* Reserve space for ./databasename/tablename.frm + NUL byte */
+  char from_b[2 + FN_REFLEN + 4 + 1], to_b[2 + FN_REFLEN + 4 + 1];
   (void) strxmov(from_b,from,ext,NullS);
   (void) strxmov(to_b,to,ext,NullS);
   return mysql_file_rename(key_file_frm, from_b, to_b, MYF(0));
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index a196303c39f..5d9f80eda70 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -2105,7 +2105,7 @@ fil_op_write_log(
 	case MLOG_FILE_RENAME2:
 		ut_ad(strchr(new_path, OS_PATH_SEPARATOR) != NULL);
 		len = strlen(new_path) + 1;
-		log_ptr = mlog_open(mtr, 2 + len);
+		log_ptr = mlog_open(mtr, 2);
 		ut_a(log_ptr);
 		mach_write_to_2(log_ptr, len);
 		log_ptr += 2;
diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc
index e011b3f5d8e..d7ec2a38f19 100644
--- a/storage/innobase/trx/trx0rec.cc
+++ b/storage/innobase/trx/trx0rec.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2019, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2021, MariaDB Corporation.
+Copyright (c) 2017, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1867,9 +1867,9 @@ trx_undo_page_report_rename(trx_t* trx, const dict_table_t* table,
 	byte* start = block->frame + first_free;
 	size_t len = strlen(table->name.m_name);
 	const size_t fixed = 2 + 1 + 11 + 11 + 2;
-	ut_ad(len <= NAME_LEN * 2 + 1);
+	ut_ad(len <= NAME_CHAR_LEN * 5 * 2 + 1);
 	/* The -10 is used in trx_undo_left() */
-	compile_time_assert((NAME_LEN * 1) * 2 + fixed
+	compile_time_assert(NAME_CHAR_LEN * 5 * 2 + fixed
 			    + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE
 			    < UNIV_PAGE_SIZE_MIN - 10 - FIL_PAGE_DATA_END);