MDEV-18596 Crash in row_mysql_store_col_in_innobase_format() on MODIFY/ADD column

innobase_build_col_map_add(): Do not assume that old_field->pack_length()
equals to field->pack_length(). Fix submitted by Aleksey Midenkov.

innobase_instant_try(): Assert that the column length of fixed-length
NOT NULL columns is only changing for ROW_FORMAT=REDUNDANT.

3 files changed, 203 insertions, 11 deletions

diff --git a/mysql-test/suite/innodb/r/instant_alter.result b/mysql-test/suite/innodb/r/instant_alter.result
index d6fd90ec27f..14b7aeece25 100644
--- a/mysql-test/suite/innodb/r/instant_alter.result
+++ b/mysql-test/suite/innodb/r/instant_alter.result
@@ -795,6 +795,58 @@ SELECT HEX(c) FROM t1;
 HEX(c)
 1
 DROP TABLE t1;
+CREATE TABLE t1 (c VARCHAR(10) NOT NULL DEFAULT 'scary') ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1() VALUES();
+ALTER TABLE t1 ADD f TINYINT NOT NULL DEFAULT -42;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+ALTER TABLE t1 MODIFY f MEDIUMINT NOT NULL DEFAULT 64802,
+MODIFY c VARCHAR(20) NOT NULL DEFAULT 'gory',
+ADD d DATETIME;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+INSERT INTO t1() VALUES();
+INSERT INTO t1 (c,f,d) VALUES ('fury', -8388608, now());
+SELECT * FROM t1;
+c	f	d
+scary	-42	NULL
+gory	64802	NULL
+fury	-8388608	1970-01-01 03:00:42
+DROP TABLE t1;
+CREATE TABLE t1 (t TINYINT PRIMARY KEY, m MEDIUMINT UNIQUE) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+SELECT table_id INTO @table_id1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+INSERT INTO t1 VALUES (-42, -123456);
+ALTER TABLE t1 CHANGE t s SMALLINT;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+SELECT table_id INTO @table_id2 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+affected rows: 1
+ALTER TABLE t1 CHANGE m i INT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: ADD INDEX. Try ALGORITHM=NOCOPY
+ALTER TABLE t1 CHANGE m i INT;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+SELECT table_id INTO @table_id3 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+affected rows: 1
+SELECT @table_id1 = @table_id2, @table_id2 = @table_id3;
+@table_id1 = @table_id2	@table_id2 = @table_id3
+0	1
+INSERT IGNORE INTO t1 VALUES (0, -123456);
+REPLACE INTO t1 VALUES(-42, 123456);
+INSERT IGNORE INTO t1 VALUES(32768, 2147483648);
+Warnings:
+Warning	1264	Out of range value for column 's' at row 1
+Warning	1264	Out of range value for column 'i' at row 1
+SELECT * FROM t1;
+s	i
+-42	-123456
+0	-123456
+-42	123456
+32767	2147483647
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1536,6 +1588,58 @@ SELECT HEX(c) FROM t1;
 HEX(c)
 1
 DROP TABLE t1;
+CREATE TABLE t1 (c VARCHAR(10) NOT NULL DEFAULT 'scary') ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1() VALUES();
+ALTER TABLE t1 ADD f TINYINT NOT NULL DEFAULT -42;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+ALTER TABLE t1 MODIFY f MEDIUMINT NOT NULL DEFAULT 64802,
+MODIFY c VARCHAR(20) NOT NULL DEFAULT 'gory',
+ADD d DATETIME;
+affected rows: 1
+info: Records: 1  Duplicates: 0  Warnings: 0
+INSERT INTO t1() VALUES();
+INSERT INTO t1 (c,f,d) VALUES ('fury', -8388608, now());
+SELECT * FROM t1;
+c	f	d
+scary	-42	NULL
+gory	64802	NULL
+fury	-8388608	1970-01-01 03:00:42
+DROP TABLE t1;
+CREATE TABLE t1 (t TINYINT PRIMARY KEY, m MEDIUMINT UNIQUE) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+SELECT table_id INTO @table_id1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+INSERT INTO t1 VALUES (-42, -123456);
+ALTER TABLE t1 CHANGE t s SMALLINT;
+affected rows: 1
+info: Records: 1  Duplicates: 0  Warnings: 0
+SELECT table_id INTO @table_id2 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+affected rows: 1
+ALTER TABLE t1 CHANGE m i INT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+ALTER TABLE t1 CHANGE m i INT;
+affected rows: 1
+info: Records: 1  Duplicates: 0  Warnings: 0
+SELECT table_id INTO @table_id3 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+affected rows: 1
+SELECT @table_id1 = @table_id2, @table_id2 = @table_id3;
+@table_id1 = @table_id2	@table_id2 = @table_id3
+0	0
+INSERT IGNORE INTO t1 VALUES (0, -123456);
+Warnings:
+Warning	1062	Duplicate entry '-123456' for key 'm'
+REPLACE INTO t1 VALUES(-42, 123456);
+INSERT IGNORE INTO t1 VALUES(32768, 2147483648);
+Warnings:
+Warning	1264	Out of range value for column 's' at row 1
+Warning	1264	Out of range value for column 'i' at row 1
+SELECT * FROM t1;
+s	i
+-42	123456
+32767	2147483647
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -2277,10 +2381,62 @@ SELECT HEX(c) FROM t1;
 HEX(c)
 1
 DROP TABLE t1;
+CREATE TABLE t1 (c VARCHAR(10) NOT NULL DEFAULT 'scary') ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1() VALUES();
+ALTER TABLE t1 ADD f TINYINT NOT NULL DEFAULT -42;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+ALTER TABLE t1 MODIFY f MEDIUMINT NOT NULL DEFAULT 64802,
+MODIFY c VARCHAR(20) NOT NULL DEFAULT 'gory',
+ADD d DATETIME;
+affected rows: 1
+info: Records: 1  Duplicates: 0  Warnings: 0
+INSERT INTO t1() VALUES();
+INSERT INTO t1 (c,f,d) VALUES ('fury', -8388608, now());
+SELECT * FROM t1;
+c	f	d
+scary	-42	NULL
+gory	64802	NULL
+fury	-8388608	1970-01-01 03:00:42
+DROP TABLE t1;
+CREATE TABLE t1 (t TINYINT PRIMARY KEY, m MEDIUMINT UNIQUE) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+SELECT table_id INTO @table_id1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+INSERT INTO t1 VALUES (-42, -123456);
+ALTER TABLE t1 CHANGE t s SMALLINT;
+affected rows: 1
+info: Records: 1  Duplicates: 0  Warnings: 0
+SELECT table_id INTO @table_id2 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+affected rows: 1
+ALTER TABLE t1 CHANGE m i INT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type INPLACE. Try ALGORITHM=COPY
+ALTER TABLE t1 CHANGE m i INT;
+affected rows: 1
+info: Records: 1  Duplicates: 0  Warnings: 0
+SELECT table_id INTO @table_id3 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+affected rows: 1
+SELECT @table_id1 = @table_id2, @table_id2 = @table_id3;
+@table_id1 = @table_id2	@table_id2 = @table_id3
+0	0
+INSERT IGNORE INTO t1 VALUES (0, -123456);
+Warnings:
+Warning	1062	Duplicate entry '-123456' for key 'm'
+REPLACE INTO t1 VALUES(-42, 123456);
+INSERT IGNORE INTO t1 VALUES(32768, 2147483648);
+Warnings:
+Warning	1264	Out of range value for column 's' at row 1
+Warning	1264	Out of range value for column 'i' at row 1
+SELECT * FROM t1;
+s	i
+-42	123456
+32767	2147483647
+DROP TABLE t1;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-171
+175
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
diff --git a/mysql-test/suite/innodb/t/instant_alter.test b/mysql-test/suite/innodb/t/instant_alter.test
index 013b2483f54..c36b5801b1e 100644
--- a/mysql-test/suite/innodb/t/instant_alter.test
+++ b/mysql-test/suite/innodb/t/instant_alter.test
@@ -681,6 +681,40 @@ ALTER TABLE t1 CHANGE b c BIT NOT NULL;
 SELECT HEX(c) FROM t1;
 DROP TABLE t1;
 
+eval CREATE TABLE t1 (c VARCHAR(10) NOT NULL DEFAULT 'scary') $engine;
+INSERT INTO t1() VALUES();
+--enable_info
+ALTER TABLE t1 ADD f TINYINT NOT NULL DEFAULT -42;
+ALTER TABLE t1 MODIFY f MEDIUMINT NOT NULL DEFAULT 64802,
+MODIFY c VARCHAR(20) NOT NULL DEFAULT 'gory',
+ADD d DATETIME;
+--disable_info
+INSERT INTO t1() VALUES();
+INSERT INTO t1 (c,f,d) VALUES ('fury', -8388608, now());
+SELECT * FROM t1;
+DROP TABLE t1;
+
+eval CREATE TABLE t1 (t TINYINT PRIMARY KEY, m MEDIUMINT UNIQUE) $engine;
+SELECT table_id INTO @table_id1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+INSERT INTO t1 VALUES (-42, -123456);
+--enable_info
+ALTER TABLE t1 CHANGE t s SMALLINT;
+SELECT table_id INTO @table_id2 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 CHANGE m i INT, ALGORITHM=INSTANT;
+ALTER TABLE t1 CHANGE m i INT;
+SELECT table_id INTO @table_id3 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
+WHERE name = 'test/t1';
+--disable_info
+SELECT @table_id1 = @table_id2, @table_id2 = @table_id3;
+INSERT IGNORE INTO t1 VALUES (0, -123456);
+REPLACE INTO t1 VALUES(-42, 123456);
+INSERT IGNORE INTO t1 VALUES(32768, 2147483648);
+SELECT * FROM t1;
+DROP TABLE t1;
+
 dec $format;
 }
 disconnect analyze;
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index b0f87fdae9e..8612aa9404b 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -4122,7 +4122,7 @@ innobase_check_foreigns(
 @param[in,out]	heap		Memory heap where allocated
 @param[out]	dfield		InnoDB data field to copy to
 @param[in]	field		MySQL value for the column
-@param[in]	old_field	Old field or NULL if new col is added	
+@param[in]	old_field	Old column if altering; NULL for ADD COLUMN
 @param[in]	comp		nonzero if in compact format. */
 static void innobase_build_col_map_add(
 	mem_heap_t*	heap,
@@ -4141,14 +4141,13 @@ static void innobase_build_col_map_add(
 		return;
 	}
 
-	ulint	size	= field->pack_length();
+	const Field& from = old_field ? *old_field : *field;
+	ulint	size	= from.pack_length();
 
 	byte*	buf	= static_cast<byte*>(mem_heap_alloc(heap, size));
 
-	const byte*	mysql_data = old_field ? old_field->ptr : field->ptr;
-
 	row_mysql_store_col_in_innobase_format(
-		dfield, buf, true, mysql_data, size, comp);
+		dfield, buf, true, from.ptr, size, comp);
 }
 
 /** Construct the translation table for reordering, dropping or
@@ -5547,6 +5546,11 @@ static bool innobase_instant_try(
 				    dict_table_get_col_name(user_table, i)));
 		DBUG_ASSERT(old || col->is_added());
 
+		ut_d(const Create_field* new_field = cf_it++);
+		/* new_field->field would point to an existing column.
+		If it is NULL, the column was added by this ALTER TABLE. */
+		ut_ad(!new_field->field == !old);
+
 		if (col->is_added()) {
 			dfield_set_data(d, col->def_val.data,
 					col->def_val.len);
@@ -5580,14 +5584,12 @@ static bool innobase_instant_try(
 						mem_heap_alloc(ctx->heap, len))
 					: NULL, true, (*af)->ptr, len,
 					dict_table_is_comp(user_table));
+				ut_ad(new_field->field->pack_length() == len
+				      || !user_table->not_redundant());
+
 			}
 		}
 
-		ut_d(const Create_field* new_field = cf_it++);
-		/* new_field->field would point to an existing column.
-		If it is NULL, the column was added by this ALTER TABLE. */
-		ut_ad(!new_field->field == !old);
-
 		bool update = old && (!ctx->first_alter_pos
 				      || i < ctx->first_alter_pos - 1);
 		ut_ad(!old || col->same_format(