MDEV-18649 Assertion supremum[7] == index.n_core_null_bytes failed

This is follow-up to MDEV-18048: Relax a too strict debug assertion

This assertion should have been relaxed when implementing the first part of
MDEV-15563: instant removal of NOT NULL attribute for ROW_FORMAT=REDUNDANT
tables.

For ROW_FORMAT=REDUNDANT, there is no bitmap of null columns;
the null flags are encoded in the end offset of each field.
We do not really care about the number of fields that can be NULL.

3 files changed, 32 insertions, 2 deletions

diff --git a/mysql-test/suite/innodb/r/instant_alter.result b/mysql-test/suite/innodb/r/instant_alter.result
index 8753977c0b8..c180e107b70 100644
--- a/mysql-test/suite/innodb/r/instant_alter.result
+++ b/mysql-test/suite/innodb/r/instant_alter.result
@@ -702,6 +702,12 @@ CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
 e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ADD COLUMN b INT;
@@ -1511,6 +1517,12 @@ CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
 e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=COMPACT;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=COMPACT;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ADD COLUMN b INT;
@@ -2320,6 +2332,12 @@ CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
 e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ADD COLUMN b INT;
@@ -2486,5 +2504,5 @@ SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-174
+178
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
diff --git a/mysql-test/suite/innodb/t/instant_alter.test b/mysql-test/suite/innodb/t/instant_alter.test
index 99f49300b2e..fbee8b79fa3 100644
--- a/mysql-test/suite/innodb/t/instant_alter.test
+++ b/mysql-test/suite/innodb/t/instant_alter.test
@@ -587,6 +587,15 @@ e INT, f INT, g INT, h INT, j INT) $engine;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
 
+# MDEV-18649 Failing assertion on ALTER for ROW_FORMAT=REDUNDANT
+eval CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) $engine;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+# Exploit MDEV-17468 to force the table definition to be reloaded
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
+
 # MDEV-18033/MDEV-18034 Failing assertion on ALTER
 eval CREATE TABLE t1 (a INT NOT NULL) $engine;
 INSERT INTO t1 VALUES (1);
diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc
index 29696f96aa0..56a1fb8d231 100644
--- a/storage/innobase/btr/btr0btr.cc
+++ b/storage/innobase/btr/btr0btr.cc
@@ -1933,7 +1933,10 @@ void btr_set_instant(buf_block_t* root, const dict_index_t& index, mtr_t* mtr)
 			ut_ad(index.table->instant);
 			ut_ad(!memcmp(infimum, field_ref_zero, 8));
 			ut_ad(!memcmp(supremum, field_ref_zero, 7));
-			ut_ad(supremum[7] == index.n_core_null_bytes);
+			/* The n_core_null_bytes only matters for
+			ROW_FORMAT=COMPACT and ROW_FORMAT=DYNAMIC tables. */
+			ut_ad(supremum[7] == index.n_core_null_bytes
+			      || !index.table->not_redundant());
 			return;
 		}
 		break;