MDEV-20590 Introduce a file format constraint to ALTER TABLEbb-10.3-MDEV-20590

If a table is altered using the MDEV-11369/MDEV-15562/MDEV-13134
ALGORITHM=INSTANT, it can force the table to use a non-canonical
format:

* A hidden metadata record at the start of the clustered index
is used to store each column's DEFAULT value. This makes it possible
to add new columns that have default values without rebuilding the table.

* Starting with MDEV-15562 in MariaDB Server 10.4, a BLOB in the
hidden metadata record is used to store column mappings. This makes
it possible to drop or reorder columns without rebuilding the table.
This also makes it possible to add columns to any position or drop
columns from any position in the table without rebuilding the table.

If a column is dropped without rebuilding the table, old records
will contain garbage in that column's former position, and new records
will be written with NULL values, empty strings, or dummy values.

This is generally not a problem. However, there may be cases where
users may want to avoid putting a table into this format.
For example, users may want to ensure that future UPDATE operations
after an ADD COLUMN will be performed in-place, to reduce write
amplification. (Instantly added columns are essentially always
variable-length.) Users might also want to avoid bugs similar to
MDEV-19916, or they may want to be able to export tables to
older versions of the server.

We will revive the previously deprecated option innodb_file_format,
with the following values:

* barracuda (0): do not allow instant add/drop/reorder,
to maintain format compatibility with MariaDB 10.x and MySQL 5.x

* strict_barracuda: Like 'barracuda', but on ALTER TABLE, if a table
(or partition) is not in the canonical format, force a table rebuild.

* append (default in 10.3): Store a hidden metadata record that
allows columns to be appended to the table (MDEV-11369);
compatible with MariaDB 10.3

* strict_append: Like 'append', but on ALTER TABLE, if a table
(or partition) is not in the 10.3-compatible format,
force the table to be rebuilt. (On 10.3, this is the same as 'append'.)

Starting with 10.4:

* mapped (default): Like 'append', but allow the metadata record to
store a column map, to support instant add/drop/reorder (MDEV-15562)
* strict_mapped: Like 'mapped', but on ALTER TABLE, if a table
(or partition) is not in the 10.4-compatible format,
force the table to be rebuilt. (On 10.4, this is the same as 'mapped'.)

8 files changed, 280 insertions, 68 deletions

diff --git a/mysql-test/suite/innodb/r/instant_alter,4k.rdiff b/mysql-test/suite/innodb/r/instant_alter,4k.rdiff
index cb65d12b09f..695638efdfc 100644
--- a/mysql-test/suite/innodb/r/instant_alter,4k.rdiff
+++ b/mysql-test/suite/innodb/r/instant_alter,4k.rdiff
@@ -1,6 +1,6 @@
 --- instant_alter.result
 +++ instant_alter,4k.result
-@@ -241,7 +241,7 @@
+@@ -243,7 +243,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -9,7 +9,7 @@
  connection default;
  ROLLBACK;
  connection analyze;
-@@ -251,7 +251,7 @@
+@@ -253,7 +253,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -18,7 +18,7 @@
  connection default;
  BEGIN;
  UPDATE t2 SET d1 = repeat(id, 200);
-@@ -262,7 +262,7 @@
+@@ -264,7 +264,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -27,7 +27,7 @@
  connection default;
  ROLLBACK;
  connection analyze;
-@@ -272,7 +272,7 @@
+@@ -274,7 +274,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -36,7 +36,7 @@
  connection default;
  ALTER TABLE t2 DROP p;
  affected rows: 0
-@@ -320,8 +320,14 @@
+@@ -322,8 +322,14 @@
  affected rows: 0
  info: Records: 0  Duplicates: 0  Warnings: 0
  ALTER TABLE t3 ADD COLUMN b BLOB NOT NULL;
@@ -52,7 +52,7 @@
  INSERT INTO t3 SET id=4;
  ERROR HY000: Field 'c2' doesn't have a default value
  INSERT INTO t3 SET id=4, c2=0, b=0xf09f98b1;
-@@ -334,7 +340,9 @@
+@@ -336,7 +342,9 @@
  ALTER TABLE t3 CHANGE t phrase TEXT DEFAULT 0xc3a4c3a448,
  CHANGE b b BLOB NOT NULL DEFAULT 'binary line of business';
  affected rows: 4
@@ -63,7 +63,7 @@
  INSERT INTO t3 SET id=5, c2=9;
  Warnings:
  Note	1265	Data truncated for column 'c7' at row 1
-@@ -348,7 +356,9 @@
+@@ -350,7 +358,9 @@
  5	9	POLYGON((1 1,2 2,3 3,1 1))	1970-01-01 03:00:42	1970-01-01 03:00:42	NULL	03:00:42	1970-01-01	ääH	binary line of business
  ALTER TABLE t3 DROP c3, DROP c7;
  affected rows: 0
@@ -74,7 +74,7 @@
  SELECT * FROM t3;
  id	c2	c4	c5	c6	c8	phrase	b
  1	1	1970-01-01 03:00:42	1970-01-01 03:00:42	NULL	1970-01-01	The quick brown fox jumps over the lazy dog	
-@@ -376,6 +386,8 @@
+@@ -378,6 +388,8 @@
  (id INT PRIMARY KEY, c1 VARCHAR(4000), c2 VARCHAR(4000), c3 VARCHAR(1000),
  p POINT NOT NULL DEFAULT ST_GeomFromText('POINT(0 0)'), SPATIAL INDEX(p))
  ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
@@ -83,7 +83,7 @@
  BEGIN;
  INSERT INTO big
  SET id=1, c1=REPEAT('a', 200), c2=REPEAT('b', 200), c3=REPEAT('c', 159);
-@@ -393,13 +405,15 @@
+@@ -395,13 +407,15 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -101,7 +101,7 @@
  CHECKSUM TABLE big;
  Table	Checksum
  test.big	1705165209
-@@ -416,7 +430,7 @@
+@@ -418,7 +432,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -110,7 +110,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -429,7 +443,7 @@
+@@ -431,7 +445,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -119,16 +119,7 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,t4,big;
-@@ -515,6 +529,8 @@
- CREATE TABLE t1 (a INT, b VARCHAR(500), c TEXT, UNIQUE(a,b)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
- ALTER TABLE t1 ADD d TEXT;
- ALTER TABLE t1 ADD PRIMARY KEY (a,b);
-+Warnings:
-+Warning	139	Row size too large (> 1979). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline.
- ALTER TABLE t1 ADD va INT AS (a) VIRTUAL;
- DROP TABLE t1;
- CREATE TABLE t1
-@@ -703,7 +719,7 @@
+@@ -730,7 +744,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -137,7 +128,7 @@
  connection default;
  ROLLBACK;
  connection analyze;
-@@ -713,7 +729,7 @@
+@@ -740,7 +754,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -146,7 +137,7 @@
  connection default;
  BEGIN;
  UPDATE t2 SET d1 = repeat(id, 200);
-@@ -724,7 +740,7 @@
+@@ -751,7 +765,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -155,7 +146,7 @@
  connection default;
  ROLLBACK;
  connection analyze;
-@@ -734,7 +750,7 @@
+@@ -761,7 +775,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -164,7 +155,7 @@
  connection default;
  ALTER TABLE t2 DROP p;
  affected rows: 0
-@@ -783,7 +799,9 @@
+@@ -810,7 +824,9 @@
  info: Records: 0  Duplicates: 0  Warnings: 0
  ALTER TABLE t3 ADD COLUMN b BLOB NOT NULL;
  affected rows: 0
@@ -175,7 +166,7 @@
  INSERT INTO t3 SET id=4;
  ERROR HY000: Field 'c2' doesn't have a default value
  INSERT INTO t3 SET id=4, c2=0, b=0xf09f98b1;
-@@ -796,7 +814,9 @@
+@@ -823,7 +839,9 @@
  ALTER TABLE t3 CHANGE t phrase TEXT DEFAULT 0xc3a4c3a448,
  CHANGE b b BLOB NOT NULL DEFAULT 'binary line of business';
  affected rows: 4
@@ -186,7 +177,7 @@
  INSERT INTO t3 SET id=5, c2=9;
  Warnings:
  Note	1265	Data truncated for column 'c7' at row 1
-@@ -810,7 +830,9 @@
+@@ -837,7 +855,9 @@
  5	9	POLYGON((1 1,2 2,3 3,1 1))	1970-01-01 03:00:42	1970-01-01 03:00:42	NULL	03:00:42	1970-01-01	ääH	binary line of business
  ALTER TABLE t3 DROP c3, DROP c7;
  affected rows: 0
@@ -197,7 +188,7 @@
  SELECT * FROM t3;
  id	c2	c4	c5	c6	c8	phrase	b
  1	1	1970-01-01 03:00:42	1970-01-01 03:00:42	NULL	1970-01-01	The quick brown fox jumps over the lazy dog	
-@@ -838,6 +860,8 @@
+@@ -865,6 +885,8 @@
  (id INT PRIMARY KEY, c1 VARCHAR(4000), c2 VARCHAR(4000), c3 VARCHAR(1000),
  p POINT NOT NULL DEFAULT ST_GeomFromText('POINT(0 0)'), SPATIAL INDEX(p))
  ENGINE=InnoDB ROW_FORMAT=COMPACT;
@@ -206,7 +197,7 @@
  BEGIN;
  INSERT INTO big
  SET id=1, c1=REPEAT('a', 200), c2=REPEAT('b', 200), c3=REPEAT('c', 159);
-@@ -855,13 +879,15 @@
+@@ -882,13 +904,15 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -224,7 +215,7 @@
  CHECKSUM TABLE big;
  Table	Checksum
  test.big	1705165209
-@@ -878,7 +904,7 @@
+@@ -905,7 +929,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -233,7 +224,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -891,7 +917,7 @@
+@@ -918,7 +942,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -242,16 +233,7 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,t4,big;
-@@ -977,6 +1003,8 @@
- CREATE TABLE t1 (a INT, b VARCHAR(500), c TEXT, UNIQUE(a,b)) ENGINE=InnoDB ROW_FORMAT=COMPACT;
- ALTER TABLE t1 ADD d TEXT;
- ALTER TABLE t1 ADD PRIMARY KEY (a,b);
-+Warnings:
-+Warning	139	Row size too large (> 1982). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline.
- ALTER TABLE t1 ADD va INT AS (a) VIRTUAL;
- DROP TABLE t1;
- CREATE TABLE t1
-@@ -1165,7 +1193,7 @@
+@@ -1217,7 +1241,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -260,7 +242,7 @@
  connection default;
  ROLLBACK;
  connection analyze;
-@@ -1175,7 +1203,7 @@
+@@ -1227,7 +1251,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -269,7 +251,7 @@
  connection default;
  BEGIN;
  UPDATE t2 SET d1 = repeat(id, 200);
-@@ -1186,7 +1214,7 @@
+@@ -1238,7 +1262,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -278,7 +260,7 @@
  connection default;
  ROLLBACK;
  connection analyze;
-@@ -1196,7 +1224,7 @@
+@@ -1248,7 +1272,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/t2';
  clust_index_size
@@ -287,7 +269,7 @@
  connection default;
  ALTER TABLE t2 DROP p;
  affected rows: 0
-@@ -1317,7 +1345,7 @@
+@@ -1369,7 +1393,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -296,7 +278,7 @@
  connection default;
  ALTER TABLE big ADD COLUMN
  (d1 INT DEFAULT 0, d2 VARCHAR(20) DEFAULT 'abcde',
-@@ -1340,7 +1368,7 @@
+@@ -1392,7 +1416,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -305,7 +287,7 @@
  connection default;
  ROLLBACK;
  CHECKSUM TABLE big;
-@@ -1353,7 +1381,7 @@
+@@ -1405,7 +1429,7 @@
  SELECT clust_index_size FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESTATS
  WHERE name = 'test/big';
  clust_index_size
@@ -314,10 +296,11 @@
  connection default;
  InnoDB		0 transactions not purged
  DROP TABLE t1,t2,t3,t4,big;
-@@ -1446,5 +1474,5 @@
+@@ -1523,6 +1547,6 @@
  FROM information_schema.global_status
  WHERE variable_name = 'innodb_instant_alter_column';
  instants
--57
-+58
+-63
++64
  SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
+ SET GLOBAL innodb_file_format = @saved_format;
diff --git a/mysql-test/suite/innodb/r/instant_alter.result b/mysql-test/suite/innodb/r/instant_alter.result
index aadb7021887..9fb49e160ee 100644
--- a/mysql-test/suite/innodb/r/instant_alter.result
+++ b/mysql-test/suite/innodb/r/instant_alter.result
@@ -1,6 +1,8 @@
 #
 # MDEV-11369: Instant ADD COLUMN for InnoDB
 #
+SET @saved_format = @@GLOBAL.innodb_file_format;
+SET GLOBAL innodb_file_format = append;
 call mtr.add_suppression("Cannot add field `.*` in table `test`.`.*` because after adding it, the row size is");
 CREATE TABLE t(a INT UNIQUE)ENGINE=InnoDB ROW_FORMAT=COMPACT;
 ALTER TABLE t ADD e INT, ROW_FORMAT=COMPRESSED;
@@ -519,6 +521,29 @@ ALTER TABLE t1 ADD PRIMARY KEY (b,a);
 ALTER TABLE t1 ADD va INT AS (a) VIRTUAL;
 DROP TABLE t1;
 SET innodb_strict_mode = OFF;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 SET a=42;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=barracuda. Try ALGORITHM=INPLACE
+SET GLOBAL innodb_file_format = strict_barracuda;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=strict_barracuda. Try ALGORITHM=INPLACE
+SET GLOBAL innodb_file_format = strict_append;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = append;
+ALTER TABLE t1 ADD c TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 1, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = strict_barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 0, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=strict_barracuda. Try ALGORITHM=INPLACE
+ALTER TABLE t1 MODIFY a INT DEFAULT 0;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+ALTER TABLE t1 MODIFY a INT DEFAULT NULL, ALGORITHM=INSTANT;
+DROP TABLE t1;
+SET GLOBAL innodb_file_format = @saved_format;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -983,6 +1008,29 @@ ALTER TABLE t1 ADD PRIMARY KEY (b,a);
 ALTER TABLE t1 ADD va INT AS (a) VIRTUAL;
 DROP TABLE t1;
 SET innodb_strict_mode = OFF;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1 SET a=42;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=barracuda. Try ALGORITHM=INPLACE
+SET GLOBAL innodb_file_format = strict_barracuda;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=strict_barracuda. Try ALGORITHM=INPLACE
+SET GLOBAL innodb_file_format = strict_append;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = append;
+ALTER TABLE t1 ADD c TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 1, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = strict_barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 0, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=strict_barracuda. Try ALGORITHM=INPLACE
+ALTER TABLE t1 MODIFY a INT DEFAULT 0;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+ALTER TABLE t1 MODIFY a INT DEFAULT NULL, ALGORITHM=INSTANT;
+DROP TABLE t1;
+SET GLOBAL innodb_file_format = @saved_format;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1447,10 +1495,34 @@ ALTER TABLE t1 ADD PRIMARY KEY (b,a);
 ALTER TABLE t1 ADD va INT AS (a) VIRTUAL;
 DROP TABLE t1;
 SET innodb_strict_mode = OFF;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 SET a=42;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=barracuda. Try ALGORITHM=INPLACE
+SET GLOBAL innodb_file_format = strict_barracuda;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=strict_barracuda. Try ALGORITHM=INPLACE
+SET GLOBAL innodb_file_format = strict_append;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = append;
+ALTER TABLE t1 ADD c TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 1, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = strict_barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 0, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: innodb_file_format=strict_barracuda. Try ALGORITHM=INPLACE
+ALTER TABLE t1 MODIFY a INT DEFAULT 0;
+affected rows: 0
+info: Records: 0  Duplicates: 0  Warnings: 0
+ALTER TABLE t1 MODIFY a INT DEFAULT NULL, ALGORITHM=INSTANT;
+DROP TABLE t1;
+SET GLOBAL innodb_file_format = @saved_format;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-57
+63
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
+SET GLOBAL innodb_file_format = @saved_format;
diff --git a/mysql-test/suite/innodb/t/instant_alter.test b/mysql-test/suite/innodb/t/instant_alter.test
index be79d7437c3..39e47a6ec91 100644
--- a/mysql-test/suite/innodb/t/instant_alter.test
+++ b/mysql-test/suite/innodb/t/instant_alter.test
@@ -4,6 +4,9 @@
 --echo # MDEV-11369: Instant ADD COLUMN for InnoDB
 --echo #
 
+SET @saved_format = @@GLOBAL.innodb_file_format;
+SET GLOBAL innodb_file_format = append;
+
 call mtr.add_suppression("Cannot add field `.*` in table `test`.`.*` because after adding it, the row size is");
 
 let $format= `SELECT CASE WHEN @@GLOBAL.innodb_page_size>16384
@@ -408,6 +411,31 @@ ALTER TABLE t1 ADD va INT AS (a) VIRTUAL;
 DROP TABLE t1;
 SET innodb_strict_mode = OFF;
 
+# MDEV-20950 innodb_file_format
+eval CREATE TABLE t1 (a INT PRIMARY KEY) $engine;
+INSERT INTO t1 SET a=42;
+SET GLOBAL innodb_file_format = barracuda;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = strict_barracuda;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = strict_append;
+ALTER TABLE t1 ADD b TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = append;
+ALTER TABLE t1 ADD c TEXT, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = barracuda;
+ALTER TABLE t1 MODIFY a INT DEFAULT 1, ALGORITHM=INSTANT;
+SET GLOBAL innodb_file_format = strict_barracuda;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 MODIFY a INT DEFAULT 0, ALGORITHM=INSTANT;
+--enable_info
+ALTER TABLE t1 MODIFY a INT DEFAULT 0;
+--disable_info
+ALTER TABLE t1 MODIFY a INT DEFAULT NULL, ALGORITHM=INSTANT;
+DROP TABLE t1;
+SET GLOBAL innodb_file_format = @saved_format;
+
 dec $format;
 let $redundant_4k= 0;
 }
@@ -416,3 +444,4 @@ SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;
+SET GLOBAL innodb_file_format = @saved_format;
diff --git a/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result b/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result
new file mode 100644
index 00000000000..2eec95c4a19
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result
@@ -0,0 +1,51 @@
+SET @start_global_value = @@global.innodb_file_format;
+SET GLOBAL innodb_file_format=Barracuda;
+select @@session.innodb_file_format;
+ERROR HY000: Variable 'innodb_file_format' is a GLOBAL variable
+show global variables like 'innodb_file_format';
+Variable_name	Value
+innodb_file_format	barracuda
+show session variables like 'innodb_file_format';
+Variable_name	Value
+innodb_file_format	barracuda
+select * from information_schema.global_variables where variable_name='innodb_file_format';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_FILE_FORMAT	barracuda
+select * from information_schema.session_variables where variable_name='innodb_file_format';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_FILE_FORMAT	barracuda
+set global innodb_file_format='Antelope';
+ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'Antelope'
+set global innodb_file_format=1.1;
+ERROR 42000: Incorrect argument type to variable 'innodb_file_format'
+set global innodb_file_format=-1;
+ERROR 42000: Variable 'innodb_file_format' can't be set to the value of '-1'
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+barracuda
+set global innodb_file_format=4;
+ERROR 42000: Variable 'innodb_file_format' can't be set to the value of '4'
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+barracuda
+set global innodb_file_format=3;
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+strict_append
+set global innodb_file_format=2;
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+append
+set global innodb_file_format=1;
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+strict_barracuda
+set global innodb_file_format=0;
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+barracuda
+set global innodb_file_format=default;
+select @@global.innodb_file_format;
+@@global.innodb_file_format
+append
+SET GLOBAL innodb_file_format = @start_global_value;
diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
index 8d39f6c14b6..6b1253b553b 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
@@ -815,15 +815,15 @@ READ_ONLY	YES
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	INNODB_FILE_FORMAT
 SESSION_VALUE	NULL
-DEFAULT_VALUE	
+DEFAULT_VALUE	add
 VARIABLE_SCOPE	GLOBAL
-VARIABLE_TYPE	VARCHAR
-VARIABLE_COMMENT	Deprecated parameter with no effect.
+VARIABLE_TYPE	ENUM
+VARIABLE_COMMENT	File format constraint for native ALTER TABLE
 NUMERIC_MIN_VALUE	NULL
 NUMERIC_MAX_VALUE	NULL
 NUMERIC_BLOCK_SIZE	NULL
-ENUM_VALUE_LIST	NULL
-READ_ONLY	YES
+ENUM_VALUE_LIST	barracuda,strict_barracuda,add,strict_add
+READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	INNODB_FILE_PER_TABLE
 SESSION_VALUE	NULL
diff --git a/mysql-test/suite/sys_vars/t/innodb_file_format_basic.test b/mysql-test/suite/sys_vars/t/innodb_file_format_basic.test
new file mode 100644
index 00000000000..32a06d77f83
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/innodb_file_format_basic.test
@@ -0,0 +1,34 @@
+--source include/have_innodb.inc
+
+SET @start_global_value = @@global.innodb_file_format;
+SET GLOBAL innodb_file_format=Barracuda;
+
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.innodb_file_format;
+show global variables like 'innodb_file_format';
+show session variables like 'innodb_file_format';
+select * from information_schema.global_variables where variable_name='innodb_file_format';
+select * from information_schema.session_variables where variable_name='innodb_file_format';
+
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_file_format='Antelope';
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_file_format=1.1;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_file_format=-1;
+select @@global.innodb_file_format;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_file_format=4;
+select @@global.innodb_file_format;
+set global innodb_file_format=3;
+select @@global.innodb_file_format;
+set global innodb_file_format=2;
+select @@global.innodb_file_format;
+set global innodb_file_format=1;
+select @@global.innodb_file_format;
+set global innodb_file_format=0;
+select @@global.innodb_file_format;
+set global innodb_file_format=default;
+select @@global.innodb_file_format;
+
+SET GLOBAL innodb_file_format = @start_global_value;
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 79051b9c247..0fd130a866e 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -204,8 +204,8 @@ static char*	innobase_reset_all_monitor_counter;
 
 static ulong	innodb_flush_method;
 
-/** Deprecated; no effect other than issuing a deprecation warning. */
-static char* innodb_file_format;
+/** File format constraint for native ALTER TABLE */
+ulong innodb_file_format;
 /** Deprecated; no effect other than issuing a deprecation warning. */
 static char* innodb_large_prefix;
 
@@ -474,6 +474,23 @@ static TYPELIB innodb_change_buffering_typelib = {
 	NULL
 };
 
+/** Allowed values of innodb_file_format */
+const char* innodb_file_format_names[] = {
+	"barracuda", /* compatible with MariaDB 5.5 to 10.2 */
+	"strict_barracuda", /* force rebuild on ALTER TABLE if needed */
+	"append",/* allow instant ADD COLUMN */
+	"strict_append", /* ditto; but maybe require rebuild in 10.4 */
+	NullS
+};
+
+/** Enumeration of innodb_file_format */
+static TYPELIB innodb_file_format_typelib = {
+	array_elements(innodb_file_format_names) - 1,
+	"innodb_file_format_typelib",
+	innodb_file_format_names,
+	NULL
+};
+
 /** Retrieve the FTS Relevance Ranking result for doc with doc_id
 of m_prebuilt->fts_doc_id
 @param[in,out]	fts_hdl	FTS handler
@@ -3701,15 +3718,12 @@ static int innodb_init_params()
 	char		*default_path;
 	ulong		num_pll_degree;
 
-	if (innodb_large_prefix || innodb_file_format) {
-		const char* p = innodb_file_format
-			? "file_format"
-			: "large_prefix";
-		sql_print_warning("The parameter innodb_%s is deprecated"
-				  " and has no effect."
+	if (innodb_large_prefix) {
+		sql_print_warning("The parameter innodb_large_prefix"
+				  " is deprecated and has no effect."
 				  " It may be removed in future releases."
 				  " See https://mariadb.com/kb/en/library/"
-				  "xtradbinnodb-file-format/", p);
+				  "xtradbinnodb-file-format/");
 	}
 
 	/* Check that values don't overflow on 32-bit systems. */
@@ -19313,9 +19327,11 @@ static MYSQL_SYSVAR_ENUM(flush_method, innodb_flush_method,
   NULL, NULL, IF_WIN(SRV_ALL_O_DIRECT_FSYNC, SRV_FSYNC),
   &innodb_flush_method_typelib);
 
-static MYSQL_SYSVAR_STR(file_format, innodb_file_format,
-  PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
-  "Deprecated parameter with no effect.", NULL, NULL, NULL);
+static MYSQL_SYSVAR_ENUM(file_format, innodb_file_format,
+  PLUGIN_VAR_RQCMDARG,
+  "File format constraint for native ALTER TABLE", NULL, NULL, 2/*add*/,
+  &innodb_file_format_typelib);
+
 static MYSQL_SYSVAR_STR(large_prefix, innodb_large_prefix,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
   "Deprecated parameter with no effect.", NULL, NULL, NULL);
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index a2839d1a7ec..b96a071ee4f 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -59,6 +59,8 @@ Smart ALTER TABLE
 #include "span.h"
 
 using st_::span;
+/** File format constraint for native ALTER TABLE */
+extern ulong innodb_file_format;
 
 static const char *MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN=
 			"INPLACE ADD or DROP of virtual columns cannot be "
@@ -954,6 +956,31 @@ ha_innobase::check_if_supported_inplace_alter(
 
 	const char* reason_rebuild = NULL;
 
+	switch (innodb_file_format) {
+	case 0: /* barracuda */
+		if (!(ha_alter_info->handler_flags
+		      & ALTER_ADD_STORED_BASE_COLUMN)) {
+			break;
+		}
+		reason_rebuild = "innodb_file_format=barracuda";
+	innodb_file_format_rebuild_reason:
+		if (ha_alter_info->handler_flags & ALTER_RECREATE_TABLE) {
+			reason_rebuild = NULL;
+		} else {
+			ha_alter_info->handler_flags |= ALTER_RECREATE_TABLE;
+			ha_alter_info->unsupported_reason = reason_rebuild;
+		}
+		break;
+	case 1: /* strict_barracuda */
+		if ((ha_alter_info->handler_flags
+		     & ALTER_ADD_STORED_BASE_COLUMN)
+		    || m_prebuilt->table->is_instant()) {
+			reason_rebuild = "innodb_file_format=strict_barracuda";
+			goto innodb_file_format_rebuild_reason;
+		}
+		break;
+	}
+
 	switch (ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) {
 	case ALTER_OPTIONS:
 		if (alter_options_need_rebuild(ha_alter_info, table)) {