MDEV-11974: MariaDB 10.2 encryption does not support spatial indexesbb-10.2-MDEV-11974

Encryption stores used key_version to
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION (offset 26)
field. Spatial indexes store RTREE Split Sequence Number
(FIL_RTREE_SPLIT_SEQ_NUM) in the same field. Both values
can't be stored in same field. Thus, current encryption
implementation does not support encrypting spatial indexes.

fil0crypt.cc: Do not encrypt page if page type is
FIL_PAGE_RTREE (this is required for background
encryption innodb-encrypt-tables=ON)

ha_innodb.cc: check_table_options() Do not allow creating
table with ENCRYPTED=YES if table contains spatial index.

3 files changed, 118 insertions, 0 deletions

diff --git a/mysql-test/suite/encryption/r/innodb-spatial-index.result b/mysql-test/suite/encryption/r/innodb-spatial-index.result
new file mode 100644
index 00000000000..7ad0af72bcd
--- /dev/null
+++ b/mysql-test/suite/encryption/r/innodb-spatial-index.result
@@ -0,0 +1,42 @@
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB
+ENCRYPTED=YES;
+ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options")
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+ALTER TABLE t1 ENCRYPTED=YES;
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+DROP TABLE t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=YES ENGINE=INNODB;
+CREATE SPATIAL INDEX b on t1(coordinate);
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+DROP TABLE t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=DEFAULT ENGINE=INNODB;
+CREATE SPATIAL INDEX b on t1(coordinate);
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 DROP INDEX b;
+INSERT INTO t1 values(2, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+INSERT INTO t1 values(3, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+DROP TABLE t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB PAGE_COMPRESSED=YES;
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+INSERT INTO t2 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+# Success!
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION > 0;
+NAME
+mysql/innodb_table_stats
+mysql/innodb_index_stats
+test/t1
+test/t2
+NULL
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+NAME
+DROP TABLE t1, t2;
diff --git a/mysql-test/suite/encryption/t/innodb-spatial-index.opt b/mysql-test/suite/encryption/t/innodb-spatial-index.opt
new file mode 100644
index 00000000000..d1f1bb9c555
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb-spatial-index.opt
@@ -0,0 +1,4 @@
+--innodb-encrypt-tables
+--innodb-encryption-rotate-key-age=15
+--innodb-encryption-threads=4
+--innodb-tablespaces-encryption
diff --git a/mysql-test/suite/encryption/t/innodb-spatial-index.test b/mysql-test/suite/encryption/t/innodb-spatial-index.test
new file mode 100644
index 00000000000..de78461c765
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb-spatial-index.test
@@ -0,0 +1,72 @@
+--source include/have_innodb.inc
+--source include/have_file_key_management_plugin.inc
+
+#
+# MDEV-11974: MariaDB 10.2 encryption does not support spatial indexes
+#
+
+#
+# (1) Do not allow creating table with ENCRYPTED=YES
+#
+#
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB
+ENCRYPTED=YES;
+
+#
+# (2) Alter table
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+ALTER TABLE t1 ENCRYPTED=YES;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+DROP TABLE t1;
+
+#
+# Index creation
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=YES ENGINE=INNODB;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+CREATE SPATIAL INDEX b on t1(coordinate);
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+DROP TABLE t1;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=DEFAULT ENGINE=INNODB;
+CREATE SPATIAL INDEX b on t1(coordinate);
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 DROP INDEX b;
+INSERT INTO t1 values(2, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+INSERT INTO t1 values(3, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+DROP TABLE t1;
+#
+# (3) Default encryption should still work
+#
+
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB PAGE_COMPRESSED=YES;
+
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+INSERT INTO t2 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+
+--let $wait_timeout=600
+--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+
+--echo # Success!
+
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION > 0;
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+
+DROP TABLE t1, t2;
\ No newline at end of file