MDEV-11974: MariaDB 10.2 encryption does not support spatial indexesbb-10.2-MDEV-11974

Encryption stores used key_version to
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION (offset 26)
field. Spatial indexes store RTREE Split Sequence Number
(FIL_RTREE_SPLIT_SEQ_NUM) in the same field. Both values
can't be stored in same field. Thus, current encryption
implementation does not support encrypting spatial indexes.

fil0crypt.cc: Do not encrypt page if page type is
FIL_PAGE_RTREE (this is required for background
encryption innodb-encrypt-tables=ON)

ha_innodb.cc: check_table_options() Do not allow creating
table with ENCRYPTED=YES if table contains spatial index.

5 files changed, 138 insertions, 4 deletions

diff --git a/mysql-test/suite/encryption/r/innodb-spatial-index.result b/mysql-test/suite/encryption/r/innodb-spatial-index.result
new file mode 100644
index 00000000000..7ad0af72bcd
--- /dev/null
+++ b/mysql-test/suite/encryption/r/innodb-spatial-index.result
@@ -0,0 +1,42 @@
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB
+ENCRYPTED=YES;
+ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options")
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+ALTER TABLE t1 ENCRYPTED=YES;
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+DROP TABLE t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=YES ENGINE=INNODB;
+CREATE SPATIAL INDEX b on t1(coordinate);
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create options")
+DROP TABLE t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=DEFAULT ENGINE=INNODB;
+CREATE SPATIAL INDEX b on t1(coordinate);
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 DROP INDEX b;
+INSERT INTO t1 values(2, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+INSERT INTO t1 values(3, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+DROP TABLE t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB PAGE_COMPRESSED=YES;
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+INSERT INTO t2 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+# Success!
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION > 0;
+NAME
+mysql/innodb_table_stats
+mysql/innodb_index_stats
+test/t1
+test/t2
+NULL
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+NAME
+DROP TABLE t1, t2;
diff --git a/mysql-test/suite/encryption/t/innodb-spatial-index.opt b/mysql-test/suite/encryption/t/innodb-spatial-index.opt
new file mode 100644
index 00000000000..d1f1bb9c555
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb-spatial-index.opt
@@ -0,0 +1,4 @@
+--innodb-encrypt-tables
+--innodb-encryption-rotate-key-age=15
+--innodb-encryption-threads=4
+--innodb-tablespaces-encryption
diff --git a/mysql-test/suite/encryption/t/innodb-spatial-index.test b/mysql-test/suite/encryption/t/innodb-spatial-index.test
new file mode 100644
index 00000000000..de78461c765
--- /dev/null
+++ b/mysql-test/suite/encryption/t/innodb-spatial-index.test
@@ -0,0 +1,72 @@
+--source include/have_innodb.inc
+--source include/have_file_key_management_plugin.inc
+
+#
+# MDEV-11974: MariaDB 10.2 encryption does not support spatial indexes
+#
+
+#
+# (1) Do not allow creating table with ENCRYPTED=YES
+#
+#
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB
+ENCRYPTED=YES;
+
+#
+# (2) Alter table
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+ALTER TABLE t1 ENCRYPTED=YES;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+DROP TABLE t1;
+
+#
+# Index creation
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=YES ENGINE=INNODB;
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+CREATE SPATIAL INDEX b on t1(coordinate);
+--replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/
+--error ER_CANT_CREATE_TABLE
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+DROP TABLE t1;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL) ENCRYPTED=DEFAULT ENGINE=INNODB;
+CREATE SPATIAL INDEX b on t1(coordinate);
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 DROP INDEX b;
+INSERT INTO t1 values(2, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+ALTER TABLE t1 ADD SPATIAL INDEX b(coordinate);
+INSERT INTO t1 values(3, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+DROP TABLE t1;
+#
+# (3) Default encryption should still work
+#
+
+CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB;
+CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT,
+c VARCHAR(256), coordinate POINT NOT NULL, SPATIAL index(coordinate)) ENGINE=INNODB PAGE_COMPRESSED=YES;
+
+INSERT INTO t1 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+INSERT INTO t2 values(1, 'secret', ST_GeomFromText('POINT(903994614 180726515)'));
+
+--let $wait_timeout=600
+--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+--source include/wait_condition.inc
+
+--echo # Success!
+
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION > 0;
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+
+DROP TABLE t1, t2;
\ No newline at end of file
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 7f545cb4060..ac1978f2673 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -629,10 +629,11 @@ fil_space_encrypt(
 
 	ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
 
-	if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR
-		|| orig_page_type==FIL_PAGE_TYPE_XDES) {
-		/* File space header or extent descriptor do not need to be
-		encrypted. */
+	if (orig_page_type == FIL_PAGE_TYPE_FSP_HDR ||
+	    orig_page_type == FIL_PAGE_TYPE_XDES ||
+	    orig_page_type == FIL_PAGE_RTREE) {
+		/* File space header, extent descriptor or spatial index
+		are not encrypted. */
 		return src_frame;
 	}
 
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index a87dba0d944..83f2e53cc83 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -12415,6 +12415,21 @@ create_table_info_t::check_table_options()
 	enum row_type	row_format = m_form->s->row_type;
 	ha_table_option_struct *options= m_form->s->option_struct;
 	fil_encryption_t encrypt = (fil_encryption_t)options->encryption;
+	bool should_encrypt = (encrypt == FIL_SPACE_ENCRYPTION_ON);
+
+	/* Currently we do not support encryption for
+	spatial indexes thus do not allow creating table with forced
+	encryption */
+	for(ulint i = 0; i < m_form->s->keys; i++) {
+		const KEY* key = m_form->key_info + i;
+		if (key->flags & HA_SPATIAL && should_encrypt) {
+			push_warning_printf(m_thd, Sql_condition::WARN_LEVEL_WARN,
+				HA_ERR_UNSUPPORTED,
+				"InnoDB: ENCRYPTED=ON not supported for table because "
+				"it contains spatial index.");
+			return "ENCRYPTED";
+		}
+	}
 
 	if (encrypt != FIL_SPACE_ENCRYPTION_DEFAULT && !m_allow_file_per_table) {
 		push_warning(