MDEV-25998 InnoDB removes the tablespace from default encrypt list earlybb-10.6-MDEV-25998

Avoid the removal of tablespace from default_encrypt_list
only when
1) Another active encryption thread working on tablespace
2) Eligible for tablespace key rotation
3) Tablespace is in flushing phase

1 files changed, 44 insertions, 2 deletions

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index a0b4b45287f..d02646396d2 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1146,6 +1146,49 @@ struct rotate_thread_t {
 	}
 };
 
+/** Avoid the removal of the tablespace from
+default_encrypt_list only when
+1) Another active encryption thread working on tablespace
+2) Eligible for tablespace key rotation
+3) Tablespace is in flushing phase
+@return true if tablespace should be removed from
+default encrypt */
+static bool fil_crypt_space_remove_list(const fil_space_t *space)
+{
+  ut_ad(space->purpose == FIL_TYPE_TABLESPACE);
+
+  fil_space_crypt_t *crypt_data = space->crypt_data;
+
+  if (crypt_data == NULL) { return srv_encrypt_tables == false;}
+
+  if (!crypt_data->is_key_found()) { return true; }
+
+  mysql_mutex_lock(&crypt_data->mutex);
+
+  if (space->is_stopping() || crypt_data->not_encrypted())
+  {
+func_exit:
+    mysql_mutex_unlock(&crypt_data->mutex);
+    return true;
+  }
+
+  if (crypt_data->rotate_state.flushing)
+    goto non_remove;
+
+  if ((crypt_data->min_key_version && !srv_encrypt_tables)
+      || (crypt_data->min_key_version == 0 && srv_encrypt_tables))
+    goto non_remove;
+
+  if (crypt_data->rotate_state.active_threads > 0)
+  {
+non_remove:
+    mysql_mutex_unlock(&crypt_data->mutex);
+    return false;
+  }
+
+  goto func_exit;
+}
+
 /***********************************************************************
 Check if space needs rotation given a key_state
 @param[in,out]		state		Key rotation state
@@ -1446,8 +1489,7 @@ inline fil_space_t *fil_system_t::default_encrypt_next(fil_space_t *space,
       If there is a change in innodb_encrypt_tables variables
       value then don't remove the last processed tablespace
       from the default encrypt list. */
-      if (released && (!recheck || space->crypt_data) &&
-          !encrypt == !srv_encrypt_tables)
+      if (released && !recheck && fil_crypt_space_remove_list(space))
       {
         ut_a(!default_encrypt_tables.empty());
         default_encrypt_tables.remove(*space);