db: De-duplicate list of removed table columns

We have a policy of removing fields from SQLAlchemy models at least one
cycle before we remove the underlying database columns. This can result
in a discrepancy between the state that our newfangled database
migration tool, alembic, sees and what's actually in the database. We
were ignoring these removed fields (and one foreign key constraint) in
two different locations for both databases: as part of the alembic
configuration and as part of the tests we have to ensure our migrations
are in sync with our models (note: the tests actually use the alembic
mechanism to detect the changes [1]). De-duplicate these.

[1] https://github.com/sqlalchemy/alembic/issues/724#issuecomment-672081357

Change-Id: I978b4e44cf7f522a70cc5ca76e6d6f1a985d5469
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>

2 files changed, 39 insertions, 51 deletions

diff --git a/nova/tests/unit/db/api/test_migrations.py b/nova/tests/unit/db/api/test_migrations.py
index f714cf1852..0f51ad12a7 100644
--- a/nova/tests/unit/db/api/test_migrations.py
+++ b/nova/tests/unit/db/api/test_migrations.py
@@ -62,30 +62,17 @@ class NovaModelsMigrationsSync(test_migrations.ModelsMigrationsSync):
             if name == 'migrate_version':
                 return False
 
+            # Define a whitelist of tables that will be removed from the DB in
+            # a later release and don't have a corresponding model anymore.
+
+            return name not in models.REMOVED_TABLES
+
         return True
 
     def filter_metadata_diff(self, diff):
         # Filter out diffs that shouldn't cause a sync failure.
         new_diff = []
 
-        # Define a whitelist of ForeignKeys that exist on the model but not in
-        # the database. They will be removed from the model at a later time.
-        fkey_whitelist = {'build_requests': ['request_spec_id']}
-
-        # Define a whitelist of columns that will be removed from the
-        # DB at a later release and aren't on a model anymore.
-
-        column_whitelist = {
-            'build_requests': [
-                'vm_state', 'instance_metadata',
-                'display_name', 'access_ip_v6', 'access_ip_v4', 'key_name',
-                'locked_by', 'image_ref', 'progress', 'request_spec_id',
-                'info_cache', 'user_id', 'task_state', 'security_groups',
-                'config_drive',
-            ],
-            'resource_providers': ['can_host'],
-        }
-
         for element in diff:
             if isinstance(element, list):
                 # modify_nullable is a list
@@ -97,18 +84,12 @@ class NovaModelsMigrationsSync(test_migrations.ModelsMigrationsSync):
                     fkey = element[1]
                     tablename = fkey.table.name
                     column_keys = fkey.column_keys
-                    if (
-                        tablename in fkey_whitelist and
-                        column_keys == fkey_whitelist[tablename]
-                    ):
+                    if (tablename, column_keys) in models.REMOVED_FKEYS:
                         continue
                 elif element[0] == 'remove_column':
-                    tablename = element[2]
-                    column = element[3]
-                    if (
-                        tablename in column_whitelist and
-                        column.name in column_whitelist[tablename]
-                    ):
+                    table = element[2]
+                    column = element[3].name
+                    if (table, column) in models.REMOVED_COLUMNS:
                         continue
 
                 new_diff.append(element)
diff --git a/nova/tests/unit/db/main/test_migrations.py b/nova/tests/unit/db/main/test_migrations.py
index c0da4cc66a..0ca7a8bf77 100644
--- a/nova/tests/unit/db/main/test_migrations.py
+++ b/nova/tests/unit/db/main/test_migrations.py
@@ -80,32 +80,39 @@ class NovaModelsMigrationsSync(test_migrations.ModelsMigrationsSync):
             if name == 'migrate_version' or name.startswith('shadow_'):
                 return False
 
-        return True
+            # Define a whitelist of tables that will be removed from the DB in
+            # a later release and don't have a corresponding model anymore.
 
-    def filter_metadata_diff(self, diff):
-        # Overriding the parent method to decide on certain attributes
-        # that maybe present in the DB but not in the models.py
-
-        def removed_column(element):
-            # Define a whitelist of columns that would be removed from the
-            # DB at a later release.
-            # NOTE(Luyao) The vpmems column was added to the schema in train,
-            # and removed from the model in train.
-            column_whitelist = {
-                'instances': ['internal_id'],
-                'instance_extra': ['vpmems'],
-            }
-
-            if element[0] != 'remove_column':
-                return False
+            return name not in models.REMOVED_TABLES
 
-            table_name, column = element[2], element[3]
-            return (
-                table_name in column_whitelist and
-                column.name in column_whitelist[table_name]
-            )
+        return True
 
-        return [element for element in diff if not removed_column(element)]
+    def filter_metadata_diff(self, diff):
+        # Filter out diffs that shouldn't cause a sync failure.
+        new_diff = []
+
+        for element in diff:
+            if isinstance(element, list):
+                # modify_nullable is a list
+                new_diff.append(element)
+            else:
+                # tuple with action as first element. Different actions have
+                # different tuple structures.
+                if element[0] == 'add_fk':
+                    fkey = element[1]
+                    tablename = fkey.table.name
+                    column_keys = fkey.column_keys
+                    if (tablename, column_keys) in models.REMOVED_FKEYS:
+                        continue
+                if element[0] == 'remove_column':
+                    table = element[2]
+                    column = element[3].name
+                    if (table, column) in models.REMOVED_COLUMNS:
+                        continue
+
+                new_diff.append(element)
+
+        return new_diff
 
 
 class TestModelsSyncSQLite(