Fix issuables state_id nil when importing projects from GitHubissue_57906_fix_github_import

Issues and merge requests imported from GitHub are having state_id
set to null. This fixes the GitHub project importer and schedule
migrations to fix state_id.

3 files changed, 98 insertions, 1 deletions

diff --git a/db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb b/db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb
new file mode 100644
index 00000000000..8e9838e1afb
--- /dev/null
+++ b/db/migrate/20190506135337_add_temporary_indexes_to_state_id.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This migration adds temporary indexes to state_id column of issues
+# and merge_requests tables. It will be used only to peform the scheduling
+# for populating state_id in a post migrate and will be removed after it.
+# Check: ScheduleSyncIssuablesStateIdWhereNil.
+
+class AddTemporaryIndexesToStateId < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    %w(issues merge_requests).each do |table|
+      add_concurrent_index(
+        table,
+        'id',
+        name: index_name_for(table),
+        where: "state_id IS NULL"
+      )
+    end
+  end
+
+  def down
+    remove_concurrent_index_by_name(:issues, index_name_for("issues"))
+    remove_concurrent_index_by_name(:merge_requests, index_name_for("merge_requests"))
+  end
+
+  def index_name_for(table)
+    "idx_on_#{table}_where_state_id_is_null"
+  end
+end
diff --git a/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
new file mode 100644
index 00000000000..4c31b5968ff
--- /dev/null
+++ b/db/post_migrate/20190506135400_schedule_sync_issuables_state_id_where_nil.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+class ScheduleSyncIssuablesStateIdWhereNil < ActiveRecord::Migration[5.1]
+  # Issues and MergeRequests imported by GitHub are being created with
+  # state_id = null, this fixes them.
+  #
+  # Part of a bigger plan: https://gitlab.com/gitlab-org/gitlab-ce/issues/51789
+
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  # 2019-05-02 gitlab.com issuable numbers
+  # issues with state_id nil: ~40000
+  # merge requests with state_id nil: ~200000
+  #
+  # Using 5000 as batch size and 120 seconds interval will create:
+  # ~8 jobs for issues - taking ~16 minutes
+  # ~40 jobs for merge requests - taking ~1.34 hours
+  #
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 120.seconds.to_i
+  ISSUES_MIGRATION = 'SyncIssuesStateId'.freeze
+  MERGE_REQUESTS_MIGRATION = 'SyncMergeRequestsStateId'.freeze
+
+  disable_ddl_transaction!
+
+  class Issue < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'issues'
+  end
+
+  class MergeRequest < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'merge_requests'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      Issue.where(state_id: nil),
+      ISSUES_MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+
+    queue_background_migration_jobs_by_range_at_intervals(
+      MergeRequest.where(state_id: nil),
+      MERGE_REQUESTS_MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+
+    # Remove temporary indexes added on "AddTemporaryIndexesToStateId"
+    remove_concurrent_index_by_name(:issues, "idx_on_issues_where_state_id_is_null")
+    remove_concurrent_index_by_name(:merge_requests, "idx_on_merge_requests_where_state_id_is_null")
+  end
+
+  def down
+    # No op
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2e77bbb51e5..3788be73ae2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190426180107) do
+ActiveRecord::Schema.define(version: 20190506135400) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"