Merge commit '8af23def1d6' into object-storage-ee-to-ce-backport

21 files changed, 637 insertions, 44 deletions

diff --git a/db/migrate/20160301174731_add_fingerprint_index.rb b/db/migrate/20160301174731_add_fingerprint_index.rb
new file mode 100644
index 00000000000..f2c3d1ba1ea
--- /dev/null
+++ b/db/migrate/20160301174731_add_fingerprint_index.rb
@@ -0,0 +1,17 @@
+# rubocop:disable all
+class AddFingerprintIndex < ActiveRecord::Migration
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  # https://gitlab.com/gitlab-org/gitlab-ee/issues/764
+  def change
+    args = [:keys, :fingerprint]
+
+    if Gitlab::Database.postgresql?
+      args << { algorithm: :concurrently }
+    end
+
+    add_index(*args) unless index_exists?(:keys, :fingerprint)
+  end
+end
diff --git a/db/migrate/20170425112128_create_pipeline_schedules_table.rb b/db/migrate/20170425112128_create_pipeline_schedules_table.rb
index 57df47f5f42..4f9c56a1ad8 100644
--- a/db/migrate/20170425112128_create_pipeline_schedules_table.rb
+++ b/db/migrate/20170425112128_create_pipeline_schedules_table.rb
@@ -17,7 +17,7 @@ class CreatePipelineSchedulesTable < ActiveRecord::Migration
       t.boolean :active, default: true
       t.datetime :deleted_at
 
-      t.timestamps
+      t.timestamps null: true
     end
 
     add_index(:ci_pipeline_schedules, :project_id)
diff --git a/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
new file mode 100644
index 00000000000..1d86a531eb3
--- /dev/null
+++ b/db/migrate/20170531180233_add_authorized_keys_enabled_to_application_settings.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddAuthorizedKeysEnabledToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :authorized_keys_enabled, :boolean, default: true, allow_null: false
+  end
+
+  def down
+    remove_column :application_settings, :authorized_keys_enabled
+  end
+end
diff --git a/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb
new file mode 100644
index 00000000000..1b360b231a8
--- /dev/null
+++ b/db/migrate/20170827123848_add_index_on_merge_request_diff_commit_sha.rb
@@ -0,0 +1,17 @@
+# rubocop:disable RemoveIndex
+
+class AddIndexOnMergeRequestDiffCommitSha < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_request_diff_commits, :sha, length: Gitlab::Database.mysql? ? 20 : nil
+  end
+
+  def down
+    remove_index :merge_request_diff_commits, :sha if index_exists? :merge_request_diff_commits, :sha
+  end
+end
diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb
index ca906b953a3..89e5b871967 100644
--- a/db/migrate/20170928124105_create_fork_networks.rb
+++ b/db/migrate/20170928124105_create_fork_networks.rb
@@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration
     if foreign_keys_for(:fork_networks, :root_project_id).any?
       remove_foreign_key :fork_networks, column: :root_project_id
     end
+
     drop_table :fork_networks
   end
 end
diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb
index 836f023efdc..8c7d9ba859a 100644
--- a/db/migrate/20170928133643_create_fork_network_members.rb
+++ b/db/migrate/20170928133643_create_fork_network_members.rb
@@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration
     if foreign_keys_for(:fork_network_members, :forked_from_project_id).any?
       remove_foreign_key :fork_network_members, column: :forked_from_project_id
     end
+
     drop_table :fork_network_members
   end
 end
diff --git a/db/migrate/20171207185153_add_merge_request_state_index.rb b/db/migrate/20171207185153_add_merge_request_state_index.rb
new file mode 100644
index 00000000000..72f846c5c38
--- /dev/null
+++ b/db/migrate/20171207185153_add_merge_request_state_index.rb
@@ -0,0 +1,18 @@
+class AddMergeRequestStateIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :merge_requests, [:source_project_id, :source_branch],
+                         where: "state = 'opened'",
+                         name: 'index_merge_requests_on_source_project_and_branch_state_opened'
+  end
+
+  def down
+    remove_concurrent_index_by_name :merge_requests,
+                                    'index_merge_requests_on_source_project_and_branch_state_opened'
+  end
+end
diff --git a/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb
new file mode 100644
index 00000000000..5dc723db9f9
--- /dev/null
+++ b/db/migrate/20171211145425_add_can_push_to_deploy_keys_projects.rb
@@ -0,0 +1,15 @@
+class AddCanPushToDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :deploy_keys_projects, :can_push, :boolean, default: false, allow_null: false
+  end
+
+  def down
+    remove_column :deploy_keys_projects, :can_push
+  end
+end
diff --git a/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb
new file mode 100644
index 00000000000..680855af945
--- /dev/null
+++ b/db/migrate/20171215113714_populate_can_push_from_deploy_keys_projects.rb
@@ -0,0 +1,64 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
+  disable_ddl_transaction!
+
+  class DeploysKeyProject < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'deploy_keys_projects'
+  end
+
+  def up
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+
+  def down
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+end
diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
index 7cf1d0cec68..d1a039ed551 100644
--- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
+++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb
@@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.5
       # Allow us to hot-patch the index manually ahead of the migration
       execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));"
@@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration
     return unless Gitlab::Database.postgresql?
 
     disable_statement_timeout
+
     if Gitlab::Database.version.to_f >= 9.2
       execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};"
     else
diff --git a/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
new file mode 100644
index 00000000000..f942b4c062e
--- /dev/null
+++ b/db/migrate/20180105212544_add_commits_count_to_merge_request_diff.rb
@@ -0,0 +1,29 @@
+class AddCommitsCountToMergeRequestDiff < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze
+  BATCH_SIZE = 5000
+  DELAY_INTERVAL = 5.minutes.to_i
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :merge_request_diffs, :commits_count, :integer
+
+    say 'Populating the MergeRequestDiff `commits_count`'
+
+    queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    remove_column :merge_request_diffs, :commits_count
+  end
+end
diff --git a/db/migrate/20180113220114_rework_redirect_routes_indexes.rb b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb
new file mode 100644
index 00000000000..ab9971be074
--- /dev/null
+++ b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb
@@ -0,0 +1,68 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ReworkRedirectRoutesIndexes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  INDEX_NAME_UNIQUE = "index_redirect_routes_on_path_unique_text_pattern_ops"
+
+  INDEX_NAME_PERM = "index_redirect_routes_on_path_text_pattern_ops_where_permanent"
+  INDEX_NAME_TEMP = "index_redirect_routes_on_path_text_pattern_ops_where_temporary"
+
+  OLD_INDEX_NAME_PATH_TPOPS = "index_redirect_routes_on_path_text_pattern_ops"
+  OLD_INDEX_NAME_PATH_LOWER = "index_on_redirect_routes_lower_path"
+
+  def up
+    disable_statement_timeout
+
+    # this is a plain btree on a single boolean column. It'll never be
+    # selective enough to be valuable. This class is called by
+    # setup_postgresql.rake so it needs to be able to handle this
+    # index not existing.
+    if index_exists?(:redirect_routes, :permanent)
+      remove_concurrent_index(:redirect_routes, :permanent)
+    end
+
+    # If we're on MySQL then the existing index on path is ok. But on
+    # Postgres we need to clean things up:
+    return unless Gitlab::Database.postgresql?
+
+    if_not_exists = Gitlab::Database.version.to_f >= 9.5 ? "IF NOT EXISTS" : ""
+
+    # Unique index on lower(path) across both types of redirect_routes:
+    execute("CREATE UNIQUE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_UNIQUE} ON redirect_routes (lower(path) varchar_pattern_ops);")
+
+    # Make two indexes on path -- one for permanent and one for temporary routes:
+    execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);")
+    execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;")
+
+    # Remove the old indexes:
+
+    # This one needed to be on lower(path) but wasn't so it's replaced with the two above
+    execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_TPOPS};"
+
+    # This one isn't needed because we only ever do = and LIKE on this
+    # column so the varchar_pattern_ops index is sufficient
+    execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_LOWER};"
+  end
+
+  def down
+    disable_statement_timeout
+
+    add_concurrent_index(:redirect_routes, :permanent)
+
+    return unless Gitlab::Database.postgresql?
+
+    execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_TPOPS} ON redirect_routes (path varchar_pattern_ops);")
+    execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_LOWER} ON redirect_routes (LOWER(path));")
+
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_UNIQUE};")
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};")
+    execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};")
+  end
+end
diff --git a/db/migrate/20180115201419_add_index_updated_at_to_issues.rb b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb
new file mode 100644
index 00000000000..a5a48fc97be
--- /dev/null
+++ b/db/migrate/20180115201419_add_index_updated_at_to_issues.rb
@@ -0,0 +1,15 @@
+class AddIndexUpdatedAtToIssues < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :issues, :updated_at
+  end
+
+  def down
+    remove_concurrent_index :issues, :updated_at
+  end
+end
diff --git a/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb b/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb
new file mode 100644
index 00000000000..7cb913bb2bf
--- /dev/null
+++ b/db/migrate/20180201145907_migrate_remaining_issues_closed_at.rb
@@ -0,0 +1,42 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class MigrateRemainingIssuesClosedAt < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+    include EachBatch
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('CopyColumn')
+    Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange')
+
+    # It's possible the cleanup job was killed which means we need to manually
+    # migrate any remaining rows.
+    migrate_remaining_rows if migrate_column_type?
+  end
+
+  def down
+  end
+
+  def migrate_remaining_rows
+    Issue.where('closed_at_for_type_change IS NULL AND closed_at IS NOT NULL').each_batch do |batch|
+      batch.update_all('closed_at_for_type_change = closed_at')
+    end
+
+    cleanup_concurrent_column_type_change(:issues, :closed_at)
+  end
+
+  def migrate_column_type?
+    # Some environments may have already executed the previous version of this
+    # migration, thus we don't need to migrate those environments again.
+    column_for('issues', 'closed_at').type == :datetime # rubocop:disable Migration/Datetime
+  end
+end
diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
index da0fcda87a6..17ad7de065d 100644
--- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
+++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb
@@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration
       predicate = namespaces[:owner_id].eq(users[:id])
                     .and(namespaces[:type].eq(nil))
                     .and(users[:username].matches(path))
+
       update_sql = if Gitlab::Database.postgresql?
                      "UPDATE users SET username = namespaces.path "\
                      "FROM namespaces WHERE #{predicate.to_sql}"
diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
index 547cc68e10e..fce1829c982 100644
--- a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
+++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
@@ -15,8 +15,6 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
   end
 
   def up
-    merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id)
-
     say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs'
     # It will update around 4_000_000 records in batches of 10_000 merge
     # requests (running between 10 minutes) and should take around 66 hours to complete.
@@ -25,7 +23,7 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
     #
     # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class.
     #
-    merge_requests.each_batch(of: BATCH_SIZE) do |relation, index|
+    MergeRequest.all.each_batch(of: BATCH_SIZE) do |relation, index|
       range = relation.pluck('MIN(id)', 'MAX(id)').first
 
       BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range)
@@ -37,32 +35,4 @@ class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migratio
     execute "update merge_request_metrics set latest_closed_by_id = null"
     execute "update merge_request_metrics set merged_by_id = null"
   end
-
-  private
-
-  # On staging:
-  # Planning time: 0.682 ms
-  # Execution time: 22033.158 ms
-  #
-  def updatable_merge_requests_union_sql
-    metrics_not_exists_clause =
-      'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)'
-
-    without_metrics_data = <<-SQL.strip_heredoc
-      merge_request_metrics.merged_by_id IS NULL OR
-      merge_request_metrics.latest_closed_by_id IS NULL OR
-      merge_request_metrics.latest_closed_at IS NULL
-    SQL
-
-    mrs_without_metrics_record = MergeRequest
-      .where(metrics_not_exists_clause)
-      .select(:id)
-
-    mrs_without_events_data = MergeRequest
-      .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id')
-      .where(without_metrics_data)
-      .select(:id)
-
-    Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql
-  end
 end
diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
new file mode 100644
index 00000000000..3e2dedfdd6a
--- /dev/null
+++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb
@@ -0,0 +1,208 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveSoftRemovedObjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  module SoftRemoved
+    extend ActiveSupport::Concern
+
+    included do
+      scope :soft_removed, -> { where('deleted_at IS NOT NULL') }
+    end
+  end
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+
+    include EachBatch
+  end
+
+  class Issue < ActiveRecord::Base
+    self.table_name = 'issues'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Namespace < ActiveRecord::Base
+    self.table_name = 'namespaces'
+
+    include EachBatch
+    include SoftRemoved
+
+    scope :soft_removed_personal, -> { soft_removed.where(type: nil) }
+    scope :soft_removed_group, -> { soft_removed.where(type: 'Group') }
+  end
+
+  class Route < ActiveRecord::Base
+    self.table_name = 'routes'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiPipelineSchedule < ActiveRecord::Base
+    self.table_name = 'ci_pipeline_schedules'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  class CiTrigger < ActiveRecord::Base
+    self.table_name = 'ci_triggers'
+
+    include EachBatch
+    include SoftRemoved
+  end
+
+  MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze
+
+  def up
+    disable_statement_timeout
+
+    remove_personal_routes
+    remove_personal_namespaces
+    remove_group_namespaces
+    remove_simple_soft_removed_rows
+  end
+
+  def down
+    # The data removed by this migration can't be restored in an automated way.
+  end
+
+  def remove_simple_soft_removed_rows
+    create_temporary_indexes
+
+    MODELS.each do |model|
+      say_with_time("Removing soft removed rows from #{model.table_name}") do
+        model.soft_removed.each_batch do |batch, index|
+          batch.delete_all
+        end
+      end
+    end
+  ensure
+    remove_temporary_indexes
+  end
+
+  def create_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      # Without this index the removal process can take a very long time. For
+      # example, getting the next ID of a batch for the `issues` table in
+      # staging would take between 15 and 20 seconds.
+      next if temporary_index_exists?(model)
+
+      say_with_time("Creating temporary index #{index_name}") do
+        add_concurrent_index(
+          model.table_name,
+          [:deleted_at, :id],
+          name: index_name,
+          where: 'deleted_at IS NOT NULL'
+        )
+      end
+    end
+  end
+
+  def remove_temporary_indexes
+    MODELS.each do |model|
+      index_name = temporary_index_name_for(model)
+
+      next unless temporary_index_exists?(model)
+
+      say_with_time("Removing temporary index #{index_name}") do
+        remove_concurrent_index_by_name(model.table_name, index_name)
+      end
+    end
+  end
+
+  def temporary_index_name_for(model)
+    "index_on_#{model.table_name}_tmp"
+  end
+
+  def temporary_index_exists?(model)
+    index_name = temporary_index_name_for(model)
+
+    index_exists?(model.table_name, [:deleted_at, :id], name: index_name)
+  end
+
+  def remove_personal_namespaces
+    # Some personal namespaces are left behind in case of GitLab.com. In these
+    # cases the associated data such as the projects and users has already been
+    # removed.
+    Namespace.soft_removed_personal.each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def remove_group_namespaces
+    admin_id = id_for_admin_user
+
+    unless admin_id
+      say 'Not scheduling soft removed groups for removal as no admin user ' \
+        'could be found. You will need to remove any such groups manually.'
+
+      return
+    end
+
+    # Left over groups can't be easily removed because we may also need to
+    # remove memberships, repositories, and other associated data. As a result
+    # we'll just schedule a Sidekiq job to remove these.
+    #
+    # As of January 5th, 2018 there are 36 groups that will be removed using
+    # this code.
+    Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index|
+      batch.each do |ns|
+        schedule_group_removal(index * 5.minutes, ns.id, admin_id)
+      end
+    end
+  end
+
+  def schedule_group_removal(delay, group_id, user_id)
+    if migrate_inline?
+      GroupDestroyWorker.new.perform(group_id, user_id)
+    else
+      GroupDestroyWorker.perform_in(delay, group_id, user_id)
+    end
+  end
+
+  def remove_personal_routes
+    namespaces = Namespace.select(1)
+      .soft_removed
+      .where('namespaces.type IS NULL')
+      .where('routes.source_type = ?', 'Namespace')
+      .where('routes.source_id = namespaces.id')
+
+    Route.where('EXISTS (?)', namespaces).each_batch do |batch|
+      batch.delete_all
+    end
+  end
+
+  def id_for_admin_user
+    User.where(admin: true).limit(1).pluck(:id).first
+  end
+
+  def migrate_inline?
+    Rails.env.test? || Rails.env.development?
+  end
+end
diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
new file mode 100644
index 00000000000..154d7a1b926
--- /dev/null
+++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb
@@ -0,0 +1,31 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveDeletedAtColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze
+  COLUMN = :deleted_at
+
+  def up
+    TABLES.each do |table|
+      remove_column(table, COLUMN) if column_exists?(table, COLUMN)
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      unless column_exists?(table, COLUMN)
+        add_column(table, COLUMN, :datetime_with_timezone)
+      end
+
+      unless index_exists?(table, COLUMN)
+        add_concurrent_index(table, COLUMN)
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb
new file mode 100644
index 00000000000..3a5850df3db
--- /dev/null
+++ b/db/post_migrate/20171215121205_post_populate_can_push_from_deploy_keys_projects.rb
@@ -0,0 +1,63 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PostPopulateCanPushFromDeployKeysProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DATABASE_NAME = Gitlab::Database.database_name
+
+  disable_ddl_transaction!
+
+  class DeploysKeyProject < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'deploy_keys_projects'
+  end
+
+  def up
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET deploy_keys_projects.can_push = #{DATABASE_NAME}.keys.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects
+          SET can_push = keys.can_push
+          FROM keys
+          WHERE deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+
+  def down
+    DeploysKeyProject.each_batch(of: 10_000) do |batch|
+      start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
+
+      if Gitlab::Database.mysql?
+        execute <<-EOF.strip_heredoc
+          UPDATE deploy_keys_projects, #{DATABASE_NAME}.keys
+          SET #{DATABASE_NAME}.keys.can_push = deploy_keys_projects.can_push
+          WHERE deploy_keys_projects.deploy_key_id = #{DATABASE_NAME}.keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      else
+        execute <<-EOF.strip_heredoc
+          UPDATE keys
+          SET can_push = deploy_keys_projects.can_push
+          FROM deploy_keys_projects
+          WHERE deploy_keys_projects.deploy_key_id = keys.id
+          AND deploy_keys_projects.id BETWEEN #{start_id} AND #{end_id}
+        EOF
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20171215121259_remove_can_push_from_keys.rb b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb
new file mode 100644
index 00000000000..0599811d986
--- /dev/null
+++ b/db/post_migrate/20171215121259_remove_can_push_from_keys.rb
@@ -0,0 +1,17 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveCanPushFromKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  def up
+    remove_column :keys, :can_push
+  end
+
+  def down
+    add_column_with_default :keys, :can_push, :boolean, default: false, allow_null: false
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index b6800ff926e..328c549284c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171230123729) do
+ActiveRecord::Schema.define(version: 20180201145907) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -154,6 +154,7 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.integer "gitaly_timeout_default", default: 55, null: false
     t.integer "gitaly_timeout_medium", default: 30, null: false
     t.integer "gitaly_timeout_fast", default: 10, null: false
+    t.boolean "authorized_keys_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -359,7 +360,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.integer "project_id"
     t.integer "owner_id"
     t.boolean "active", default: true
-    t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
   end
@@ -469,7 +469,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
 
   create_table "ci_triggers", force: :cascade do |t|
     t.string "token"
-    t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer "project_id"
@@ -630,6 +629,7 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.integer "project_id", null: false
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.boolean "can_push", default: false, null: false
   end
 
   add_index "deploy_keys_projects", ["project_id"], name: "index_deploy_keys_projects_on_project_id", using: :btree
@@ -863,7 +863,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.integer "iid"
     t.integer "updated_by_id"
     t.boolean "confidential", default: false, null: false
-    t.datetime "deleted_at"
     t.date "due_date"
     t.integer "moved_to_id"
     t.integer "lock_version"
@@ -880,7 +879,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
 
   add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree
   add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree
-  add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree
   add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree
   add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree
@@ -891,6 +889,7 @@ ActiveRecord::Schema.define(version: 20171230123729) do
   add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree
   add_index "issues", ["state"], name: "index_issues_on_state", using: :btree
   add_index "issues", ["title"], name: "index_issues_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"}
+  add_index "issues", ["updated_at"], name: "index_issues_on_updated_at", using: :btree
   add_index "issues", ["updated_by_id"], name: "index_issues_on_updated_by_id", where: "(updated_by_id IS NOT NULL)", using: :btree
 
   create_table "keys", force: :cascade do |t|
@@ -902,7 +901,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.string "type"
     t.string "fingerprint"
     t.boolean "public", default: false, null: false
-    t.boolean "can_push", default: false, null: false
     t.datetime "last_used_at"
   end
 
@@ -1020,6 +1018,7 @@ ActiveRecord::Schema.define(version: 20171230123729) do
   end
 
   add_index "merge_request_diff_commits", ["merge_request_diff_id", "relative_order"], name: "index_merge_request_diff_commits_on_mr_diff_id_and_order", unique: true, using: :btree
+  add_index "merge_request_diff_commits", ["sha"], name: "index_merge_request_diff_commits_on_sha", using: :btree
 
   create_table "merge_request_diff_files", id: false, force: :cascade do |t|
     t.integer "merge_request_diff_id", null: false
@@ -1047,6 +1046,7 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.string "real_size"
     t.string "head_commit_sha"
     t.string "start_commit_sha"
+    t.integer "commits_count"
   end
 
   add_index "merge_request_diffs", ["merge_request_id", "id"], name: "index_merge_request_diffs_on_merge_request_id_and_id", using: :btree
@@ -1090,7 +1090,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.boolean "merge_when_pipeline_succeeds", default: false, null: false
     t.integer "merge_user_id"
     t.string "merge_commit_sha"
-    t.datetime "deleted_at"
     t.string "in_progress_merge_commit_sha"
     t.integer "lock_version"
     t.text "title_html"
@@ -1109,13 +1108,13 @@ ActiveRecord::Schema.define(version: 20171230123729) do
   add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree
   add_index "merge_requests", ["author_id"], name: "index_merge_requests_on_author_id", using: :btree
   add_index "merge_requests", ["created_at"], name: "index_merge_requests_on_created_at", using: :btree
-  add_index "merge_requests", ["deleted_at"], name: "index_merge_requests_on_deleted_at", using: :btree
   add_index "merge_requests", ["description"], name: "index_merge_requests_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
   add_index "merge_requests", ["head_pipeline_id"], name: "index_merge_requests_on_head_pipeline_id", using: :btree
   add_index "merge_requests", ["latest_merge_request_diff_id"], name: "index_merge_requests_on_latest_merge_request_diff_id", using: :btree
   add_index "merge_requests", ["merge_user_id"], name: "index_merge_requests_on_merge_user_id", where: "(merge_user_id IS NOT NULL)", using: :btree
   add_index "merge_requests", ["milestone_id"], name: "index_merge_requests_on_milestone_id", using: :btree
   add_index "merge_requests", ["source_branch"], name: "index_merge_requests_on_source_branch", using: :btree
+  add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_and_branch_state_opened", where: "((state)::text = 'opened'::text)", using: :btree
   add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_id_and_source_branch", using: :btree
   add_index "merge_requests", ["target_branch"], name: "index_merge_requests_on_target_branch", using: :btree
   add_index "merge_requests", ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid", unique: true, using: :btree
@@ -1169,7 +1168,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
     t.boolean "share_with_group_lock", default: false
     t.integer "visibility_level", default: 20, null: false
     t.boolean "request_access_enabled", default: false, null: false
-    t.datetime "deleted_at"
     t.text "description_html"
     t.boolean "lfs_enabled"
     t.integer "parent_id"
@@ -1179,7 +1177,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
   end
 
   add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree
-  add_index "namespaces", ["deleted_at"], name: "index_namespaces_on_deleted_at", using: :btree
   add_index "namespaces", ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree
   add_index "namespaces", ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"}
   add_index "namespaces", ["owner_id"], name: "index_namespaces_on_owner_id", using: :btree
@@ -1546,8 +1543,6 @@ ActiveRecord::Schema.define(version: 20171230123729) do
   end
 
   add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree
-  add_index "redirect_routes", ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"}
-  add_index "redirect_routes", ["permanent"], name: "index_redirect_routes_on_permanent", using: :btree
   add_index "redirect_routes", ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree
 
   create_table "releases", force: :cascade do |t|