Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all

10 files changed, 91 insertions, 47 deletions

diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb
new file mode 100644
index 00000000000..f90f35a913d
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_stage_index.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class MigrateStageIndex
+      def perform(start_id, stop_id)
+        migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql|
+          ActiveRecord::Base.connection.execute(sql)
+        end
+      end
+
+      private
+
+      def migrate_stage_index_sql(start_id, stop_id)
+        if Gitlab::Database.postgresql?
+          <<~SQL
+            WITH freqs AS (
+              SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds
+                WHERE stage_id BETWEEN #{start_id} AND #{stop_id}
+                  AND stage_idx IS NOT NULL
+                GROUP BY stage_id, stage_idx
+            ), indexes AS (
+              SELECT DISTINCT stage_id, last_value(stage_idx)
+                OVER (PARTITION BY stage_id ORDER BY freq ASC) AS index
+                FROM freqs
+            )
+
+            UPDATE ci_stages SET position = indexes.index
+              FROM indexes WHERE indexes.stage_id = ci_stages.id
+                AND ci_stages.position IS NULL;
+          SQL
+        else
+          <<~SQL
+            UPDATE ci_stages
+              SET position =
+                (SELECT stage_idx FROM ci_builds
+                  WHERE ci_builds.stage_id = ci_stages.id
+                  GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1)
+            WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id}
+              AND ci_stages.position IS NULL
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
index 551483d0aaa..73f36735e35 100644
--- a/lib/gitlab/ci/cron_parser.rb
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -6,7 +6,7 @@ module Gitlab
 
       def initialize(cron, cron_timezone = 'UTC')
         @cron = cron
-        @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name
+        @cron_timezone = timezone_name(cron_timezone)
       end
 
       def next_time_from(time)
@@ -24,6 +24,12 @@ module Gitlab
 
       private
 
+      def timezone_name(timezone)
+        ActiveSupport::TimeZone.find_tzinfo(timezone).name
+      rescue TZInfo::InvalidTimezoneIdentifier
+        timezone
+      end
+
       # NOTE:
       # cron_timezone can only accept timezones listed in TZInfo::Timezone.
       # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted,
diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb
index c101f30d6e8..2b58d9863a0 100644
--- a/lib/gitlab/ci/pipeline/seed/stage.rb
+++ b/lib/gitlab/ci/pipeline/seed/stage.rb
@@ -19,6 +19,7 @@ module Gitlab
 
           def attributes
             { name: @attributes.fetch(:name),
+              position: @attributes.fetch(:index),
               pipeline: @pipeline,
               project: @pipeline.project }
           end
diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb
new file mode 100644
index 00000000000..d7e3ce08b32
--- /dev/null
+++ b/lib/gitlab/database/arel_methods.rb
@@ -0,0 +1,18 @@
+module Gitlab
+  module Database
+    module ArelMethods
+      private
+
+      # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer`
+      # was removed.
+      # Remove this file and inline this method when removing rails5? code.
+      def arel_update_manager
+        if Gitlab.rails5?
+          Arel::UpdateManager.new
+        else
+          Arel::UpdateManager.new(ActiveRecord::Base)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 77079e5e72b..c21bae5e16b 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module Database
     module MigrationHelpers
+      include Gitlab::Database::ArelMethods
+
       BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
       BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
 
@@ -314,7 +316,7 @@ module Gitlab
           stop_arel = yield table, stop_arel if block_given?
           stop_row = exec_query(stop_arel.to_sql).to_hash.first
 
-          update_arel = Arel::UpdateManager.new(ActiveRecord::Base)
+          update_arel = arel_update_manager
             .table(table)
             .set([[table[column], value]])
             .where(table[:id].gteq(start_id))
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index 1a697396ff1..14de28a1d08 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -3,6 +3,8 @@ module Gitlab
     module RenameReservedPathsMigration
       module V1
         class RenameBase
+          include Gitlab::Database::ArelMethods
+
           attr_reader :paths, :migration
 
           delegate :update_column_in_batches,
@@ -62,10 +64,10 @@ module Gitlab
                                             old_full_path,
                                             new_full_path)
 
-            update = Arel::UpdateManager.new(ActiveRecord::Base)
-                       .table(routes)
-                       .set([[routes[:path], replace_statement]])
-                       .where(Arel::Nodes::SqlLiteral.new(filter))
+            update = arel_update_manager
+              .table(routes)
+              .set([[routes[:path], replace_statement]])
+              .where(Arel::Nodes::SqlLiteral.new(filter))
 
             execute(update.to_sql)
           end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index de0044fc149..84d37f77fbb 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -1569,7 +1569,8 @@ module Gitlab
       end
 
       def checksum
-        gitaly_migrate(:calculate_checksum) do |is_enabled|
+        gitaly_migrate(:calculate_checksum,
+                      status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
           if is_enabled
             gitaly_repository_client.calculate_checksum
           else
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index a7e055ac444..c741dabe168 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -19,6 +19,7 @@ module Gitlab
       gon.gitlab_logo            = ActionController::Base.helpers.asset_path('gitlab_logo.png')
       gon.sprite_icons           = IconsHelper.sprite_icon_path
       gon.sprite_file_icons      = IconsHelper.sprite_file_icons_path
+      gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites')
       gon.test_env               = Rails.env.test?
       gon.suggested_label_colors = LabelsHelper.suggested_colors
 
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
index 10bec7a90da..e5a0fdae7ef 100644
--- a/lib/gitlab/redis/shared_state.rb
+++ b/lib/gitlab/redis/shared_state.rb
@@ -5,6 +5,8 @@ module Gitlab
   module Redis
     class SharedState < ::Gitlab::Redis::Wrapper
       SESSION_NAMESPACE = 'session:gitlab'.freeze
+      USER_SESSIONS_NAMESPACE = 'session:user:gitlab'.freeze
+      USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'.freeze
       DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze
       REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze
 
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index 156115f8a8f..4a691d640b3 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -294,17 +294,7 @@ module Gitlab
     #   add_namespace("default", "gitlab")
     #
     def add_namespace(storage, name)
-      Gitlab::GitalyClient.migrate(:add_namespace,
-                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
-        else
-          path = full_path(storage, name)
-          FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name)
-        end
-      end
-    rescue Errno::EEXIST => e
-      Rails.logger.warn("Directory exists as a file: #{e} at: #{path}")
+      Gitlab::GitalyClient::NamespaceService.new(storage).add(name)
     rescue GRPC::InvalidArgument => e
       raise ArgumentError, e.message
     end
@@ -316,14 +306,7 @@ module Gitlab
     #   rm_namespace("default", "gitlab")
     #
     def rm_namespace(storage, name)
-      Gitlab::GitalyClient.migrate(:remove_namespace,
-                               status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
-        else
-          FileUtils.rm_r(full_path(storage, name), force: true)
-        end
-      end
+      Gitlab::GitalyClient::NamespaceService.new(storage).remove(name)
     rescue GRPC::InvalidArgument => e
       raise ArgumentError, e.message
     end
@@ -335,17 +318,7 @@ module Gitlab
     #   mv_namespace("/path/to/storage", "gitlab", "gitlabhq")
     #
     def mv_namespace(storage, old_name, new_name)
-      Gitlab::GitalyClient.migrate(:rename_namespace,
-                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage)
-            .rename(old_name, new_name)
-        else
-          break false if exists?(storage, new_name) || !exists?(storage, old_name)
-
-          FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name))
-        end
-      end
+      Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name)
     rescue GRPC::InvalidArgument
       false
     end
@@ -370,17 +343,8 @@ module Gitlab
     #   exists?(storage, 'gitlab')
     #   exists?(storage, 'gitlab/cookies.git')
     #
-    # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385
     def exists?(storage, dir_name)
-      Gitlab::GitalyClient.migrate(:namespace_exists,
-                                   status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
-        if enabled
-          Gitlab::GitalyClient::NamespaceService.new(storage)
-            .exists?(dir_name)
-        else
-          File.exist?(full_path(storage, dir_name))
-        end
-      end
+      Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name)
     end
 
     protected