Merge branch 'live-trace-v2' into live-trace-v2-efficient-destroy-all

11 files changed, 122 insertions, 13 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 1619c1a09ee..a9bab5c56cf 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -136,6 +136,7 @@ module API
 
       def self.preload_relation(projects_relation, options =  {})
         projects_relation.preload(:project_feature, :route)
+                         .preload(:import_state)
                          .preload(namespace: [:route, :owner],
                                   tags: :taggings)
       end
@@ -242,13 +243,18 @@ module API
       expose :requested_at
     end
 
-    class Group < Grape::Entity
-      expose :id, :name, :path, :description, :visibility
+    class BasicGroupDetails < Grape::Entity
+      expose :id
+      expose :web_url
+      expose :name
+    end
+
+    class Group < BasicGroupDetails
+      expose :path, :description, :visibility
       expose :lfs_enabled?, as: :lfs_enabled
       expose :avatar_url do |group, options|
         group.avatar_url(only_path: false)
       end
-      expose :web_url
       expose :request_access_enabled
       expose :full_name, :full_path
 
@@ -984,6 +990,13 @@ module API
           options[:current_user].authorized_projects.where(id: runner.projects)
         end
       end
+      expose :groups, with: Entities::BasicGroupDetails do |runner, options|
+        if options[:current_user].admin?
+          runner.groups
+        else
+          options[:current_user].authorized_groups.where(id: runner.groups)
+        end
+      end
     end
 
     class RunnerRegistrationDetails < Grape::Entity
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 4d4fbe50f9f..67896ae1fc5 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -23,10 +23,13 @@ module API
         runner =
           if runner_registration_token_valid?
             # Create shared runner. Requires admin access
-            Ci::Runner.create(attributes.merge(is_shared: true))
+            Ci::Runner.create(attributes.merge(is_shared: true, runner_type: :instance_type))
           elsif project = Project.find_by(runners_token: params[:token])
-            # Create a specific runner for project.
-            project.runners.create(attributes)
+            # Create a specific runner for the project
+            project.runners.create(attributes.merge(runner_type: :project_type))
+          elsif group = Group.find_by(runners_token: params[:token])
+            # Create a specific runner for the group
+            group.runners.create(attributes.merge(runner_type: :group_type))
           end
 
         break forbidden! unless runner
diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb
index ae365fcdfaa..f79ce6bb809 100644
--- a/lib/gitlab/auth/omniauth_identity_linker_base.rb
+++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb
@@ -17,6 +17,10 @@ module Gitlab
         @changed
       end
 
+      def failed?
+        error_message.present?
+      end
+
       def error_message
         identity.validate
 
diff --git a/lib/gitlab/background_migration/populate_import_state.rb b/lib/gitlab/background_migration/populate_import_state.rb
new file mode 100644
index 00000000000..695a2a713c5
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_import_state.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This background migration creates all the records on the
+    # import state table for projects that are considered imports or forks
+    class PopulateImportState
+      def perform(start_id, end_id)
+        move_attributes_data_to_import_state(start_id, end_id)
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+
+      def move_attributes_data_to_import_state(start_id, end_id)
+        Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}")
+
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO project_mirror_data (project_id, status, jid, last_error)
+          SELECT id, import_status, import_jid, import_error
+          FROM projects
+          WHERE projects.import_status != 'none'
+          AND projects.id BETWEEN #{start_id} AND #{end_id}
+          AND NOT EXISTS (
+            SELECT id
+            FROM project_mirror_data
+            WHERE project_id = projects.id
+          )
+        SQL
+
+        ActiveRecord::Base.connection.execute <<~SQL
+          UPDATE projects
+          SET import_status = 'none'
+          WHERE import_status != 'none'
+          AND id BETWEEN #{start_id} AND #{end_id}
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/rollback_import_state_data.rb b/lib/gitlab/background_migration/rollback_import_state_data.rb
new file mode 100644
index 00000000000..a7c986747d8
--- /dev/null
+++ b/lib/gitlab/background_migration/rollback_import_state_data.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This background migration migrates all the data of import_state
+    # back to the projects table for projects that are considered imports or forks
+    class RollbackImportStateData
+      def perform(start_id, end_id)
+        move_attributes_data_to_project(start_id, end_id)
+      end
+
+      def move_attributes_data_to_project(start_id, end_id)
+        Rails.logger.info("#{self.class.name} - Moving import attributes data to projects table: #{start_id} - #{end_id}")
+
+        if Gitlab::Database.mysql?
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE projects, project_mirror_data
+            SET
+              projects.import_status = project_mirror_data.status,
+              projects.import_jid = project_mirror_data.jid,
+              projects.import_error = project_mirror_data.last_error
+            WHERE project_mirror_data.project_id = projects.id
+            AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id}
+          SQL
+        else
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE projects
+            SET
+              import_status = project_mirror_data.status,
+              import_jid = project_mirror_data.jid,
+              import_error = project_mirror_data.last_error
+            FROM project_mirror_data
+            WHERE project_mirror_data.project_id = projects.id
+            AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id}
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
index 70732d26bbd..b5eb0cfa2f0 100644
--- a/lib/gitlab/ci/pipeline/chain/build.rb
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -14,7 +14,8 @@ module Gitlab
               trigger_requests: Array(@command.trigger_request),
               user: @command.current_user,
               pipeline_schedule: @command.schedule,
-              protected: @command.protected_ref?
+              protected: @command.protected_ref?,
+              variables_attributes: Array(@command.variables_attributes)
             )
 
             @pipeline.set_config_source
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index a1849b01c5d..a53c80d34f7 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -7,7 +7,7 @@ module Gitlab # rubocop:disable Naming/FileName
           :origin_ref, :checkout_sha, :after_sha, :before_sha,
           :trigger_request, :schedule,
           :ignore_skip_ci, :save_incompleted,
-          :seeds_block
+          :seeds_block, :variables_attributes
         ) do
           include Gitlab::Utils::StrongMemoize
 
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 84d37f77fbb..60ce8cfc195 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -20,6 +20,9 @@ module Gitlab
         GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
       ].freeze
       SEARCH_CONTEXT_LINES = 3
+      # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698
+      # We copied these two prefixes into gitaly-go, so don't change these
+      # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX)
       REBASE_WORKTREE_PREFIX = 'rebase'.freeze
       SQUASH_WORKTREE_PREFIX = 'squash'.freeze
       GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze
@@ -1671,10 +1674,14 @@ module Gitlab
         end
       end
 
+      # This function is duplicated in Gitaly-Go, don't change it!
+      # https://gitlab.com/gitlab-org/gitaly/merge_requests/698
       def fresh_worktree?(path)
         File.exist?(path) && !clean_stuck_worktree(path)
       end
 
+      # This function is duplicated in Gitaly-Go, don't change it!
+      # https://gitlab.com/gitlab-org/gitaly/merge_requests/698
       def clean_stuck_worktree(path)
         return false unless File.mtime(path) < 15.minutes.ago
 
diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb
index 6da11e6ef08..b02b123c98e 100644
--- a/lib/gitlab/github_import/parallel_importer.rb
+++ b/lib/gitlab/github_import/parallel_importer.rb
@@ -32,7 +32,8 @@ module Gitlab
         Gitlab::SidekiqStatus
           .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
 
-        project.update_column(:import_jid, jid)
+        project.ensure_import_state
+        project.import_state&.update_column(:jid, jid)
 
         Stage::ImportRepositoryWorker
           .perform_async(project.id)
diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb
index 7edd0ad2033..b04d678cf98 100644
--- a/lib/gitlab/legacy_github_import/importer.rb
+++ b/lib/gitlab/legacy_github_import/importer.rb
@@ -78,7 +78,8 @@ module Gitlab
       def handle_errors
         return unless errors.any?
 
-        project.update_column(:import_error, {
+        project.ensure_import_state
+        project.import_state&.update_column(:last_error, {
           message: 'The remote data could not be fully imported.',
           errors: errors
         }.to_json)
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index ae136202f0c..08f6a54776f 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -25,9 +25,9 @@ module Gitlab
     end
 
     TEMPLATES_TABLE = [
-      ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, gemfile, rakefile, and .gitlab-ci.yml file, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'),
-      ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw, pom.xml, and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'),
-      ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure and .gitlab-ci.yml file to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express')
+      ProjectTemplate.new('rails', 'Ruby on Rails', 'Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/rails'),
+      ProjectTemplate.new('spring', 'Spring', 'Includes an MVC structure, mvnw and pom.xml to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/spring'),
+      ProjectTemplate.new('express', 'NodeJS Express', 'Includes an MVC structure to help you get started.', 'https://gitlab.com/gitlab-org/project-templates/express')
     ].freeze
 
     class << self