Merge commit '11c67e7c2f992299ff5918ce67995b73d1e0be6d' into object-storage-ee-to-ce-backport

7 files changed, 150 insertions, 12 deletions

diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 259a755d724..a42f02a84fd 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -3,8 +3,8 @@ namespace :gitlab do
     desc 'GitLab | Assets | Compile all frontend assets'
     task compile: [
       'yarn:check',
-      'rake:assets:precompile',
       'gettext:po_to_json',
+      'rake:assets:precompile',
       'webpack:compile',
       'fix_urls'
     ]
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 654f638c454..dfade1f3885 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -398,6 +398,35 @@ namespace :gitlab do
     end
   end
 
+  namespace :orphans do
+    desc 'Gitlab | Check for orphaned namespaces and repositories'
+    task check: :environment do
+      warn_user_is_not_gitlab
+      checks = [
+        SystemCheck::Orphans::NamespaceCheck,
+        SystemCheck::Orphans::RepositoryCheck
+      ]
+
+      SystemCheck.run('Orphans', checks)
+    end
+
+    desc 'GitLab | Check for orphaned namespaces in the repositories path'
+    task check_namespaces: :environment do
+      warn_user_is_not_gitlab
+      checks = [SystemCheck::Orphans::NamespaceCheck]
+
+      SystemCheck.run('Orphans', checks)
+    end
+
+    desc 'GitLab | Check for orphaned repositories in the repositories path'
+    task check_repositories: :environment do
+      warn_user_is_not_gitlab
+      checks = [SystemCheck::Orphans::RepositoryCheck]
+
+      SystemCheck.run('Orphans', checks)
+    end
+  end
+
   namespace :user do
     desc "GitLab | Check the integrity of a specific user's repositories"
     task :check_repos, [:username] => :environment do |t, args|
diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake
index 7ccda04a35f..b4d05f5995a 100644
--- a/lib/tasks/gitlab/dev.rake
+++ b/lib/tasks/gitlab/dev.rake
@@ -4,7 +4,10 @@ namespace :gitlab do
     task :ee_compat_check, [:branch] => :environment do |_, args|
       opts =
         if ENV['CI']
-          { branch: ENV['CI_COMMIT_REF_NAME'] }
+          {
+            ce_repo: ENV['CI_REPOSITORY_URL'],
+            branch: ENV['CI_COMMIT_REF_NAME']
+          }
         else
           unless args[:branch]
             puts "Must specify a branch as an argument".color(:red)
@@ -13,7 +16,10 @@ namespace :gitlab do
           args
         end
 
-      if Gitlab::EeCompatCheck.new(opts || {}).check
+      if File.basename(Rails.root) == 'gitlab-ee'
+        puts "Skipping EE projects"
+        exit 0
+      elsif Gitlab::EeCompatCheck.new(opts || {}).check
         exit 0
       else
         exit 1
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 08677a98fc1..8377fe3269d 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -50,6 +50,8 @@ namespace :gitlab do
     # only generate a configuration for the most common and simplest case: when
     # we have exactly one Gitaly process and we are sure it is running locally
     # because it uses a Unix socket.
+    # For development and testing purposes, an extra storage is added to gitaly,
+    # which is not known to Rails, but must be explicitly stubbed.
     def gitaly_configuration_toml(gitaly_ruby: true)
       storages = []
       address = nil
@@ -67,6 +69,11 @@ namespace :gitlab do
 
         storages << { name: key, path: val['path'] }
       end
+
+      if Rails.env.test?
+        storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
+      end
+
       config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
       config[:auth] = { token: 'secret' } if Rails.env.test?
       config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 42825f29e32..0e6aed32c52 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -79,7 +79,7 @@ namespace :gitlab do
         if File.exist?(path_to_repo)
           print '-'
         else
-          if Gitlab::Shell.new.add_repository(project.repository_storage_path,
+          if Gitlab::Shell.new.add_repository(project.repository_storage,
                                               project.disk_path)
             print '.'
           else
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
new file mode 100644
index 00000000000..e05be4a3405
--- /dev/null
+++ b/lib/tasks/gitlab/storage.rake
@@ -0,0 +1,85 @@
+namespace :gitlab do
+  namespace :storage do
+    desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
+    task migrate_to_hashed: :environment do
+      legacy_projects_count = Project.with_legacy_storage.count
+
+      if legacy_projects_count == 0
+        puts 'There are no projects using legacy storage. Nothing to do!'
+
+        next
+      end
+
+      print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
+
+      project_id_batches do |start, finish|
+        StorageMigratorWorker.perform_async(start, finish)
+
+        print '.'
+      end
+
+      puts ' Done!'
+    end
+
+    desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
+    task legacy_projects: :environment do
+      projects_summary(Project.with_legacy_storage)
+    end
+
+    desc 'Gitlab | Storage | List existing projects using Legacy Storage'
+    task list_legacy_projects: :environment do
+      projects_list(Project.with_legacy_storage)
+    end
+
+    desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
+    task hashed_projects: :environment do
+      projects_summary(Project.with_hashed_storage)
+    end
+
+    desc 'Gitlab | Storage | List existing projects using Hashed Storage'
+    task list_hashed_projects: :environment do
+      projects_list(Project.with_hashed_storage)
+    end
+
+    def batch_size
+      ENV.fetch('BATCH', 200).to_i
+    end
+
+    def project_id_batches(&block)
+      Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+        ids = relation.pluck(:id)
+
+        yield ids.min, ids.max
+      end
+    end
+
+    def projects_summary(relation)
+      projects_count = relation.count
+      puts "* Found #{projects_count} projects".color(:green)
+
+      projects_count
+    end
+
+    def projects_list(relation)
+      projects_count = projects_summary(relation)
+
+      projects = relation.with_route
+      limit = ENV.fetch('LIMIT', 500).to_i
+
+      return unless projects_count > 0
+
+      puts "  ! Displaying first #{limit} projects..." if projects_count > limit
+
+      counter = 0
+      projects.find_in_batches(batch_size: batch_size) do |batch|
+        batch.each do |project|
+          counter += 1
+
+          puts "  - #{project.full_path} (id: #{project.id})".color(:red)
+
+          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake
index 4d485108cf6..7f86fd7b45e 100644
--- a/lib/tasks/import.rake
+++ b/lib/tasks/import.rake
@@ -39,13 +39,19 @@ class GithubImport
   def import!
     @project.force_import_start
 
+    import_success = false
+
     timings = Benchmark.measure do
-      Github::Import.new(@project, @options).execute
+      import_success = Github::Import.new(@project, @options).execute
     end
 
-    puts "Import finished. Timings: #{timings}".color(:green)
-
-    @project.import_finish
+    if import_success
+      @project.import_finish
+      puts "Import finished. Timings: #{timings}".color(:green)
+    else
+      puts "Import was not successful. Errors were as follows:"
+      puts @project.import_error
+    end
   end
 
   def new_project
@@ -53,18 +59,23 @@ class GithubImport
       namespace_path, _sep, name = @project_path.rpartition('/')
       namespace = find_or_create_namespace(namespace_path)
 
-      Projects::CreateService.new(
+      project = Projects::CreateService.new(
         @current_user,
         name: name,
         path: name,
         description: @repo['description'],
         namespace_id: namespace.id,
         visibility_level: visibility_level,
-        import_type: 'github',
-        import_source: @repo['full_name'],
-        import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@"),
         skip_wiki: @repo['has_wiki']
       ).execute
+
+      project.update!(
+        import_type: 'github',
+        import_source: @repo['full_name'],
+        import_url: @repo['clone_url'].sub('://', "://#{@options[:token]}@")
+      )
+
+      project
     end
   end