5 files changed, 92 insertions, 71 deletions

diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
index 99b3168d9eb..2301ec9b228 100644
--- a/lib/tasks/brakeman.rake
+++ b/lib/tasks/brakeman.rake
@@ -2,7 +2,7 @@ desc 'Security check via brakeman'
 task :brakeman do
   # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
   # requests are welcome!
-  if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z))
+  if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
     puts 'Security check succeed'
   else
     puts 'Security check failed'
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 8ae1b6a626a..eb0f757aea7 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -1,11 +1,14 @@
 namespace :gitlab do
   namespace :cleanup do
+    HASHED_REPOSITORY_NAME = '@hashed'.freeze
+
     desc "GitLab | Cleanup | Clean namespaces"
     task dirs: :environment  do
       warn_user_is_not_gitlab
       remove_flag = ENV['REMOVE']
 
-      namespaces = Namespace.pluck(:path)
+      namespaces  = Namespace.pluck(:path)
+      namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
       Gitlab.config.repositories.storages.each do |name, repository_storage|
         git_base_path = repository_storage['path']
         all_dirs = Dir.glob(git_base_path + '/*')
@@ -59,7 +62,11 @@ namespace :gitlab do
               .sub(%r{^/*}, '')
               .chomp('.git')
               .chomp('.wiki')
-            next if Project.find_by_full_path(repo_with_namespace)
+
+            # TODO ignoring hashed repositories for now.  But revisit to fully support
+            # possible orphaned hashed repos
+            next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace)
+
             new_path = path + move_suffix
             puts path.inspect + ' -> ' + new_path.inspect
             File.rename(path, new_path)
@@ -75,6 +82,7 @@ namespace :gitlab do
 
       User.find_each do |user|
         next unless user.ldap_user?
+
         print "#{user.name} (#{user.ldap_identity.extern_uid}) ..."
         if Gitlab::LDAP::Access.allowed?(user)
           puts " [OK]".color(:green)
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 8377fe3269d..4d880c05f99 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -14,18 +14,18 @@ namespace :gitlab do
 
       checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
 
+      command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE]
+
       _, status = Gitlab::Popen.popen(%w[which gmake])
-      command = status.zero? ? ['gmake'] : ['make']
+      command << (status.zero? ? 'gmake' : 'make')
 
-      if Rails.env.test?
-        command += %W[BUNDLE_PATH=#{Bundler.bundle_path}]
-      end
+      command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
 
       Dir.chdir(args.dir) do
         create_gitaly_configuration
         # In CI we run scripts/gitaly-test-build instead of this command
         unless ENV['CI'].present?
-          Bundler.with_original_env { run_command!(%w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + command) }
+          Bundler.with_original_env { run_command!(command) }
         end
       end
     end
@@ -78,13 +78,18 @@ namespace :gitlab do
       config[:auth] = { token: 'secret' } if Rails.env.test?
       config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby
       config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
+      config[:bin_dir] = Gitlab.config.gitaly.client_path
+
       TOML.dump(config)
     end
 
     def create_gitaly_configuration
-      File.open("config.toml", "w") do |f|
+      File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f|
         f.puts gitaly_configuration_toml
       end
+    rescue Errno::EEXIST
+      puts "Skipping config.toml generation:"
+      puts "A configuration file already exists."
     rescue ArgumentError => e
       puts "Skipping config.toml generation:"
       puts e.message
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
deleted file mode 100644
index 6cbc83b8973..00000000000
--- a/lib/tasks/gitlab/sidekiq.rake
+++ /dev/null
@@ -1,47 +0,0 @@
-namespace :gitlab do
-  namespace :sidekiq do
-    QUEUE = 'queue:post_receive'.freeze
-
-    desc 'Drop all Sidekiq PostReceive jobs for a given project'
-    task :drop_post_receive, [:project] => :environment do |t, args|
-      unless args.project.present?
-        abort "Please specify the project you want to drop PostReceive jobs for:\n  rake gitlab:sidekiq:drop_post_receive[group/project]"
-      end
-      project_path = Project.find_by_full_path(args.project).repository.path_to_repo
-
-      Sidekiq.redis do |redis|
-        unless redis.exists(QUEUE)
-          abort "Queue #{QUEUE} is empty"
-        end
-
-        temp_queue = "#{QUEUE}_#{Time.now.to_i}"
-        redis.rename(QUEUE, temp_queue)
-
-        # At this point, then post_receive queue is empty. It may be receiving
-        # new jobs already. We will repopulate it with the old jobs, skipping the
-        # ones we want to drop.
-        dropped = 0
-        while (job = redis.lpop(temp_queue))
-          if repo_path(job) == project_path
-            dropped += 1
-          else
-            redis.rpush(QUEUE, job)
-          end
-        end
-        # The temp_queue will delete itself after we have popped all elements
-        # from it
-
-        puts "Dropped #{dropped} jobs containing #{project_path} from #{QUEUE}"
-      end
-    end
-
-    def repo_path(job)
-      job_args = JSON.parse(job)['args']
-      if job_args
-        job_args.first
-      else
-        nil
-      end
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index e05be4a3405..8ac73bc8ff2 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -2,10 +2,10 @@ namespace :gitlab do
   namespace :storage do
     desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
     task migrate_to_hashed: :environment do
-      legacy_projects_count = Project.with_legacy_storage.count
+      legacy_projects_count = Project.with_unmigrated_storage.count
 
       if legacy_projects_count == 0
-        puts 'There are no projects using legacy storage. Nothing to do!'
+        puts 'There are no projects requiring storage migration. Nothing to do!'
 
         next
       end
@@ -23,22 +23,42 @@ namespace :gitlab do
 
     desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
     task legacy_projects: :environment do
-      projects_summary(Project.with_legacy_storage)
+      relation_summary('projects', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Legacy Storage'
     task list_legacy_projects: :environment do
-      projects_list(Project.with_legacy_storage)
+      projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
     task hashed_projects: :environment do
-      projects_summary(Project.with_hashed_storage)
+      relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
     desc 'Gitlab | Storage | List existing projects using Hashed Storage'
     task list_hashed_projects: :environment do
-      projects_list(Project.with_hashed_storage)
+      projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
+    task legacy_attachments: :environment do
+      relation_summary('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Legacy Storage'
+    task list_legacy_attachments: :environment do
+      attachments_list('attachments using Legacy Storage', legacy_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage'
+    task hashed_attachments: :environment do
+      relation_summary('attachments using Hashed Storage', hashed_attachments_relation)
+    end
+
+    desc 'Gitlab | Storage | List existing project attachments using Hashed Storage'
+    task list_hashed_attachments: :environment do
+      attachments_list('attachments using Hashed Storage', hashed_attachments_relation)
     end
 
     def batch_size
@@ -46,29 +66,43 @@ namespace :gitlab do
     end
 
     def project_id_batches(&block)
-      Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
+      Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
         ids = relation.pluck(:id)
 
         yield ids.min, ids.max
       end
     end
 
-    def projects_summary(relation)
-      projects_count = relation.count
-      puts "* Found #{projects_count} projects".color(:green)
+    def legacy_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+          ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def hashed_attachments_relation
+      Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments])
+        JOIN projects
+        ON (uploads.model_type='Project' AND uploads.model_id=projects.id)
+      SQL
+    end
+
+    def relation_summary(relation_name, relation)
+      relation_count = relation.count
+      puts "* Found #{relation_count} #{relation_name}".color(:green)
 
-      projects_count
+      relation_count
     end
 
-    def projects_list(relation)
-      projects_count = projects_summary(relation)
+    def projects_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
 
       projects = relation.with_route
       limit = ENV.fetch('LIMIT', 500).to_i
 
-      return unless projects_count > 0
+      return unless relation_count > 0
 
-      puts "  ! Displaying first #{limit} projects..." if projects_count > limit
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
 
       counter = 0
       projects.find_in_batches(batch_size: batch_size) do |batch|
@@ -81,5 +115,26 @@ namespace :gitlab do
         end
       end
     end
+
+    def attachments_list(relation_name, relation)
+      relation_count = relation_summary(relation_name, relation)
+
+      limit = ENV.fetch('LIMIT', 500).to_i
+
+      return unless relation_count > 0
+
+      puts "  ! Displaying first #{limit} #{relation_name}..." if relation_count > limit
+
+      counter = 0
+      relation.find_in_batches(batch_size: batch_size) do |batch|
+        batch.each do |upload|
+          counter += 1
+
+          puts "  - #{upload.path} (id: #{upload.id})".color(:red)
+
+          return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
+        end
+      end
+    end
   end
 end