[ci skip] Merge branch 'master' into 44427-state-management-with-vuex

* master: (544 commits)
  Bulk deleting refs is handled by Gitaly by default
  Allow assigning and filtering issuables by ancestor group labels
  Fix links to subdirectories of a directory with a plus character in its path
  Add banzai filter to detect commit message trailers and properly link the users
  Render MR commit SHA instead "diffs" when viable
  Fix a transient failure by removing unneeded expectations
  Revert changelog entry for removed feature
  Revert "Allow CI/CD Jobs being grouped on version strings"
  Add support for Sidekiq JSON logging
  Resolve "Protected branches count is wrong when a wildcard includes several protected branches"
  Remove unused form for admin application settings
  Use standard codequality job
  Resolve "Allow the configuration of a project's merge method via the API"
  Move the rest of application settings to expandable blocks
  [Rails5] Rename `sort` methods to `sort_by_attribute`
  Add better LDAP connection handling
  Updated components to PascalCase
  Handle invalid params when trying update_username
  Move network related app settings to expandable blocks
  [Rails5] Update Gemfile.rails5.lock [ci skip]
  ...

9 files changed, 91 insertions, 15 deletions

diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
new file mode 100644
index 00000000000..bfca4bfb3f7
--- /dev/null
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -0,0 +1,25 @@
+require 'logger'
+require 'resolv-replace'
+
+desc "GitLab | Migrate files for artifacts to comply with new storage format"
+namespace :gitlab do
+  namespace :artifacts do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of artifacts')
+
+      Ci::Build.joins(:project)
+        .with_artifacts_stored_locally
+        .find_each(batch_size: 10) do |build|
+        begin
+          build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
+          build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
+
+          logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
+        rescue => e
+          logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 2403f57f05a..abef8cd2bcc 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -61,7 +61,7 @@ namespace :gitlab do
       puts "Repo base directory exists?"
 
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage['path']
+        repo_base_path = repository_storage.legacy_disk_path
         print "#{name}... "
 
         if File.exist?(repo_base_path)
@@ -86,7 +86,7 @@ namespace :gitlab do
       puts "Repo storage directories are symlinks?"
 
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage['path']
+        repo_base_path = repository_storage.legacy_disk_path
         print "#{name}... "
 
         unless File.exist?(repo_base_path)
@@ -110,7 +110,7 @@ namespace :gitlab do
       puts "Repo paths access is drwxrws---?"
 
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage['path']
+        repo_base_path = repository_storage.legacy_disk_path
         print "#{name}... "
 
         unless File.exist?(repo_base_path)
@@ -140,7 +140,7 @@ namespace :gitlab do
       puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?"
 
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_base_path = repository_storage['path']
+        repo_base_path = repository_storage.legacy_disk_path
         print "#{name}... "
 
         unless File.exist?(repo_base_path)
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 2453079911d..d6d15285489 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
       namespaces  = Namespace.pluck(:path)
       namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        git_base_path = repository_storage['path']
+        git_base_path = repository_storage.legacy_disk_path
         all_dirs = Dir.glob(git_base_path + '/*')
 
         puts git_base_path.color(:yellow)
@@ -54,7 +54,7 @@ namespace :gitlab do
 
       move_suffix = "+orphaned+#{Time.now.to_i}"
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_root = repository_storage['path']
+        repo_root = repository_storage.legacy_disk_path
         # Look for global repos (legacy, depth 1) and normal repos (depth 2)
         IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
           find.each_line do |path|
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 45e9a1a1c72..47ed522aec3 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -68,7 +68,7 @@ namespace :gitlab do
       puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}"
       puts "Repository storage paths:"
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        puts "- #{name}: \t#{repository_storage['path']}"
+        puts "- #{name}: \t#{repository_storage.legacy_disk_path}"
       end
       puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}"
       puts "Git:\t\t#{Gitlab.config.git.bin_path}"
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
new file mode 100644
index 00000000000..a45e5ca91e0
--- /dev/null
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -0,0 +1,22 @@
+require 'logger'
+
+desc "GitLab | Migrate LFS objects to remote storage"
+namespace :gitlab do
+  namespace :lfs do
+    task migrate: :environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting transfer of LFS files to object storage')
+
+      LfsObject.with_files_stored_locally
+        .find_each(batch_size: 10) do |lfs_object|
+          begin
+            lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
+
+            logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
+          rescue => e
+            logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
+          end
+        end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake
index 7728c485e8d..6b22499a5c8 100644
--- a/lib/tasks/gitlab/two_factor.rake
+++ b/lib/tasks/gitlab/two_factor.rake
@@ -1,7 +1,7 @@
 namespace :gitlab do
   namespace :two_factor do
     desc "GitLab | Disable Two-factor authentication (2FA) for all users"
-    task disable_for_all_users: :environment do
+    task disable_for_all_users: :gitlab_environment do
       scope = User.with_two_factor
       count = scope.count
 
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
new file mode 100644
index 00000000000..78e18992a8e
--- /dev/null
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -0,0 +1,34 @@
+namespace :gitlab do
+  namespace :uploads do
+    desc 'GitLab | Uploads | Migrate the uploaded files to object storage'
+    task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
+      batch_size     = ENV.fetch('BATCH', 200).to_i
+      @to_store      = ObjectStorage::Store::REMOTE
+      @mounted_as    = args.mounted_as&.gsub(':', '')&.to_sym
+      @uploader_class = args.uploader_class.constantize
+      @model_class    = args.model_class.constantize
+
+      uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches
+    end
+
+    def enqueue_batch(batch, index)
+      job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+                                                         @model_class,
+                                                         @mounted_as,
+                                                         @to_store)
+      puts "Enqueued job ##{index}: #{job}"
+    rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+      # continue for the next batch
+      puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
+    end
+
+    def uploads
+      Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+      Upload
+        .where(store: [nil, ObjectStorage::Store::LOCAL],
+               uploader: @uploader_class.to_s,
+               model_type: @model_class.base_class.sti_name)
+    end
+  end
+end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index 1c7a8a90f5c..af30ecb0e9b 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -7,8 +7,8 @@ task setup_postgresql: :environment do
   require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes')
   require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
   require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
-  require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb')
   require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb')
+  require Rails.root.join('db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb')
 
   NamespacesProjectsPathLowerIndexes.new.up
   AddUsersLowerUsernameEmailIndexes.new.up
@@ -17,6 +17,6 @@ task setup_postgresql: :environment do
   AddLowerPathIndexToRedirectRoutes.new.up
   IndexRedirectRoutesPathForLike.new.up
   AddIndexOnNamespacesLowerName.new.up
-  ReworkRedirectRoutesIndexes.new.up
   UsersNameLowerIndex.new.up
+  AddPathIndexToRedirectRoutes.new.up
 end
diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake
index 3e01f91d32c..b52af81fc16 100644
--- a/lib/tasks/test.rake
+++ b/lib/tasks/test.rake
@@ -4,8 +4,3 @@ desc "GitLab | Run all tests"
 task :test do
   Rake::Task["gitlab:test"].invoke
 end
-
-unless Rails.env.production?
-  desc "GitLab | Run all tests on CI with simplecov"
-  task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec]
-end