Merge branch '4163-move-uploads-to-object-storage' into 'master'

Move uploads to object storage

Closes #4163

See merge request gitlab-org/gitlab-ee!3867

10 files changed, 21 insertions, 20 deletions

diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 80feb629d54..1f80646a2ea 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -215,9 +215,9 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
 
-        artifacts_upload_path = JobArtifactUploader.artifacts_upload_path
-        artifacts = uploaded_file(:file, artifacts_upload_path)
-        metadata = uploaded_file(:metadata, artifacts_upload_path)
+        workhorse_upload_path = JobArtifactUploader.workhorse_upload_path
+        artifacts = uploaded_file(:file, workhorse_upload_path)
+        metadata = uploaded_file(:metadata, workhorse_upload_path)
 
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 7a582a20056..4383124d150 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -3,7 +3,7 @@ require 'backup/files'
 module Backup
   class Artifacts < Files
     def initialize
-      super('artifacts', LegacyArtifactUploader.local_store_path)
+      super('artifacts', JobArtifactUploader.root)
     end
 
     def create_files_dir
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
index 81e95e5832d..759bdeb4bb3 100644
--- a/lib/gitlab/background_migration/populate_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb
@@ -143,7 +143,7 @@ module Gitlab
         end
 
         def absolute_path
-          File.join(CarrierWave.root, path)
+          File.join(Gitlab.config.uploads.storage_path, path)
         end
       end
 
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
index 476c46341ae..8d126a34dff 100644
--- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
@@ -10,9 +10,12 @@ module Gitlab
 
       FIND_BATCH_SIZE = 500
       RELATIVE_UPLOAD_DIR = "uploads".freeze
-      ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze
+      ABSOLUTE_UPLOAD_DIR = File.join(
+        Gitlab.config.uploads.storage_path,
+        RELATIVE_UPLOAD_DIR
+      )
       FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
-      START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/}
+      START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}
       EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
       EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
 
@@ -80,7 +83,7 @@ module Gitlab
         paths = []
 
         stdout.each_line("\0") do |line|
-          paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '')
+          paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '')
 
           if paths.size >= batch_size
             yield(paths)
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index 8fab5489616..3fdc3c27f73 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -27,7 +27,7 @@ module Gitlab
           with_link_in_tmp_dir(file.file) do |open_tmp_file|
             new_uploader.store!(open_tmp_file)
           end
-          new_uploader.to_markdown
+          new_uploader.markdown_link
         end
       end
 
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index 627a487d577..2f08dda55fd 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -17,15 +17,13 @@ module Gitlab
         false
       end
 
-      private
+      def uploads_path
+        FileUploader.absolute_base_dir(@project)
+      end
 
       def uploads_export_path
         File.join(@shared.export_path, 'uploads')
       end
-
-      def uploads_path
-        FileUploader.dynamic_path_segment(@project)
-      end
     end
   end
 end
diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb
index b5f41240529..7d7400bdabf 100644
--- a/lib/gitlab/uploads_transfer.rb
+++ b/lib/gitlab/uploads_transfer.rb
@@ -1,7 +1,7 @@
 module Gitlab
   class UploadsTransfer < ProjectTransfer
     def root_dir
-      File.join(CarrierWave.root, FileUploader.base_dir)
+      FileUploader.root
     end
   end
 end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 5ab6cd5a4ef..dfe8acd4833 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -51,14 +51,14 @@ module Gitlab
 
       def lfs_upload_ok(oid, size)
         {
-          StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload",
+          StoreLFSPath: LfsObjectUploader.workhorse_upload_path,
           LfsOid: oid,
           LfsSize: size
         }
       end
 
       def artifact_upload_ok
-        { TempPath: JobArtifactUploader.artifacts_upload_path }
+        { TempPath: JobArtifactUploader.workhorse_upload_path }
       end
 
       def send_git_blob(repository, blob)
diff --git a/lib/tasks/gitlab/artifacts.rake b/lib/tasks/gitlab/artifacts.rake
index 494317d99c7..bfca4bfb3f7 100644
--- a/lib/tasks/gitlab/artifacts.rake
+++ b/lib/tasks/gitlab/artifacts.rake
@@ -12,8 +12,8 @@ namespace :gitlab do
         .with_artifacts_stored_locally
         .find_each(batch_size: 10) do |build|
         begin
-          build.artifacts_file.migrate!(ObjectStoreUploader::REMOTE_STORE)
-          build.artifacts_metadata.migrate!(ObjectStoreUploader::REMOTE_STORE)
+          build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
+          build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
 
           logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage")
         rescue => e
diff --git a/lib/tasks/gitlab/lfs.rake b/lib/tasks/gitlab/lfs.rake
index c17c05f8589..a45e5ca91e0 100644
--- a/lib/tasks/gitlab/lfs.rake
+++ b/lib/tasks/gitlab/lfs.rake
@@ -10,7 +10,7 @@ namespace :gitlab do
       LfsObject.with_files_stored_locally
         .find_each(batch_size: 10) do |lfs_object|
           begin
-            lfs_object.file.migrate!(LfsObjectUploader::REMOTE_STORE)
+            lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE)
 
             logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage")
           rescue => e