Merge branch 'jej/lfs-object-storage' into 'master'

Can migrate LFS objects to S3 style object storage

Closes #2841

See merge request !2760

2 files changed, 79 insertions, 30 deletions

diff --git a/spec/controllers/projects/artifacts_controller_spec.rb b/spec/controllers/projects/artifacts_controller_spec.rb
index caa63e7bd22..2bd8f8e2bfc 100644
--- a/spec/controllers/projects/artifacts_controller_spec.rb
+++ b/spec/controllers/projects/artifacts_controller_spec.rb
@@ -22,7 +22,7 @@ describe Projects::ArtifactsController do
 
   describe 'GET download' do
     it 'sends the artifacts file' do
-      expect(controller).to receive(:send_file).with(job.artifacts_file.path, disposition: 'attachment').and_call_original
+      expect(controller).to receive(:send_file).with(job.artifacts_file.path, hash_including(disposition: 'attachment')).and_call_original
 
       get :download, namespace_id: project.namespace, project_id: project, job_id: job
     end
@@ -66,19 +66,52 @@ describe Projects::ArtifactsController do
 
   describe 'GET raw' do
     context 'when the file exists' do
-      it 'serves the file using workhorse' do
-        get :raw, namespace_id: project.namespace, project_id: project, job_id: job, path: 'ci_artifacts.txt'
+      let(:path) { 'ci_artifacts.txt' }
+      let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline, artifacts_file_store: store, artifacts_metadata_store: store) }
 
-        send_data = response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]
+      shared_examples 'a valid file' do
+        it 'serves the file using workhorse' do
+          subject
 
-        expect(send_data).to start_with('artifacts-entry:')
+          expect(send_data).to start_with('artifacts-entry:')
 
-        base64_params = send_data.sub(/\Aartifacts\-entry:/, '')
-        params = JSON.parse(Base64.urlsafe_decode64(base64_params))
+          expect(params.keys).to eq(%w(Archive Entry))
+          expect(params['Archive']).to start_with(archive_path)
+          # On object storage, the URL can end with a query string
+          expect(params['Archive']).to match(/build_artifacts.zip(\?[^?]+)?$/)
+          expect(params['Entry']).to eq(Base64.encode64('ci_artifacts.txt'))
+        end
+
+        def send_data
+          response.headers[Gitlab::Workhorse::SEND_DATA_HEADER]
+        end
 
-        expect(params.keys).to eq(%w(Archive Entry))
-        expect(params['Archive']).to end_with('build_artifacts.zip')
-        expect(params['Entry']).to eq(Base64.encode64('ci_artifacts.txt'))
+        def params
+          @params ||= begin
+            base64_params = send_data.sub(/\Aartifacts\-entry:/, '')
+            JSON.parse(Base64.urlsafe_decode64(base64_params))
+          end
+        end
+      end
+
+      context 'when using local file storage' do
+        it_behaves_like 'a valid file' do
+          let(:store) { ObjectStoreUploader::LOCAL_STORE }
+          let(:archive_path) { ArtifactUploader.local_store_path }
+        end
+      end
+
+      context 'when using remote file storage' do
+        before do
+          stub_artifacts_object_storage
+        end
+
+        it_behaves_like 'a valid file' do
+          let!(:artifact) { create(:ci_job_artifact, :archive, :remote_store, job: job) }
+          let!(:job) { create(:ci_build, :success, pipeline: pipeline) }
+          let(:store) { ObjectStorage::Store::REMOTE }
+          let(:archive_path) { 'https://' }
+        end
       end
     end
   end
diff --git a/spec/controllers/projects/raw_controller_spec.rb b/spec/controllers/projects/raw_controller_spec.rb
index b4eaab29fed..7e3d6574e8d 100644
--- a/spec/controllers/projects/raw_controller_spec.rb
+++ b/spec/controllers/projects/raw_controller_spec.rb
@@ -8,10 +8,7 @@ describe Projects::RawController do
       let(:id) { 'master/README.md' }
 
       it 'delivers ASCII file' do
-        get(:show,
-            namespace_id: public_project.namespace.to_param,
-            project_id: public_project,
-            id: id)
+        get_show(public_project, id)
 
         expect(response).to have_http_status(200)
         expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8')
@@ -25,10 +22,7 @@ describe Projects::RawController do
       let(:id) { 'master/files/images/6049019_460s.jpg' }
 
       it 'sets image content type header' do
-        get(:show,
-            namespace_id: public_project.namespace.to_param,
-            project_id: public_project,
-            id: id)
+        get_show(public_project, id)
 
         expect(response).to have_http_status(200)
         expect(response.header['Content-Type']).to eq('image/jpeg')
@@ -54,21 +48,40 @@ describe Projects::RawController do
 
           it 'serves the file' do
             expect(controller).to receive(:send_file).with("#{Gitlab.config.shared.path}/lfs-objects/91/ef/f75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897", filename: 'lfs_object.iso', disposition: 'attachment')
-            get(:show,
-                namespace_id: public_project.namespace.to_param,
-                project_id: public_project,
-                id: id)
+            get_show(public_project, id)
 
             expect(response).to have_http_status(200)
           end
+
+          context 'and lfs uses object storage' do
+            before do
+              lfs_object.file = fixture_file_upload(Rails.root + "spec/fixtures/dk.png", "`/png")
+              lfs_object.save!
+              stub_lfs_object_storage
+              lfs_object.file.migrate!(LfsObjectUploader::REMOTE_STORE)
+            end
+
+            it 'responds with redirect to file' do
+              get_show(public_project, id)
+
+              expect(response).to have_gitlab_http_status(302)
+              expect(response.location).to include(lfs_object.reload.file.path)
+            end
+
+            it 'sets content disposition' do
+              get_show(public_project, id)
+
+              file_uri = URI.parse(response.location)
+              params = CGI.parse(file_uri.query)
+
+              expect(params["response-content-disposition"].first).to eq 'attachment;filename="lfs_object.iso"'
+            end
+          end
         end
 
         context 'when project does not have access' do
           it 'does not serve the file' do
-            get(:show,
-                namespace_id: public_project.namespace.to_param,
-                project_id: public_project,
-                id: id)
+            get_show(public_project, id)
 
             expect(response).to have_http_status(404)
           end
@@ -81,10 +94,7 @@ describe Projects::RawController do
         end
 
         it 'delivers ASCII file' do
-          get(:show,
-              namespace_id: public_project.namespace.to_param,
-              project_id: public_project,
-              id: id)
+          get_show(public_project, id)
 
           expect(response).to have_http_status(200)
           expect(response.header['Content-Type']).to eq('text/plain; charset=utf-8')
@@ -95,4 +105,10 @@ describe Projects::RawController do
       end
     end
   end
+
+  def get_show(project, id)
+    get(:show, namespace_id: project.namespace.to_param,
+               project_id: project,
+               id: id)
+  end
 end