summaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
Diffstat (limited to 'app')
-rw-r--r--app/controllers/concerns/send_file_upload.rb14
-rw-r--r--app/controllers/projects/artifacts_controller.rb7
-rw-r--r--app/controllers/projects/lfs_storage_controller.rb3
-rw-r--r--app/controllers/projects/raw_controller.rb3
-rw-r--r--app/models/ci/build.rb21
-rw-r--r--app/models/lfs_object.rb2
-rw-r--r--app/services/projects/update_pages_service.rb30
-rw-r--r--app/uploaders/artifact_uploader.rb32
-rw-r--r--app/uploaders/lfs_object_uploader.rb21
-rw-r--r--app/uploaders/object_store_uploader.rb197
-rw-r--r--app/views/projects/jobs/_sidebar.html.haml6
-rw-r--r--app/workers/object_storage_upload_worker.rb19
12 files changed, 289 insertions, 66 deletions
diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb
new file mode 100644
index 00000000000..d4de4cf1fda
--- /dev/null
+++ b/app/controllers/concerns/send_file_upload.rb
@@ -0,0 +1,14 @@
+module SendFileUpload
+ def send_upload(file_upload, send_params: {}, redirect_params: {}, attachment: nil)
+ if attachment
+ redirect_params[:query] = { "response-content-disposition" => "attachment;filename=#{attachment.inspect}" }
+ send_params.merge!(filename: attachment, disposition: 'attachment')
+ end
+
+ if file_upload.file_storage?
+ send_file file_upload.path, send_params
+ else
+ redirect_to file_upload.url(**redirect_params)
+ end
+ end
+end
diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb
index 0837451cc49..3995a2fc37a 100644
--- a/app/controllers/projects/artifacts_controller.rb
+++ b/app/controllers/projects/artifacts_controller.rb
@@ -1,6 +1,7 @@
class Projects::ArtifactsController < Projects::ApplicationController
include ExtractsPath
include RendersBlob
+ include SendFileUpload
layout 'project'
before_action :authorize_read_build!
@@ -10,11 +11,7 @@ class Projects::ArtifactsController < Projects::ApplicationController
before_action :entry, only: [:file]
def download
- if artifacts_file.file_storage?
- send_file artifacts_file.path, disposition: 'attachment'
- else
- redirect_to artifacts_file.url
- end
+ send_upload(artifacts_file, attachment: artifacts_file.filename)
end
def browse
diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb
index 32759672b6c..134892b5d7b 100644
--- a/app/controllers/projects/lfs_storage_controller.rb
+++ b/app/controllers/projects/lfs_storage_controller.rb
@@ -1,6 +1,7 @@
class Projects::LfsStorageController < Projects::GitHttpClientController
include LfsRequest
include WorkhorseRequest
+ include SendFileUpload
skip_before_action :verify_workhorse_api!, only: [:download, :upload_finalize]
@@ -11,7 +12,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
return
end
- send_file lfs_object.file.path, content_type: "application/octet-stream"
+ send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" })
end
def upload_authorize
diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb
index a02cc477e08..9bc774b7636 100644
--- a/app/controllers/projects/raw_controller.rb
+++ b/app/controllers/projects/raw_controller.rb
@@ -2,6 +2,7 @@
class Projects::RawController < Projects::ApplicationController
include ExtractsPath
include BlobHelper
+ include SendFileUpload
before_action :require_non_empty_project
before_action :assign_ref_vars
@@ -31,7 +32,7 @@ class Projects::RawController < Projects::ApplicationController
lfs_object = find_lfs_object
if lfs_object && lfs_object.project_allowed_access?(@project)
- send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment'
+ send_upload(lfs_object.file, attachment: @blob.name)
else
render_404
end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 6ca46ae89c1..ebeab87d8fc 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -34,6 +34,7 @@ module Ci
scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) }
scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
+ scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, ArtifactUploader::LOCAL_STORE]) }
scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
scope :ref_protected, -> { where(protected: true) }
@@ -324,17 +325,27 @@ module Ci
!artifacts_expired? && artifacts_file.exists?
end
+ def browsable_artifacts?
+ artifacts_metadata?
+ end
+
+ def downloadable_single_artifacts_file?
+ artifacts_metadata? && artifacts_file.file_storage?
+ end
+
def artifacts_metadata?
artifacts? && artifacts_metadata.exists?
end
def artifacts_metadata_entry(path, **options)
- metadata = Gitlab::Ci::Build::Artifacts::Metadata.new(
- artifacts_metadata.path,
- path,
- **options)
+ artifacts_metadata.use_file do |metadata_path|
+ metadata = Gitlab::Ci::Build::Artifacts::Metadata.new(
+ metadata_path,
+ path,
+ **options)
- metadata.to_entry
+ metadata.to_entry
+ end
end
def erase_artifacts!
diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb
index b7cf96abe83..0056b0f80f4 100644
--- a/app/models/lfs_object.rb
+++ b/app/models/lfs_object.rb
@@ -4,6 +4,8 @@ class LfsObject < ActiveRecord::Base
validates :oid, presence: true, uniqueness: true
+ scope :with_files_stored_locally, ->() { where(file_store: [nil, LfsObjectUploader::LOCAL_STORE]) }
+
mount_uploader :file, LfsObjectUploader
def storage_project(project)
diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb
index d34903c9989..78cf195d3b3 100644
--- a/app/services/projects/update_pages_service.rb
+++ b/app/services/projects/update_pages_service.rb
@@ -69,9 +69,9 @@ module Projects
end
def extract_archive!(temp_path)
- if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz')
+ if artifacts_filename.ends_with?('.tar.gz') || artifacts_filename.ends_with?('.tgz')
extract_tar_archive!(temp_path)
- elsif artifacts.ends_with?('.zip')
+ elsif artifacts_filename.ends_with?('.zip')
extract_zip_archive!(temp_path)
else
raise 'unsupported artifacts format'
@@ -79,11 +79,13 @@ module Projects
end
def extract_tar_archive!(temp_path)
- results = Open3.pipeline(%W(gunzip -c #{artifacts}),
- %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
- %W(tar -x -C #{temp_path} #{SITE_PATH}),
- err: '/dev/null')
- raise 'pages failed to extract' unless results.compact.all?(&:success?)
+ build.artifacts_file.use_file do |artifacts_path|
+ results = Open3.pipeline(%W(gunzip -c #{artifacts_path}),
+ %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
+ %W(tar -x -C #{temp_path} #{SITE_PATH}),
+ err: '/dev/null')
+ raise 'pages failed to extract' unless results.compact.all?(&:success?)
+ end
end
def extract_zip_archive!(temp_path)
@@ -101,8 +103,10 @@ module Projects
# -n never overwrite existing files
# We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories
site_path = File.join(SITE_PATH, '*')
- unless system(*%W(unzip -qq -n #{artifacts} #{site_path} -d #{temp_path}))
- raise 'pages failed to extract'
+ build.artifacts_file.use_file do |artifacts_path|
+ unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path}))
+ raise 'pages failed to extract'
+ end
end
end
@@ -133,6 +137,10 @@ module Projects
1 + max_size / BLOCK_SIZE
end
+ def artifacts_filename
+ build.artifacts_file.filename
+ end
+
def max_size
max_pages_size = current_application_settings.max_pages_size.megabytes
@@ -161,10 +169,6 @@ module Projects
build.ref
end
- def artifacts
- build.artifacts_file.path
- end
-
def latest_sha
project.commit(build.ref).try(:sha).to_s
end
diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb
index 14addb6cf14..f6e32aae2fd 100644
--- a/app/uploaders/artifact_uploader.rb
+++ b/app/uploaders/artifact_uploader.rb
@@ -1,39 +1,17 @@
-class ArtifactUploader < GitlabUploader
- storage :file
+class ArtifactUploader < ObjectStoreUploader
+ storage_options Gitlab.config.artifacts
- attr_reader :job, :field
-
- def self.local_artifacts_store
+ def self.local_store_path
Gitlab.config.artifacts.path
end
def self.artifacts_upload_path
- File.join(self.local_artifacts_store, 'tmp/uploads/')
- end
-
- def initialize(job, field)
- @job, @field = job, field
- end
-
- def store_dir
- default_local_path
- end
-
- def cache_dir
- File.join(self.class.local_artifacts_store, 'tmp/cache')
- end
-
- def work_dir
- File.join(self.class.local_artifacts_store, 'tmp/work')
+ File.join(self.local_store_path, 'tmp/uploads/')
end
private
- def default_local_path
- File.join(self.class.local_artifacts_store, default_path)
- end
-
def default_path
- File.join(job.created_at.utc.strftime('%Y_%m'), job.project_id.to_s, job.id.to_s)
+ File.join(subject.created_at.utc.strftime('%Y_%m'), subject.project_id.to_s, subject.id.to_s)
end
end
diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb
index d11ebf0f9ca..8a5f599c1d3 100644
--- a/app/uploaders/lfs_object_uploader.rb
+++ b/app/uploaders/lfs_object_uploader.rb
@@ -1,19 +1,18 @@
-class LfsObjectUploader < GitlabUploader
- storage :file
+class LfsObjectUploader < ObjectStoreUploader
+ storage_options Gitlab.config.lfs
+ after :store, :schedule_migration_to_object_storage
- def store_dir
- "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
- end
-
- def cache_dir
- "#{Gitlab.config.lfs.storage_path}/tmp/cache"
+ def self.local_store_path
+ Gitlab.config.lfs.storage_path
end
def filename
- model.oid[4..-1]
+ subject.oid[4..-1]
end
- def work_dir
- File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work')
+ private
+
+ def default_path
+ "#{subject.oid[0, 2]}/#{subject.oid[2, 2]}"
end
end
diff --git a/app/uploaders/object_store_uploader.rb b/app/uploaders/object_store_uploader.rb
new file mode 100644
index 00000000000..3a742d4f715
--- /dev/null
+++ b/app/uploaders/object_store_uploader.rb
@@ -0,0 +1,197 @@
+require 'fog/aws'
+require 'carrierwave/storage/fog'
+
+class ObjectStoreUploader < GitlabUploader
+ before :store, :set_default_local_store
+ before :store, :verify_license!
+
+ LOCAL_STORE = 1
+ REMOTE_STORE = 2
+
+ class << self
+ def storage_options(options)
+ @storage_options = options
+ end
+
+ def object_store_options
+ @storage_options&.object_store
+ end
+
+ def object_store_enabled?
+ object_store_options&.enabled
+ end
+
+ def background_upload_enabled?
+ object_store_options&.background_upload
+ end
+
+ def object_store_credentials
+ @object_store_credentials ||= object_store_options&.connection&.to_hash&.deep_symbolize_keys
+ end
+
+ def object_store_directory
+ object_store_options&.remote_directory
+ end
+
+ def local_store_path
+ raise NotImplementedError
+ end
+ end
+
+ attr_reader :subject, :field
+
+ def initialize(subject, field)
+ @subject = subject
+ @field = field
+ end
+
+ def object_store
+ subject.public_send(:"#{field}_store")
+ end
+
+ def object_store=(value)
+ @storage = nil
+ subject.public_send(:"#{field}_store=", value)
+ end
+
+ def store_dir
+ if file_storage?
+ default_local_path
+ else
+ default_path
+ end
+ end
+
+ def use_file
+ if file_storage?
+ return yield path
+ end
+
+ begin
+ cache_stored_file!
+ yield cache_path
+ ensure
+ cache_storage.delete_dir!(cache_path(nil))
+ end
+ end
+
+ def filename
+ super || file&.filename
+ end
+
+ def migrate!(new_store)
+ raise 'Undefined new store' unless new_store
+
+ return unless object_store != new_store
+ return unless file
+
+ old_file = file
+ old_store = object_store
+
+ # for moving remote file we need to first store it locally
+ cache_stored_file! unless file_storage?
+
+ # change storage
+ self.object_store = new_store
+
+ storage.store!(file).tap do |new_file|
+ # since we change storage store the new storage
+ # in case of failure delete new file
+ begin
+ subject.save!
+ rescue => e
+ new_file.delete
+ self.object_store = old_store
+ raise e
+ end
+
+ old_file.delete
+ end
+ end
+
+ def schedule_migration_to_object_storage(new_file)
+ if self.class.object_store_enabled? && licensed? && file_storage?
+ ObjectStorageUploadWorker.perform_async(self.class.name, subject.class.name, field, subject.id)
+ end
+ end
+
+ def fog_directory
+ self.class.object_store_options.remote_directory
+ end
+
+ def fog_credentials
+ self.class.object_store_options.connection
+ end
+
+ def fog_public
+ false
+ end
+
+ def move_to_store
+ file.try(:storage) == storage
+ end
+
+ def move_to_cache
+ file.try(:storage) == cache_storage
+ end
+
+ # We block storing artifacts on Object Storage, not receiving
+ def verify_license!(new_file)
+ return if file_storage?
+
+ raise 'Object Storage feature is missing' unless licensed?
+ end
+
+ def exists?
+ file.try(:exists?)
+ end
+
+ def cache_dir
+ File.join(self.class.local_store_path, 'tmp/cache')
+ end
+
+ # Override this if you don't want to save local files by default to the Rails.root directory
+ def work_dir
+ # Default path set by CarrierWave:
+ # https://github.com/carrierwaveuploader/carrierwave/blob/v1.1.0/lib/carrierwave/uploader/cache.rb#L182
+ # CarrierWave.tmp_path
+ File.join(self.class.local_store_path, 'tmp/work')
+ end
+
+ def licensed?
+ License.feature_available?(:object_storage)
+ end
+
+ private
+
+ def set_default_local_store(new_file)
+ self.object_store = LOCAL_STORE unless self.object_store
+ end
+
+ def default_local_path
+ File.join(self.class.local_store_path, default_path)
+ end
+
+ def default_path
+ raise NotImplementedError
+ end
+
+ def storage
+ @storage ||=
+ if object_store == REMOTE_STORE
+ remote_storage
+ else
+ local_storage
+ end
+ end
+
+ def remote_storage
+ raise 'Object Storage is not enabled' unless self.class.object_store_enabled?
+
+ CarrierWave::Storage::Fog.new(self)
+ end
+
+ def local_storage
+ CarrierWave::Storage::File.new(self)
+ end
+end
diff --git a/app/views/projects/jobs/_sidebar.html.haml b/app/views/projects/jobs/_sidebar.html.haml
index b5067367802..5a12607afa4 100644
--- a/app/views/projects/jobs/_sidebar.html.haml
+++ b/app/views/projects/jobs/_sidebar.html.haml
@@ -35,9 +35,9 @@
= link_to download_project_job_artifacts_path(@project, @build), rel: 'nofollow', download: '', class: 'btn btn-sm btn-default' do
Download
- - if @build.artifacts_metadata?
- = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do
- Browse
+ - if @build.browsable_artifacts?
+ = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do
+ Browse
- if @build.trigger_request
.build-widget.block
diff --git a/app/workers/object_storage_upload_worker.rb b/app/workers/object_storage_upload_worker.rb
new file mode 100644
index 00000000000..0a374c4323f
--- /dev/null
+++ b/app/workers/object_storage_upload_worker.rb
@@ -0,0 +1,19 @@
+class ObjectStorageUploadWorker
+ include Sidekiq::Worker
+ include DedicatedSidekiqQueue
+
+ def perform(uploader_class_name, subject_class_name, file_field, subject_id)
+ uploader_class = uploader_class_name.constantize
+ subject_class = subject_class_name.constantize
+
+ return unless uploader_class.object_store_enabled?
+ return unless uploader_class.background_upload_enabled?
+
+ subject = subject_class.find(subject_id)
+ file = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend
+
+ return unless file.licensed?
+
+ file.migrate!(uploader_class::REMOTE_STORE)
+ end
+end
View Lorry Controller Status