diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/controllers/concerns/send_file_upload.rb | 14 | ||||
| -rw-r--r-- | app/controllers/projects/artifacts_controller.rb | 12 | ||||
| -rw-r--r-- | app/controllers/projects/lfs_storage_controller.rb | 3 | ||||
| -rw-r--r-- | app/controllers/projects/raw_controller.rb | 3 | ||||
| -rw-r--r-- | app/models/ci/build.rb | 17 | ||||
| -rw-r--r-- | app/models/ci/job_artifact.rb | 7 | ||||
| -rw-r--r-- | app/models/lfs_object.rb | 11 | ||||
| -rw-r--r-- | app/services/projects/update_pages_service.rb | 30 | ||||
| -rw-r--r-- | app/uploaders/job_artifact_uploader.rb | 20 | ||||
| -rw-r--r-- | app/uploaders/legacy_artifact_uploader.rb | 20 | ||||
| -rw-r--r-- | app/uploaders/lfs_object_uploader.rb | 18 | ||||
| -rw-r--r-- | app/uploaders/object_store_uploader.rb | 215 | ||||
| -rw-r--r-- | app/views/projects/jobs/_sidebar.html.haml | 6 | ||||
| -rw-r--r-- | app/workers/object_storage_upload_worker.rb | 23 |
14 files changed, 322 insertions, 77 deletions
diff --git a/app/controllers/concerns/send_file_upload.rb b/app/controllers/concerns/send_file_upload.rb new file mode 100644 index 00000000000..d4de4cf1fda --- /dev/null +++ b/app/controllers/concerns/send_file_upload.rb @@ -0,0 +1,14 @@ +module SendFileUpload + def send_upload(file_upload, send_params: {}, redirect_params: {}, attachment: nil) + if attachment + redirect_params[:query] = { "response-content-disposition" => "attachment;filename=#{attachment.inspect}" } + send_params.merge!(filename: attachment, disposition: 'attachment') + end + + if file_upload.file_storage? + send_file file_upload.path, send_params + else + redirect_to file_upload.url(**redirect_params) + end + end +end diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb index 0837451cc49..abc283d7aa9 100644 --- a/app/controllers/projects/artifacts_controller.rb +++ b/app/controllers/projects/artifacts_controller.rb @@ -1,6 +1,7 @@ class Projects::ArtifactsController < Projects::ApplicationController include ExtractsPath include RendersBlob + include SendFileUpload layout 'project' before_action :authorize_read_build! @@ -10,11 +11,7 @@ class Projects::ArtifactsController < Projects::ApplicationController before_action :entry, only: [:file] def download - if artifacts_file.file_storage? - send_file artifacts_file.path, disposition: 'attachment' - else - redirect_to artifacts_file.url - end + send_upload(artifacts_file, attachment: artifacts_file.filename) end def browse @@ -45,8 +42,7 @@ class Projects::ArtifactsController < Projects::ApplicationController end def raw - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:path]) + path = Gitlab::Ci::Build::Artifacts::Path.new(params[:path]) send_artifacts_entry(build, path) end @@ -75,7 +71,7 @@ class Projects::ArtifactsController < Projects::ApplicationController end def validate_artifacts! - render_404 unless build && build.artifacts? + render_404 unless build&.artifacts? end def build diff --git a/app/controllers/projects/lfs_storage_controller.rb b/app/controllers/projects/lfs_storage_controller.rb index 293869345bd..5b0f3d11d9e 100644 --- a/app/controllers/projects/lfs_storage_controller.rb +++ b/app/controllers/projects/lfs_storage_controller.rb @@ -1,6 +1,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController include LfsRequest include WorkhorseRequest + include SendFileUpload skip_before_action :verify_workhorse_api!, only: [:download, :upload_finalize] @@ -11,7 +12,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController return end - send_file lfs_object.file.path, content_type: "application/octet-stream" + send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" }) end def upload_authorize diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index a02cc477e08..9bc774b7636 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -2,6 +2,7 @@ class Projects::RawController < Projects::ApplicationController include ExtractsPath include BlobHelper + include SendFileUpload before_action :require_non_empty_project before_action :assign_ref_vars @@ -31,7 +32,7 @@ class Projects::RawController < Projects::ApplicationController lfs_object = find_lfs_object if lfs_object && lfs_object.project_allowed_access?(@project) - send_file lfs_object.file.path, filename: @blob.name, disposition: 'attachment' + send_upload(lfs_object.file, attachment: @blob.name) else render_404 end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 6012dbba1b9..b65daa376d2 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -45,6 +45,7 @@ module Ci end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } + scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::LOCAL_STORE]) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } @@ -360,13 +361,19 @@ module Ci project.running_or_pending_build_count(force: true) end + def browsable_artifacts? + artifacts_metadata? + end + def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 84fc6863567..1aea897aaca 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -1,5 +1,6 @@ module Ci class JobArtifact < ActiveRecord::Base + include AfterCommitQueue extend Gitlab::Ci::Model belongs_to :project @@ -9,6 +10,12 @@ module Ci mount_uploader :file, JobArtifactUploader + after_save if: :file_changed?, on: [:create, :update] do + run_after_commit do + file.schedule_migration_to_object_storage + end + end + enum file_type: { archive: 1, metadata: 2 diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index fc586fa216e..6ad792aab30 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,11 +1,22 @@ class LfsObject < ActiveRecord::Base + prepend EE::LfsObject + include AfterCommitQueue + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects validates :oid, presence: true, uniqueness: true + scope :with_files_stored_locally, ->() { where(file_store: [nil, LfsObjectUploader::LOCAL_STORE]) } + mount_uploader :file, LfsObjectUploader + after_save if: :file_changed?, on: [:create, :update] do + run_after_commit do + file.schedule_migration_to_object_storage + end + end + def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index a773222bf17..98a82d82cdc 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -69,9 +69,9 @@ module Projects end def extract_archive!(temp_path) - if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz') + if artifacts_filename.ends_with?('.tar.gz') || artifacts_filename.ends_with?('.tgz') extract_tar_archive!(temp_path) - elsif artifacts.ends_with?('.zip') + elsif artifacts_filename.ends_with?('.zip') extract_zip_archive!(temp_path) else raise 'unsupported artifacts format' @@ -79,11 +79,13 @@ module Projects end def extract_tar_archive!(temp_path) - results = Open3.pipeline(%W(gunzip -c #{artifacts}), - %W(dd bs=#{BLOCK_SIZE} count=#{blocks}), - %W(tar -x -C #{temp_path} #{SITE_PATH}), - err: '/dev/null') - raise 'pages failed to extract' unless results.compact.all?(&:success?) + build.artifacts_file.use_file do |artifacts_path| + results = Open3.pipeline(%W(gunzip -c #{artifacts_path}), + %W(dd bs=#{BLOCK_SIZE} count=#{blocks}), + %W(tar -x -C #{temp_path} #{SITE_PATH}), + err: '/dev/null') + raise 'pages failed to extract' unless results.compact.all?(&:success?) + end end def extract_zip_archive!(temp_path) @@ -101,8 +103,10 @@ module Projects # -n never overwrite existing files # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories site_path = File.join(SITE_PATH, '*') - unless system(*%W(unzip -qq -n #{artifacts} #{site_path} -d #{temp_path})) - raise 'pages failed to extract' + build.artifacts_file.use_file do |artifacts_path| + unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path})) + raise 'pages failed to extract' + end end end @@ -133,6 +137,10 @@ module Projects 1 + max_size / BLOCK_SIZE end + def artifacts_filename + build.artifacts_file.filename + end + def max_size max_pages_size = current_application_settings.max_pages_size.megabytes @@ -161,10 +169,6 @@ module Projects build.ref end - def artifacts - build.artifacts_file.path - end - def latest_sha project.commit(build.ref).try(:sha).to_s end diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb index 15dfb5a5763..a0757dbe6b2 100644 --- a/app/uploaders/job_artifact_uploader.rb +++ b/app/uploaders/job_artifact_uploader.rb @@ -1,5 +1,5 @@ -class JobArtifactUploader < GitlabUploader - storage :file +class JobArtifactUploader < ObjectStoreUploader + storage_options Gitlab.config.artifacts def self.local_store_path Gitlab.config.artifacts.path @@ -15,24 +15,8 @@ class JobArtifactUploader < GitlabUploader model.size end - def store_dir - default_local_path - end - - def cache_dir - File.join(self.class.local_store_path, 'tmp/cache') - end - - def work_dir - File.join(self.class.local_store_path, 'tmp/work') - end - private - def default_local_path - File.join(self.class.local_store_path, default_path) - end - def default_path creation_date = model.created_at.utc.strftime('%Y_%m_%d') diff --git a/app/uploaders/legacy_artifact_uploader.rb b/app/uploaders/legacy_artifact_uploader.rb index 4f7f8a63108..476a46c1754 100644 --- a/app/uploaders/legacy_artifact_uploader.rb +++ b/app/uploaders/legacy_artifact_uploader.rb @@ -1,5 +1,5 @@ -class LegacyArtifactUploader < GitlabUploader - storage :file +class LegacyArtifactUploader < ObjectStoreUploader + storage_options Gitlab.config.artifacts def self.local_store_path Gitlab.config.artifacts.path @@ -9,24 +9,8 @@ class LegacyArtifactUploader < GitlabUploader File.join(self.local_store_path, 'tmp/uploads/') end - def store_dir - default_local_path - end - - def cache_dir - File.join(self.class.local_store_path, 'tmp/cache') - end - - def work_dir - File.join(self.class.local_store_path, 'tmp/work') - end - private - def default_local_path - File.join(self.class.local_store_path, default_path) - end - def default_path File.join(model.created_at.utc.strftime('%Y_%m'), model.project_id.to_s, model.id.to_s) end diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb index d11ebf0f9ca..fa42e4710b7 100644 --- a/app/uploaders/lfs_object_uploader.rb +++ b/app/uploaders/lfs_object_uploader.rb @@ -1,19 +1,17 @@ -class LfsObjectUploader < GitlabUploader - storage :file +class LfsObjectUploader < ObjectStoreUploader + storage_options Gitlab.config.lfs - def store_dir - "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}" - end - - def cache_dir - "#{Gitlab.config.lfs.storage_path}/tmp/cache" + def self.local_store_path + Gitlab.config.lfs.storage_path end def filename model.oid[4..-1] end - def work_dir - File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work') + private + + def default_path + "#{model.oid[0, 2]}/#{model.oid[2, 2]}" end end diff --git a/app/uploaders/object_store_uploader.rb b/app/uploaders/object_store_uploader.rb new file mode 100644 index 00000000000..bb25dc4219f --- /dev/null +++ b/app/uploaders/object_store_uploader.rb @@ -0,0 +1,215 @@ +require 'fog/aws' +require 'carrierwave/storage/fog' + +class ObjectStoreUploader < GitlabUploader + before :store, :set_default_local_store + before :store, :verify_license! + + LOCAL_STORE = 1 + REMOTE_STORE = 2 + + class << self + def storage_options(options) + @storage_options = options + end + + def object_store_options + @storage_options&.object_store + end + + def object_store_enabled? + object_store_options&.enabled + end + + def background_upload_enabled? + object_store_options&.background_upload + end + + def object_store_credentials + @object_store_credentials ||= object_store_options&.connection&.to_hash&.deep_symbolize_keys + end + + def object_store_directory + object_store_options&.remote_directory + end + + def local_store_path + raise NotImplementedError + end + end + + def file_storage? + storage.is_a?(CarrierWave::Storage::File) + end + + def file_cache_storage? + cache_storage.is_a?(CarrierWave::Storage::File) + end + + def real_object_store + model.public_send(store_serialization_column) # rubocop:disable GitlabSecurity/PublicSend + end + + def object_store + subject.public_send(:"#{field}_store") + end + + def object_store=(value) + @storage = nil + model.public_send(:"#{store_serialization_column}=", value) # rubocop:disable GitlabSecurity/PublicSend + end + + def store_dir + if file_storage? + default_local_path + else + default_path + end + end + + def use_file + if file_storage? + return yield path + end + + begin + cache_stored_file! + yield cache_path + ensure + cache_storage.delete_dir!(cache_path(nil)) + end + end + + def filename + super || file&.filename + end + + def migrate!(new_store) + raise 'Undefined new store' unless new_store + + return unless object_store != new_store + return unless file + + old_file = file + old_store = object_store + + # for moving remote file we need to first store it locally + cache_stored_file! unless file_storage? + + # change storage + self.object_store = new_store + + with_callbacks(:store, file) do + storage.store!(file).tap do |new_file| + # since we change storage store the new storage + # in case of failure delete new file + begin + model.save! + rescue => e + new_file.delete + self.object_store = old_store + raise e + end + + old_file.delete + end + end + end + + def schedule_migration_to_object_storage(*args) + return unless self.class.object_store_enabled? + return unless self.class.background_upload_enabled? + return unless self.licensed? + return unless self.file_storage? + + ObjectStorageUploadWorker.perform_async(self.class.name, model.class.name, mounted_as, model.id) + end + + def fog_directory + self.class.object_store_options.remote_directory + end + + def fog_credentials + self.class.object_store_options.connection + end + + def fog_public + false + end + + def move_to_store + file.try(:storage) == storage + end + + def move_to_cache + file.try(:storage) == cache_storage + end + + # We block storing artifacts on Object Storage, not receiving + def verify_license!(new_file) + return if file_storage? + + raise 'Object Storage feature is missing' unless licensed? + end + + def exists? + file.try(:exists?) + end + + def cache_dir + File.join(self.class.local_store_path, 'tmp/cache') + end + + # Override this if you don't want to save local files by default to the Rails.root directory + def work_dir + # Default path set by CarrierWave: + # https://github.com/carrierwaveuploader/carrierwave/blob/v1.1.0/lib/carrierwave/uploader/cache.rb#L182 + # CarrierWave.tmp_path + File.join(self.class.local_store_path, 'tmp/work') + end + + def licensed? + License.feature_available?(:object_storage) + end + + private + + def set_default_local_store(new_file) + self.object_store = LOCAL_STORE unless self.object_store + end + + def default_local_path + File.join(self.class.local_store_path, default_path) + end + + def default_path + raise NotImplementedError + end + + def serialization_column + model.class.uploader_option(mounted_as, :mount_on) || mounted_as + end + + def store_serialization_column + :"#{serialization_column}_store" + end + + def storage + @storage ||= + if object_store == REMOTE_STORE + remote_storage + else + local_storage + end + end + + def remote_storage + raise 'Object Storage is not enabled' unless self.class.object_store_enabled? + + CarrierWave::Storage::Fog.new(self) + end + + def local_storage + CarrierWave::Storage::File.new(self) + end +end diff --git a/app/views/projects/jobs/_sidebar.html.haml b/app/views/projects/jobs/_sidebar.html.haml index a71333497e6..5e7d60da353 100644 --- a/app/views/projects/jobs/_sidebar.html.haml +++ b/app/views/projects/jobs/_sidebar.html.haml @@ -35,9 +35,9 @@ = link_to download_project_job_artifacts_path(@project, @build), rel: 'nofollow', download: '', class: 'btn btn-sm btn-default' do Download - - if @build.artifacts_metadata? - = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do - Browse + - if @build.browsable_artifacts? + = link_to browse_project_job_artifacts_path(@project, @build), class: 'btn btn-sm btn-default' do + Browse - if @build.trigger_request .build-widget.block diff --git a/app/workers/object_storage_upload_worker.rb b/app/workers/object_storage_upload_worker.rb new file mode 100644 index 00000000000..0b9411ff2df --- /dev/null +++ b/app/workers/object_storage_upload_worker.rb @@ -0,0 +1,23 @@ +class ObjectStorageUploadWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + sidekiq_options retry: 5 + + def perform(uploader_class_name, subject_class_name, file_field, subject_id) + uploader_class = uploader_class_name.constantize + subject_class = subject_class_name.constantize + + return unless uploader_class.object_store_enabled? + return unless uploader_class.background_upload_enabled? + + subject = subject_class.find_by(id: subject_id) + return unless subject + + file = subject.public_send(file_field) # rubocop:disable GitlabSecurity/PublicSend + + return unless file.licensed? + + file.migrate!(uploader_class::REMOTE_STORE) + end +end |