diff options
author | James Lopez <james@jameslopez.es> | 2018-08-29 15:41:56 +0200 |
---|---|---|
committer | James Lopez <james@jameslopez.es> | 2018-09-06 16:52:42 +0200 |
commit | a2aa505805478540b3150b2f9093c8658e18597d (patch) | |
tree | 7f080afe99e546f743e14c8efa5c6a30c2527554 /lib | |
parent | a2ea32dd44cc4a104e404325c73a77151913a946 (diff) | |
download | gitlab-ce-a2aa505805478540b3150b2f9093c8658e18597d.tar.gz |
Refactor code to remove object storage flag from Import/Export
Updated docs, refactor import/export code
Fix AvatarUploader path issue
Fix project export upload webhook error
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/project_export.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/import_export/avatar_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/avatar_saver.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_restorer.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_saver.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/template_helper.rb | 17 | ||||
-rw-r--r-- | lib/tasks/gitlab/update_templates.rake | 4 |
13 files changed, 21 insertions, 96 deletions
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 15c57a2fc02..8954a0c0a4c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -21,11 +21,7 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - path = user_project.export_project_path - - if path - present_disk_file!(path, File.basename(path), 'application/gzip') - elsif user_project.export_project_object_exists? + if user_project.export_project_object_exists? present_carrierwave_file!(user_project.import_export_upload.export_file) else render_api_error!('404 Not found or has expired', 404) diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index be3710c5b7f..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,10 +40,6 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end - def object_storage? - Feature.enabled?(:import_export_object_storage) - end - def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index dce8f89c0ab..064092681f8 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -40,15 +40,11 @@ module Gitlab def send_file Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file && !object_storage? + export_file.close if export_file end def export_file - if object_storage? - project.import_export_upload.export_file.file.open - else - File.open(project.export_project_path) - end + project.import_export_upload.export_file.open end def send_file_options @@ -63,11 +59,7 @@ module Gitlab end def export_size - if object_storage? - project.import_export_upload.export_file.file.size - else - File.size(project.export_project_path) - end + project.import_export_upload.export_file.file.size end end end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..ded05f73cf8 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -19,7 +19,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 31ef0490cb3..6ffebf83dd2 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -14,19 +12,12 @@ module Gitlab Gitlab::ImportExport::UploadsManager.new( project: @project, shared: @shared, - relative_export_path: 'avatar', - from: avatar_path + relative_export_path: 'avatar' ).save rescue => e @shared.error(e) false end - - private - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4e179f63d8c..72d5b9b830c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -92,8 +92,6 @@ module Gitlab end def remove_import_file - return unless Gitlab::ImportExport.object_storage? - upload = @project.import_export_upload return unless upload&.import_file&.file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f4106e03a57..00ea4b833e2 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -199,7 +199,7 @@ module Gitlab end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 3cd153a4fd2..a66f32f91d2 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") - save_on_object_storage if use_object_storage? + save_on_object_storage else @shared.error(Gitlab::ImportExport::Error.new(error_message)) false @@ -27,10 +27,8 @@ module Gitlab @shared.error(e) false ensure - if use_object_storage? - remove_archive - remove_export_path - end + remove_archive + remove_export_path end private @@ -59,12 +57,8 @@ module Gitlab upload.save! end - def use_object_storage? - Gitlab::ImportExport.object_storage? - end - def error_message - "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + "Unable to save #{archive_file} into #{@shared.export_path}." end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e0d4235e65b..8511319cb1c 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -5,18 +5,13 @@ module Gitlab UPLOADS_BATCH_SIZE = 100 - def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + def initialize(project:, shared:, relative_export_path: 'uploads') @project = project @shared = shared @relative_export_path = relative_export_path - @from = from || default_uploads_path end def save - if File.file?(@from) && @relative_export_path == 'avatar' - copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) - end - copy_project_uploads true @@ -55,17 +50,11 @@ module Gitlab copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) else - next unless Gitlab::ImportExport.object_storage? - download_and_copy(uploader) end end end - def default_uploads_path - FileUploader.absolute_base_dir(@project) - end - def uploads_export_path @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 25f85936227..b4313ff4cb4 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,30 +2,14 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - if Gitlab::ImportExport.object_storage? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).restore - elsif File.directory?(uploads_export_path) - copy_files(uploads_export_path, uploads_path) - - true - else - true # Proceed without uploads - end + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - @uploads_export_path ||= File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index b3f17af5661..0275f686c5e 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index f24a01e6cf5..fc498dde723 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -1,24 +1,9 @@ module Gitlab module TemplateHelper - include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file) return unless file - if Gitlab::ImportExport.object_storage? - params[:import_export_upload] = ImportExportUpload.new(import_file: file) - else - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file.path, import_upload_path) - - params[:import_source] = import_upload_path - end - end - - def import_upload_path - strong_memoize(:import_upload_path) do - Gitlab::ImportExport.import_upload_path(filename: tmp_filename) - end + params[:import_export_upload] = ImportExportUpload.new(import_file: file) end def tmp_filename diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a25f7ce59c7..3c873ba3a20 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -6,6 +6,8 @@ namespace :gitlab do desc "GitLab | Update project templates" task :update_project_templates do + include Gitlab::ImportExport::CommandLineUtil + if Rails.env.production? puts "This rake task is not meant fo production instances".red exit(1) @@ -52,7 +54,7 @@ namespace :gitlab do end Projects::ImportExport::ExportService.new(project, admin).execute - FileUtils.cp(project.export_project_path, template.archive_path) + download_or_copy_upload(project.import_export_upload.export_file, template.archive_path) Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end |