diff options
| -rw-r--r-- | changelogs/unreleased/sh-fix-backup-specific-rake-task.yml | 5 | ||||
| -rw-r--r-- | lib/backup/artifacts.rb | 6 | ||||
| -rw-r--r-- | lib/backup/builds.rb | 6 | ||||
| -rw-r--r-- | lib/backup/database.rb | 16 | ||||
| -rw-r--r-- | lib/backup/lfs.rb | 6 | ||||
| -rw-r--r-- | lib/backup/manager.rb | 55 | ||||
| -rw-r--r-- | lib/backup/pages.rb | 6 | ||||
| -rw-r--r-- | lib/backup/registry.rb | 6 | ||||
| -rw-r--r-- | lib/backup/repository.rb | 10 | ||||
| -rw-r--r-- | lib/backup/uploads.rb | 6 | ||||
| -rw-r--r-- | lib/tasks/gitlab/backup.rake | 132 | ||||
| -rw-r--r-- | spec/lib/backup/manager_spec.rb | 2 | ||||
| -rw-r--r-- | spec/lib/backup/repository_spec.rb | 11 | ||||
| -rw-r--r-- | spec/tasks/gitlab/backup_rake_spec.rb | 10 |
14 files changed, 163 insertions, 114 deletions
diff --git a/changelogs/unreleased/sh-fix-backup-specific-rake-task.yml b/changelogs/unreleased/sh-fix-backup-specific-rake-task.yml new file mode 100644 index 00000000000..71b121710ee --- /dev/null +++ b/changelogs/unreleased/sh-fix-backup-specific-rake-task.yml @@ -0,0 +1,5 @@ +--- +title: Fix backup creation and restore for specific Rake tasks +merge_request: +author: +type: fixed diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 6a5a223a614..45a935ab352 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -2,7 +2,11 @@ require 'backup/files' module Backup class Artifacts < Files - def initialize + attr_reader :progress + + def initialize(progress) + @progress = progress + super('artifacts', JobArtifactUploader.root) end end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index f869916e199..adf85ca4719 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -2,7 +2,11 @@ require 'backup/files' module Backup class Builds < Files - def initialize + attr_reader :progress + + def initialize(progress) + @progress = progress + super('builds', Settings.gitlab_ci.builds_path) end end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 5e6828de597..1608f7ad02d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -2,9 +2,11 @@ require 'yaml' module Backup class Database + attr_reader :progress attr_reader :config, :db_file_name - def initialize + def initialize(progress) + @progress = progress @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end @@ -19,12 +21,12 @@ module Backup dump_pid = case config["adapter"] when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " + progress.print "Dumping MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " + progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. if Gitlab.config.backup.pg_schema @@ -53,12 +55,12 @@ module Backup restore_pid = case config["adapter"] when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " + progress.print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " + progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env spawn('psql', config['database'], in: decompress_rd) end @@ -111,9 +113,9 @@ module Backup def report_success(success) if success - $progress.puts '[DONE]'.color(:green) + progress.puts '[DONE]'.color(:green) else - $progress.puts '[FAILED]'.color(:red) + progress.puts '[FAILED]'.color(:red) end end end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 4e234e50a7a..185ff8ae6bd 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -2,7 +2,11 @@ require 'backup/files' module Backup class Lfs < Files - def initialize + attr_reader :progress + + def initialize(progress) + @progress = progress + super('lfs', Settings.lfs.storage_path) end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f27ce4d2b2b..a8da0c7edef 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -4,6 +4,12 @@ module Backup FOLDERS_TO_BACKUP = %w[repositories db].freeze FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze + attr_reader :progress + + def initialize(progress) + @progress = progress + end + def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -14,11 +20,11 @@ module Backup end # create archive - $progress.print "Creating backup archive: #{tar_file} ... " + progress.print "Creating backup archive: #{tar_file} ... " # Set file permissions on open to prevent chmod races. tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - $progress.puts "done".color(:green) + progress.puts "done".color(:green) else puts "creating archive #{tar_file} failed".color(:red) abort 'Backup failed' @@ -29,11 +35,11 @@ module Backup end def upload - $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " + progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - $progress.puts "skipped".color(:yellow) + progress.puts "skipped".color(:yellow) return end @@ -43,7 +49,7 @@ module Backup multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, storage_class: Gitlab.config.backup.upload.storage_class) - $progress.puts "done".color(:green) + progress.puts "done".color(:green) else puts "uploading backup to #{remote_directory} failed".color(:red) abort 'Backup failed' @@ -51,13 +57,13 @@ module Backup end def cleanup - $progress.print "Deleting tmp directories ... " + progress.print "Deleting tmp directories ... " backup_contents.each do |dir| next unless File.exist?(File.join(backup_path, dir)) if FileUtils.rm_rf(File.join(backup_path, dir)) - $progress.puts "done".color(:green) + progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) abort 'Backup failed' @@ -67,7 +73,7 @@ module Backup def remove_old # delete backups - $progress.print "Deleting old backups ... " + progress.print "Deleting old backups ... " keep_time = Gitlab.config.backup.keep_time.to_i if keep_time > 0 @@ -88,31 +94,32 @@ module Backup FileUtils.rm(file) removed += 1 rescue => e - $progress.puts "Deleting #{file} failed: #{e.message}".color(:red) + progress.puts "Deleting #{file} failed: #{e.message}".color(:red) end end end end - $progress.puts "done. (#{removed} removed)".color(:green) + progress.puts "done. (#{removed} removed)".color(:green) else - $progress.puts "skipping".color(:yellow) + progress.puts "skipping".color(:yellow) end end + # rubocop: disable Metrics/AbcSize def unpack Dir.chdir(backup_path) do # check for existing backups in the backup dir if backup_file_list.empty? - $progress.puts "No backups found in #{backup_path}" - $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + progress.puts "No backups found in #{backup_path}" + progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 elsif backup_file_list.many? && ENV["BACKUP"].nil? - $progress.puts 'Found more than one backup:' + progress.puts 'Found more than one backup:' # print list of available backups - $progress.puts " " + available_timestamps.join("\n ") - $progress.puts 'Please specify which one you want to restore:' - $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + progress.puts " " + available_timestamps.join("\n ") + progress.puts 'Please specify which one you want to restore:' + progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 end @@ -123,31 +130,31 @@ module Backup end unless File.exist?(tar_file) - $progress.puts "The backup file #{tar_file} does not exist!" + progress.puts "The backup file #{tar_file} does not exist!" exit 1 end - $progress.print 'Unpacking backup ... ' + progress.print 'Unpacking backup ... ' unless Kernel.system(*%W(tar -xf #{tar_file})) - $progress.puts 'unpacking backup failed'.color(:red) + progress.puts 'unpacking backup failed'.color(:red) exit 1 else - $progress.puts 'done'.color(:green) + progress.puts 'done'.color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - $progress.puts(<<~HEREDOC.color(:red)) + progress.puts(<<~HEREDOC.color(:red)) GitLab version mismatch: Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup! Please switch to the following version and try again: version: #{settings[:gitlab_version]} HEREDOC - $progress.puts - $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" + progress.puts + progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 end end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 5830b209d6e..542e35a7c7c 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -2,7 +2,11 @@ require 'backup/files' module Backup class Pages < Files - def initialize + attr_reader :progress + + def initialize(progress) + @progress = progress + super('pages', Gitlab.config.pages.path) end end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 91698669402..35821805797 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -2,7 +2,11 @@ require 'backup/files' module Backup class Registry < Files - def initialize + attr_reader :progress + + def initialize(progress) + @progress = progress + super('registry', Settings.registry.path) end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 65e06fd78c0..44ad991c72a 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -6,6 +6,12 @@ module Backup include Backup::Helper # rubocop:disable Metrics/AbcSize + attr_reader :progress + + def initialize(progress) + @progress = progress + end + def dump prepare @@ -217,10 +223,6 @@ module Backup Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } end - def progress - $progress - end - def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index d46e2cd869d..49b117a7ee3 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,7 +2,11 @@ require 'backup/files' module Backup class Uploads < Files - def initialize + attr_reader :progress + + def initialize(progress) + @progress = progress + super('uploads', Rails.root.join('public/uploads')) end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 24e37f6c6cc..e96fbb64372 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -6,7 +6,6 @@ namespace :gitlab do desc "GitLab | Create a backup of the GitLab system" task create: :gitlab_environment do warn_user_is_not_gitlab - configure_cron_mode Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke @@ -17,7 +16,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:lfs:create"].invoke Rake::Task["gitlab:backup:registry:create"].invoke - backup = Backup::Manager.new + backup = Backup::Manager.new(progress) backup.pack backup.cleanup backup.remove_old @@ -27,9 +26,8 @@ namespace :gitlab do desc 'GitLab | Restore a previously created backup' task restore: :gitlab_environment do warn_user_is_not_gitlab - configure_cron_mode - backup = Backup::Manager.new + backup = Backup::Manager.new(progress) backup.unpack unless backup.skipped?('db') @@ -49,9 +47,9 @@ namespace :gitlab do # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade - $progress.puts 'Cleaning the database ... '.color(:blue) + progress.puts 'Cleaning the database ... '.color(:blue) Rake::Task['gitlab:db:drop_tables'].invoke - $progress.puts 'done'.color(:green) + progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke rescue Gitlab::TaskAbortedByUserError puts "Quitting...".color(:red) @@ -74,173 +72,173 @@ namespace :gitlab do namespace :repo do task create: :gitlab_environment do - $progress.puts "Dumping repositories ...".color(:blue) + progress.puts "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Repository.new.dump - $progress.puts "done".color(:green) + Backup::Repository.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring repositories ...".color(:blue) - Backup::Repository.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring repositories ...".color(:blue) + Backup::Repository.new(progress).restore + progress.puts "done".color(:green) end end namespace :db do task create: :gitlab_environment do - $progress.puts "Dumping database ... ".color(:blue) + progress.puts "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Database.new.dump - $progress.puts "done".color(:green) + Backup::Database.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring database ... ".color(:blue) - Backup::Database.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring database ... ".color(:blue) + Backup::Database.new(progress).restore + progress.puts "done".color(:green) end end namespace :builds do task create: :gitlab_environment do - $progress.puts "Dumping builds ... ".color(:blue) + progress.puts "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Builds.new.dump - $progress.puts "done".color(:green) + Backup::Builds.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring builds ... ".color(:blue) - Backup::Builds.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring builds ... ".color(:blue) + Backup::Builds.new(progress).restore + progress.puts "done".color(:green) end end namespace :uploads do task create: :gitlab_environment do - $progress.puts "Dumping uploads ... ".color(:blue) + progress.puts "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Uploads.new.dump - $progress.puts "done".color(:green) + Backup::Uploads.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring uploads ... ".color(:blue) - Backup::Uploads.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring uploads ... ".color(:blue) + Backup::Uploads.new(progress).restore + progress.puts "done".color(:green) end end namespace :artifacts do task create: :gitlab_environment do - $progress.puts "Dumping artifacts ... ".color(:blue) + progress.puts "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Artifacts.new.dump - $progress.puts "done".color(:green) + Backup::Artifacts.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring artifacts ... ".color(:blue) - Backup::Artifacts.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring artifacts ... ".color(:blue) + Backup::Artifacts.new(progress).restore + progress.puts "done".color(:green) end end namespace :pages do task create: :gitlab_environment do - $progress.puts "Dumping pages ... ".color(:blue) + progress.puts "Dumping pages ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("pages") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Pages.new.dump - $progress.puts "done".color(:green) + Backup::Pages.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring pages ... ".color(:blue) - Backup::Pages.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring pages ... ".color(:blue) + Backup::Pages.new(progress).restore + progress.puts "done".color(:green) end end namespace :lfs do task create: :gitlab_environment do - $progress.puts "Dumping lfs objects ... ".color(:blue) + progress.puts "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Lfs.new.dump - $progress.puts "done".color(:green) + Backup::Lfs.new(progress).dump + progress.puts "done".color(:green) end end task restore: :gitlab_environment do - $progress.puts "Restoring lfs objects ... ".color(:blue) - Backup::Lfs.new.restore - $progress.puts "done".color(:green) + progress.puts "Restoring lfs objects ... ".color(:blue) + Backup::Lfs.new(progress).restore + progress.puts "done".color(:green) end end namespace :registry do task create: :gitlab_environment do - $progress.puts "Dumping container registry images ... ".color(:blue) + progress.puts "Dumping container registry images ... ".color(:blue) if Gitlab.config.registry.enabled if ENV["SKIP"] && ENV["SKIP"].include?("registry") - $progress.puts "[SKIPPED]".color(:cyan) + progress.puts "[SKIPPED]".color(:cyan) else - Backup::Registry.new.dump - $progress.puts "done".color(:green) + Backup::Registry.new(progress).dump + progress.puts "done".color(:green) end else - $progress.puts "[DISABLED]".color(:cyan) + progress.puts "[DISABLED]".color(:cyan) end end task restore: :gitlab_environment do - $progress.puts "Restoring container registry images ... ".color(:blue) + progress.puts "Restoring container registry images ... ".color(:blue) if Gitlab.config.registry.enabled - Backup::Registry.new.restore - $progress.puts "done".color(:green) + Backup::Registry.new(progress).restore + progress.puts "done".color(:green) else - $progress.puts "[DISABLED]".color(:cyan) + progress.puts "[DISABLED]".color(:cyan) end end end - def configure_cron_mode + def progress if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a # StringIO. require 'stringio' - $progress = StringIO.new + StringIO.new else - $progress = $stdout + $stdout end end end # namespace end: backup diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb index 84688845fa5..23c04a1a101 100644 --- a/spec/lib/backup/manager_spec.rb +++ b/spec/lib/backup/manager_spec.rb @@ -5,6 +5,8 @@ describe Backup::Manager do let(:progress) { StringIO.new } + subject { described_class.new(progress) } + before do allow(progress).to receive(:puts) allow(progress).to receive(:print) diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repository_spec.rb index b1ea9c0b622..20ea981ec47 100644 --- a/spec/lib/backup/repository_spec.rb +++ b/spec/lib/backup/repository_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' describe Backup::Repository do let(:progress) { StringIO.new } let!(:project) { create(:project, :wiki_repo) } + subject { described_class.new(progress) } before do allow(progress).to receive(:puts) @@ -24,14 +25,12 @@ describe Backup::Repository do end it 'does not raise error' do - expect { described_class.new.dump }.not_to raise_error + expect { subject.dump }.not_to raise_error end end end describe '#restore' do - subject { described_class.new } - let(:timestamp) { Time.utc(2017, 3, 22) } let(:temp_dirs) do Gitlab.config.repositories.storages.map do |name, storage| @@ -102,20 +101,20 @@ describe Backup::Repository do it 'invalidates the emptiness cache' do expect(wiki.repository).to receive(:expire_emptiness_caches).once - described_class.new.send(:empty_repo?, wiki) + subject.send(:empty_repo?, wiki) end context 'wiki repo has content' do let!(:wiki_page) { create(:wiki_page, wiki: wiki) } it 'returns true, regardless of bad cache value' do - expect(described_class.new.send(:empty_repo?, wiki)).to be(false) + expect(subject.send(:empty_repo?, wiki)).to be(false) end end context 'wiki repo does not have content' do it 'returns true, regardless of bad cache value' do - expect(described_class.new.send(:empty_repo?, wiki)).to be_truthy + expect(subject.send(:empty_repo?, wiki)).to be_truthy end end end diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index a2e5642a72c..c9252bebb2e 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -125,6 +125,16 @@ describe 'gitlab:app namespace rake task' do expect(Dir.entries(File.join(project.repository.path, 'custom_hooks'))).to include("dummy.txt") end end + + context 'specific backup tasks' do + let(:task_list) { %w(db repo uploads builds artifacts pages lfs registry) } + + it 'prints a progress message to stdout' do + task_list.each do |task| + expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout + end + end + end end context 'tar creation' do |