From d3734fbd89c069d35856b440f12109af8a7ef9c9 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 14:43:19 +0200 Subject: Use tar for intermediate backup storage During the backup we create an intermediate copy of two directories: builds and uploads. Instead of creating many small files with 'cp -r', we now use tar (and fast gzip) to create single intermediate files. This saves on disk IO and disk space while creating a backup. --- lib/backup/files.rb | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 lib/backup/files.rb (limited to 'lib/backup/files.rb') diff --git a/lib/backup/files.rb b/lib/backup/files.rb new file mode 100644 index 00000000000..d0a6e8f27be --- /dev/null +++ b/lib/backup/files.rb @@ -0,0 +1,39 @@ +require 'open3' + +module Backup + class Files + attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir + + def initialize(app_files_dir) + @app_files_dir = File.realpath(app_files_dir) + @name = File.basename(app_files_dir) + @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_dir = Gitlab.config.backup.path + @backup_tarball = File.join(@backup_dir, name + '.tar.gz') + end + + # Copy files from public/files to backup/files + def dump + FileUtils.mkdir_p(Gitlab.config.backup.path) + run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end + + def restore + backup_existing_files_dir + + run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball) + end + + def backup_existing_files_dir + timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + if File.exists?(app_files_dir) + FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + end + end + + def run_pipeline!(cmd_list, options={}) + status_list = Open3.pipeline(*cmd_list, options) + abort 'Backup failed' unless status_list.compact.all?(&:success?) + end + end +end -- cgit v1.2.1 From 7b71727c562b6f6337a180ae136be94bf0f6ed31 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:10:13 +0200 Subject: Remove old "files" tarball explicitly --- lib/backup/files.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/backup/files.rb') diff --git a/lib/backup/files.rb b/lib/backup/files.rb index d0a6e8f27be..1b08e3324d7 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -15,6 +15,7 @@ module Backup # Copy files from public/files to backup/files def dump FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.rm_f(backup_tarball) run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end -- cgit v1.2.1 From 7d58489fd908b2263f02e8919b1bd0b3fae1201d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:22:08 +0200 Subject: Remove unused variable --- lib/backup/files.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib/backup/files.rb') diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 1b08e3324d7..5a210a0e464 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -2,14 +2,13 @@ require 'open3' module Backup class Files - attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir def initialize(app_files_dir) @app_files_dir = File.realpath(app_files_dir) @name = File.basename(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) - @backup_dir = Gitlab.config.backup.path - @backup_tarball = File.join(@backup_dir, name + '.tar.gz') + @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end # Copy files from public/files to backup/files -- cgit v1.2.1 From 58260a0327a953499a07e9cad8d9aaad2d25699b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 16 Oct 2015 17:16:17 +0200 Subject: Do no rely on basename of builds, uploads --- lib/backup/files.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib/backup/files.rb') diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 5a210a0e464..654b4d1c896 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -4,9 +4,9 @@ module Backup class Files attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir - def initialize(app_files_dir) + def initialize(name, app_files_dir) + @name = name @app_files_dir = File.realpath(app_files_dir) - @name = File.basename(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end @@ -15,13 +15,14 @@ module Backup def dump FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) - run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end def restore backup_existing_files_dir + create_files_dir - run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball) + run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir -- cgit v1.2.1