Merge branch 'master' into sync-all-repos

8 files changed, 115 insertions, 97 deletions

diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
new file mode 100644
index 00000000000..51fa3867e67
--- /dev/null
+++ b/lib/backup/artifacts.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Artifacts < Files
+    def initialize
+      super('artifacts', ArtifactUploader.artifacts_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index 6f56f680bb9..635967f4bd4 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,34 +1,13 @@
-module Backup
-  class Builds
-    attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
+require 'backup/files'
 
+module Backup
+  class Builds < Files
     def initialize
-      @app_builds_dir = Settings.gitlab_ci.builds_path
-      @backup_dir = Gitlab.config.backup.path
-      @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds')
-    end
-
-    # Copy builds from builds directory to backup/builds
-    def dump
-      FileUtils.rm_rf(backup_builds_dir)
-      # Ensure the parent dir of backup_builds_dir exists
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_builds_dir before us
-      FileUtils.mkdir(backup_builds_dir, mode: 0700)
-      FileUtils.cp_r(app_builds_dir, backup_dir)
-    end
-
-    def restore
-      backup_existing_builds_dir
-
-      FileUtils.cp_r(backup_builds_dir, app_builds_dir)
+      super('builds', Settings.gitlab_ci.builds_path)
     end
 
-    def backup_existing_builds_dir
-      timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
-      if File.exists?(app_builds_dir)
-        FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
-      end
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
     end
   end
 end
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 959ac4b7868..67b2a64bd10 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -2,26 +2,26 @@ require 'yaml'
 
 module Backup
   class Database
-    attr_reader :config, :db_dir
+    attr_reader :config, :db_file_name
 
     def initialize
       @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
-      @db_dir = File.join(Gitlab.config.backup.path, 'db')
+      @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
     end
 
     def dump
-      FileUtils.rm_rf(@db_dir)
-      # Ensure the parent dir of @db_dir exists
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create @db_dir before us
-      FileUtils.mkdir(@db_dir, mode: 0700)
+      FileUtils.mkdir_p(File.dirname(db_file_name))
+      FileUtils.rm_f(db_file_name)
+      compress_rd, compress_wr = IO.pipe
+      compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600])
+      compress_rd.close
 
-      success = case config["adapter"]
+      dump_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Dumping MySQL database #{config['database']} ... "
         # Workaround warnings from MySQL 5.6 about passwords on cmd line
         ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-        system('mysqldump', *mysql_args, config['database'], out: db_file_name)
+        spawn('mysqldump', *mysql_args, config['database'], out: compress_wr)
       when "postgresql" then
         $progress.print "Dumping PostgreSQL database #{config['database']} ... "
         pg_env
@@ -30,48 +30,42 @@ module Backup
           pgsql_args << "-n"
           pgsql_args << Gitlab.config.backup.pg_schema
         end
-        system('pg_dump', *pgsql_args, config['database'], out: db_file_name)
+        spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr)
       end
-      report_success(success)
-      abort 'Backup failed' unless success
+      compress_wr.close
+
+      success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? }
 
-      $progress.print 'Compressing database ... '
-      success = system('gzip', db_file_name)
       report_success(success)
-      abort 'Backup failed: compress error' unless success
+      abort 'Backup failed' unless success
     end
 
     def restore
-      $progress.print 'Decompressing database ... '
-      success = system('gzip', '-d', db_file_name_gz)
-      report_success(success)
-      abort 'Restore failed: decompress error' unless success
+      decompress_rd, decompress_wr = IO.pipe
+      decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+      decompress_wr.close
 
-      success = case config["adapter"]
+      restore_pid = case config["adapter"]
       when /^mysql/ then
         $progress.print "Restoring MySQL database #{config['database']} ... "
         # Workaround warnings from MySQL 5.6 about passwords on cmd line
         ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-        system('mysql', *mysql_args, config['database'], in: db_file_name)
+        spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
       when "postgresql" then
         $progress.print "Restoring PostgreSQL database #{config['database']} ... "
         pg_env
-        system('psql', config['database'], '-f', db_file_name)
+        spawn('psql', config['database'], in: decompress_rd)
       end
+      decompress_rd.close
+
+      success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+
       report_success(success)
       abort 'Restore failed' unless success
     end
 
     protected
 
-    def db_file_name
-      File.join(db_dir, 'database.sql')
-    end
-
-    def db_file_name_gz
-      File.join(db_dir, 'database.sql.gz')
-    end
-
     def mysql_args
       args = {
         'host'      => '--host',
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
new file mode 100644
index 00000000000..654b4d1c896
--- /dev/null
+++ b/lib/backup/files.rb
@@ -0,0 +1,40 @@
+require 'open3'
+
+module Backup
+  class Files
+    attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir
+
+    def initialize(name, app_files_dir)
+      @name = name
+      @app_files_dir = File.realpath(app_files_dir)
+      @files_parent_dir = File.realpath(File.join(@app_files_dir, '..'))
+      @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
+    end
+
+    # Copy files from public/files to backup/files
+    def dump
+      FileUtils.mkdir_p(Gitlab.config.backup.path)
+      FileUtils.rm_f(backup_tarball)
+      run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600])
+    end
+
+    def restore
+      backup_existing_files_dir
+      create_files_dir
+
+      run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball)
+    end
+
+    def backup_existing_files_dir
+      timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}")
+      if File.exists?(app_files_dir)
+        FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path))
+      end
+    end
+
+    def run_pipeline!(cmd_list, options={})
+      status_list = Open3.pipeline(*cmd_list, options)
+      abort 'Backup failed' unless status_list.compact.all?(&:success?)
+    end
+  end
+end
diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb
new file mode 100644
index 00000000000..4153467fbee
--- /dev/null
+++ b/lib/backup/lfs.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Lfs < Files
+    def initialize
+      super('lfs', Settings.lfs.storage_path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index ac63f89c6ec..099062eeb8b 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -45,7 +45,8 @@ module Backup
       directory = connection.directories.get(remote_directory)
 
       if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
-          multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size)
+          multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size,
+          encryption: Gitlab.config.backup.upload.encryption)
         $progress.puts "done".green
       else
         puts "uploading backup to #{remote_directory} failed".red
@@ -55,7 +56,7 @@ module Backup
 
     def cleanup
       $progress.print "Deleting tmp directories ... "
-      
+
       backup_contents.each do |dir|
         next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
 
@@ -75,7 +76,7 @@ module Backup
 
       if keep_time > 0
         removed = 0
-        
+
         Dir.chdir(Gitlab.config.backup.path) do
           file_list = Dir.glob('*_gitlab_backup.tar')
           file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ }
@@ -149,17 +150,15 @@ module Backup
     private
 
     def backup_contents
-      folders_to_backup + ["backup_information.yml"]
+      folders_to_backup + archives_to_backup + ["backup_information.yml"]
     end
 
-    def folders_to_backup
-      folders = %w{repositories db uploads builds}
-
-      if ENV["SKIP"]
-        return folders.reject{ |folder| ENV["SKIP"].include?(folder) }
-      end
+    def archives_to_backup
+      %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
+    end
 
-      folders
+    def folders_to_backup
+      %w{repositories db}.reject{ |name| skipped?(name) }
     end
 
     def settings
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 4d70f7883dd..a82a7e1f7bf 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -35,7 +35,7 @@ module Backup
           if wiki.repository.empty?
             $progress.puts " [SKIPPED]".cyan
           else
-            cmd = %W(git --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
+            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
             output, status = Gitlab::Popen.popen(cmd)
             if status.zero?
               $progress.puts " [DONE]".green
@@ -67,7 +67,7 @@ module Backup
           FileUtils.mkdir_p(path_to_repo(project))
           cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)})
         else
-          cmd = %W(git init --bare #{path_to_repo(project)})
+          cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)})
         end
 
         if system(*cmd, silent)
@@ -87,7 +87,7 @@ module Backup
           # that was initialized with ProjectWiki.new() and then
           # try to restore with 'git clone --bare'.
           FileUtils.rm_rf(path_to_repo(wiki))
-          cmd = %W(git clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
+          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
 
           if system(*cmd, silent)
             $progress.puts " [DONE]".green
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index 1f9626644e6..9261f77f3c9 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -1,34 +1,14 @@
+require 'backup/files'
+
 module Backup
-  class Uploads
-    attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir
+  class Uploads < Files
 
     def initialize
-      @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads'))
-      @backup_dir = Gitlab.config.backup.path
-      @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads')
-    end
-
-    # Copy uploads from public/uploads to backup/uploads
-    def dump
-      FileUtils.rm_rf(backup_uploads_dir)
-      # Ensure the parent dir of backup_uploads_dir exists
-      FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_uploads_dir before us
-      FileUtils.mkdir(backup_uploads_dir, mode: 0700)
-      FileUtils.cp_r(app_uploads_dir, backup_dir)
-    end
-
-    def restore
-      backup_existing_uploads_dir
-
-      FileUtils.cp_r(backup_uploads_dir, app_uploads_dir)
+      super('uploads', Rails.root.join('public/uploads'))
     end
 
-    def backup_existing_uploads_dir
-      timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}")
-      if File.exists?(app_uploads_dir)
-        FileUtils.mv(app_uploads_dir, File.expand_path(timestamped_uploads_path))
-      end
+    def create_files_dir
+      Dir.mkdir(app_files_dir)
     end
   end
 end