diff options
Diffstat (limited to 'lib/ci/backup')
| -rw-r--r-- | lib/ci/backup/builds.rb | 32 | ||||
| -rw-r--r-- | lib/ci/backup/database.rb | 94 | ||||
| -rw-r--r-- | lib/ci/backup/manager.rb | 158 |
3 files changed, 284 insertions, 0 deletions
diff --git a/lib/ci/backup/builds.rb b/lib/ci/backup/builds.rb new file mode 100644 index 00000000000..832a5ab8fdc --- /dev/null +++ b/lib/ci/backup/builds.rb @@ -0,0 +1,32 @@ +module Ci + module Backup + class Builds + attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir + + def initialize + @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) + @backup_dir = GitlabCi.config.backup.path + @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') + end + + # Copy builds from builds directory to backup/builds + def dump + FileUtils.mkdir_p(backup_builds_dir) + FileUtils.cp_r(app_builds_dir, backup_dir) + end + + def restore + backup_existing_builds_dir + + FileUtils.cp_r(backup_builds_dir, app_builds_dir) + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end + end +end diff --git a/lib/ci/backup/database.rb b/lib/ci/backup/database.rb new file mode 100644 index 00000000000..f7fa3f1833a --- /dev/null +++ b/lib/ci/backup/database.rb @@ -0,0 +1,94 @@ +require 'yaml' + +module Ci + module Backup + class Database + attr_reader :config, :db_dir + + def initialize + @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] + @db_dir = File.join(GitlabCi.config.backup.path, 'db') + FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir) + end + + def dump + success = case config["adapter"] + when /^mysql/ then + $progress.print "Dumping MySQL database #{config['database']} ... " + system('mysqldump', *mysql_args, config['database'], out: db_file_name) + when "postgresql" then + $progress.print "Dumping PostgreSQL database #{config['database']} ... " + pg_env + system('pg_dump', config['database'], out: db_file_name) + end + report_success(success) + abort 'Backup failed' unless success + end + + def restore + success = case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + system('mysql', *mysql_args, config['database'], in: db_file_name) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE + # statements like MySQL. + drop_all_tables + drop_all_postgres_sequences + pg_env + system('psql', config['database'], '-f', db_file_name) + end + report_success(success) + abort 'Restore failed' unless success + end + + protected + + def db_file_name + File.join(db_dir, 'database.sql') + end + + def mysql_args + args = { + 'host' => '--host', + 'port' => '--port', + 'socket' => '--socket', + 'username' => '--user', + 'encoding' => '--default-character-set', + 'password' => '--password' + } + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact + end + + def pg_env + ENV['PGUSER'] = config["username"] if config["username"] + ENV['PGHOST'] = config["host"] if config["host"] + ENV['PGPORT'] = config["port"].to_s if config["port"] + ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + end + + def report_success(success) + if success + $progress.puts '[DONE]'.green + else + $progress.puts '[FAILED]'.red + end + end + + def drop_all_tables + connection = ActiveRecord::Base.connection + connection.tables.each do |table| + connection.drop_table(table) + end + end + + def drop_all_postgres_sequences + connection = ActiveRecord::Base.connection + connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence| + connection.execute("DROP SEQUENCE #{sequence['relname']}") + end + end + end + end +end diff --git a/lib/ci/backup/manager.rb b/lib/ci/backup/manager.rb new file mode 100644 index 00000000000..2e9d6df7139 --- /dev/null +++ b/lib/ci/backup/manager.rb @@ -0,0 +1,158 @@ +module Ci + module Backup + class Manager + def pack + # saving additional informations + s = {} + s[:db_version] = "#{ActiveRecord::Migrator.current_version}" + s[:backup_created_at] = Time.now + s[:gitlab_version] = GitlabCi::VERSION + s[:tar_version] = tar_version + tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz" + + Dir.chdir(GitlabCi.config.backup.path) do + File.open("#{GitlabCi.config.backup.path}/backup_information.yml", + "w+") do |file| + file << s.to_yaml.gsub(/^---\n/,'') + end + + FileUtils.chmod(0700, ["db", "builds"]) + + # create archive + $progress.print "Creating backup archive: #{tar_file} ... " + orig_umask = File.umask(0077) + if Kernel.system('tar', '-czf', tar_file, *backup_contents) + $progress.puts "done".green + else + puts "creating archive #{tar_file} failed".red + abort 'Backup failed' + end + File.umask(orig_umask) + + upload(tar_file) + end + end + + def upload(tar_file) + remote_directory = GitlabCi.config.backup.upload.remote_directory + $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " + + connection_settings = GitlabCi.config.backup.upload.connection + if connection_settings.blank? + $progress.puts "skipped".yellow + return + end + + connection = ::Fog::Storage.new(connection_settings) + directory = connection.directories.get(remote_directory) + + if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, + multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size) + $progress.puts "done".green + else + puts "uploading backup to #{remote_directory} failed".red + abort 'Backup failed' + end + end + + def cleanup + $progress.print "Deleting tmp directories ... " + + backup_contents.each do |dir| + next unless File.exist?(File.join(GitlabCi.config.backup.path, dir)) + + if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir)) + $progress.puts "done".green + else + puts "deleting tmp directory '#{dir}' failed".red + abort 'Backup failed' + end + end + end + + def remove_old + # delete backups + $progress.print "Deleting old backups ... " + keep_time = GitlabCi.config.backup.keep_time.to_i + + if keep_time > 0 + removed = 0 + + Dir.chdir(GitlabCi.config.backup.path) do + file_list = Dir.glob('*_gitlab_ci_backup.tar.gz') + file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ } + file_list.sort.each do |timestamp| + if Time.at(timestamp) < (Time.now - keep_time) + if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz)) + removed += 1 + end + end + end + end + + $progress.puts "done. (#{removed} removed)".green + else + $progress.puts "skipping".yellow + end + end + + def unpack + Dir.chdir(GitlabCi.config.backup.path) + + # check for existing backups in the backup dir + file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i } + puts "no backups found" if file_list.count == 0 + + if file_list.count > 1 && ENV["BACKUP"].nil? + puts "Found more than one backup, please specify which one you want to restore:" + puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + exit 1 + end + + tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz") + + unless File.exists?(tar_file) + puts "The specified backup doesn't exist!" + exit 1 + end + + $progress.print "Unpacking backup ... " + + unless Kernel.system(*%W(tar -xzf #{tar_file})) + puts "unpacking backup failed".red + exit 1 + else + $progress.puts "done".green + end + + ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings[:gitlab_version] != GitlabCi::VERSION + puts "GitLab CI version mismatch:".red + puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red + puts " Please switch to the following version and try again:".red + puts " version: #{settings[:gitlab_version]}".red + puts + puts "Hint: git checkout v#{settings[:gitlab_version]}" + exit 1 + end + end + + def tar_version + tar_version = `tar --version` + tar_version.force_encoding('locale').split("\n").first + end + + private + + def backup_contents + ["db", "builds", "backup_information.yml"] + end + + def settings + @settings ||= YAML.load_file("backup_information.yml") + end + end + end +end |