From 143cd58c398b693db1b9d02f7267db39a8acb87c Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 16 May 2016 17:17:57 -0500 Subject: Added backup of container registry --- lib/backup/manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/backup/manager.rb') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 4962f5e53ce..7d0608f09da 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -157,7 +157,7 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + %w{uploads builds artifacts lfs registry}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup -- cgit v1.2.1 From 12bd78103153fce845eed1566918083b52e814df Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 23 May 2016 10:17:07 -0500 Subject: Fix backups if registry is disabled --- lib/backup/manager.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'lib/backup/manager.rb') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7d0608f09da..5a727f5be5b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,5 +1,8 @@ module Backup class Manager + ARCHIVES_TO_BACKUP = %w{uploads builds artifacts lfs registry} + FOLDERS_TO_BACKUP = %w{repositories db} + def pack # Make sure there is a connection ActiveRecord::Base.connection.reconnect! @@ -157,11 +160,17 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts lfs registry}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + (ARCHIVES_TO_BACKUP - disabled_features).map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - %w{repositories db}.reject{ |name| skipped?(name) } + (FOLDERS_TO_BACKUP - disabled_features).reject{ |name| skipped?(name) } + end + + def disabled_features + features = [] + features << 'registry' unless Gitlab.config.registry.enabled + features end def settings -- cgit v1.2.1 From 718fe268c48b3b77b2b3d901b0c688a06e82053d Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 24 May 2016 22:52:31 -0500 Subject: Improve after review --- lib/backup/manager.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib/backup/manager.rb') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 5a727f5be5b..3e07096e6cc 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,7 +1,7 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w{uploads builds artifacts lfs registry} - FOLDERS_TO_BACKUP = %w{repositories db} + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] + FOLDERS_TO_BACKUP = %w[repositories db] def pack # Make sure there is a connection @@ -150,7 +150,7 @@ module Backup end def skipped?(item) - settings[:skipped] && settings[:skipped].include?(item) + settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item) end private @@ -160,11 +160,11 @@ module Backup end def archives_to_backup - (ARCHIVES_TO_BACKUP - disabled_features).map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + ARCHIVES_TO_BACKUP.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup - (FOLDERS_TO_BACKUP - disabled_features).reject{ |name| skipped?(name) } + FOLDERS_TO_BACKUP.reject{ |name| skipped?(name) } end def disabled_features -- cgit v1.2.1 From b826c9ff17b701344028832d523522a3cb925a23 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Tue, 31 May 2016 13:29:37 -0400 Subject: Create the specified remote directory during backup This is idempotent, so there's no harm calling it if the directory already exists. Closes #12710 --- lib/backup/manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/backup/manager.rb') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 3e07096e6cc..660ca8c2923 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -48,7 +48,7 @@ module Backup end connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.get(remote_directory) + directory = connection.directories.create(key: remote_directory) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, -- cgit v1.2.1 From 903946c78a0f73e5cbdfce7b93d31c4d1bd045cd Mon Sep 17 00:00:00 2001 From: Connor Shea Date: Wed, 1 Jun 2016 16:37:15 -0600 Subject: Replace colorize gem with rainbow. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Colorize is a gem licensed under the GPLv2, so we can’t use it in GitLab without relicensing GitLab under the terms of the GPL. Rainbow is licensed under the MIT license and does the exact same thing as Colorize, so Rainbow was added in place of Colorize. The syntax is slightly different for Rainbow vs. Colorize, and was updated in accordance. The gem is still a dependency of Spinach, so it’s included in the development/test environments, but won’t be packaged with the actual product, and therefore doesn’t require we relicense the product. An attempt at relicensing Colorize was made, but didn’t succeed as the library owner never responded. Rainbow library: https://github.com/sickill/rainbow Relevant issue regarding licensing in GitLab's gems: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3775 --- lib/backup/manager.rb | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'lib/backup/manager.rb') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 660ca8c2923..9dd665441a0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -27,9 +27,9 @@ module Backup # Set file permissions on open to prevent chmod races. tar_system_options = {out: [tar_file, 'w', Gitlab.config.backup.archive_permissions]} if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "creating archive #{tar_file} failed".red + puts "creating archive #{tar_file} failed".color(:red) abort 'Backup failed' end @@ -43,7 +43,7 @@ module Backup connection_settings = Gitlab.config.backup.upload.connection if connection_settings.blank? - $progress.puts "skipped".yellow + $progress.puts "skipped".color(:yellow) return end @@ -53,9 +53,9 @@ module Backup if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "uploading backup to #{remote_directory} failed".red + puts "uploading backup to #{remote_directory} failed".color(:red) abort 'Backup failed' end end @@ -67,9 +67,9 @@ module Backup next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) - $progress.puts "done".green + $progress.puts "done".color(:green) else - puts "deleting tmp directory '#{dir}' failed".red + puts "deleting tmp directory '#{dir}' failed".color(:red) abort 'Backup failed' end end @@ -95,9 +95,9 @@ module Backup end end - $progress.puts "done. (#{removed} removed)".green + $progress.puts "done. (#{removed} removed)".color(:green) else - $progress.puts "skipping".yellow + $progress.puts "skipping".color(:yellow) end end @@ -124,20 +124,20 @@ module Backup $progress.print "Unpacking backup ... " unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".red + puts "unpacking backup failed".color(:red) exit 1 else - $progress.puts "done".green + $progress.puts "done".color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - puts "GitLab version mismatch:".red - puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".red - puts " Please switch to the following version and try again:".red - puts " version: #{settings[:gitlab_version]}".red + puts "GitLab version mismatch:".color(:red) + puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) + puts " Please switch to the following version and try again:".color(:red) + puts " version: #{settings[:gitlab_version]}".color(:red) puts puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 -- cgit v1.2.1 From f03df228155ae2d8dd779bd1a8e4078698b23c06 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 8 Jun 2016 18:04:31 -0700 Subject: Only create the backup directory if it is local Closes #12710 --- lib/backup/manager.rb | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'lib/backup/manager.rb') diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 9dd665441a0..2ff3e3bdfb0 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -38,7 +38,6 @@ module Backup end def upload(tar_file) - remote_directory = Gitlab.config.backup.upload.remote_directory $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " connection_settings = Gitlab.config.backup.upload.connection @@ -47,8 +46,7 @@ module Backup return end - connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.create(key: remote_directory) + directory = connect_to_remote_directory(connection_settings) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, @@ -155,6 +153,23 @@ module Backup private + def connect_to_remote_directory(connection_settings) + connection = ::Fog::Storage.new(connection_settings) + + # We only attempt to create the directory for local backups. For AWS + # and other cloud providers, we cannot guarantee the user will have + # permission to create the bucket. + if connection.service == ::Fog::Storage::Local + connection.directories.create(key: remote_directory) + else + connection.directories.get(remote_directory) + end + end + + def remote_directory + Gitlab.config.backup.upload.remote_directory + end + def backup_contents folders_to_backup + archives_to_backup + ["backup_information.yml"] end -- cgit v1.2.1