Merge branch 'gitaly-disk-access-2' into 'master'

Find and mark more Git disk access locations

See merge request gitlab-org/gitlab-ce!19363

10 files changed, 36 insertions, 27 deletions

diff --git a/lib/backup.rb b/lib/backup.rb
new file mode 100644
index 00000000000..e2c62af23ae
--- /dev/null
+++ b/lib/backup.rb
@@ -0,0 +1,3 @@
+module Backup
+  Error = Class.new(StandardError)
+end
diff --git a/lib/backup/database.rb b/lib/backup/database.rb
index 1608f7ad02d..086ca5986bd 100644
--- a/lib/backup/database.rb
+++ b/lib/backup/database.rb
@@ -44,7 +44,7 @@ module Backup
       end
 
       report_success(success)
-      abort 'Backup failed' unless success
+      raise Backup::Error, 'Backup failed' unless success
     end
 
     def restore
@@ -72,7 +72,7 @@ module Backup
       end
 
       report_success(success)
-      abort 'Restore failed' unless success
+      abort Backup::Error, 'Restore failed' unless success
     end
 
     protected
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 9895db9e451..d769a3ee7b0 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -26,7 +26,7 @@ module Backup
 
         unless status.zero?
           puts output
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
 
         run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600])
@@ -39,7 +39,11 @@ module Backup
     def restore
       backup_existing_files_dir
 
-      run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+      run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball)
+    end
+
+    def tar
+      system(*%w[gtar --version], out: '/dev/null') ? 'gtar' : 'tar'
     end
 
     def backup_existing_files_dir
@@ -61,7 +65,7 @@ module Backup
 
     def run_pipeline!(cmd_list, options = {})
       status_list = Open3.pipeline(*cmd_list, options)
-      abort 'Backup failed' unless status_list.compact.all?(&:success?)
+      raise Backup::Error, 'Backup failed' unless status_list.compact.all?(&:success?)
     end
   end
 end
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index a8da0c7edef..a3641505196 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -27,7 +27,7 @@ module Backup
           progress.puts "done".color(:green)
         else
           puts "creating archive #{tar_file} failed".color(:red)
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
 
         upload
@@ -52,7 +52,7 @@ module Backup
         progress.puts "done".color(:green)
       else
         puts "uploading backup to #{remote_directory} failed".color(:red)
-        abort 'Backup failed'
+        raise Backup::Error, 'Backup failed'
       end
     end
 
@@ -66,7 +66,7 @@ module Backup
           progress.puts "done".color(:green)
         else
           puts "deleting tmp directory '#{dir}' failed".color(:red)
-          abort 'Backup failed'
+          raise Backup::Error, 'Backup failed'
         end
       end
     end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 84670d6582e..1b1c83d9fb3 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -17,7 +17,10 @@ module Backup
 
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{display_repo_path(project)} ... "
-        path_to_project_repo = path_to_repo(project)
+
+        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(project)
+        end
         path_to_project_bundle = path_to_bundle(project)
 
         # Create namespace dir or hashed path if missing
@@ -51,7 +54,9 @@ module Backup
         end
 
         wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = path_to_repo(wiki)
+        path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(wiki)
+        end
         path_to_wiki_bundle = path_to_bundle(wiki)
 
         if File.exist?(path_to_wiki_repo)
@@ -111,7 +116,9 @@ module Backup
       # TODO: Need to find a way to do this for gitaly
       # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
       in_path(path_to_tars(project)) do |dir|
-        path_to_project_repo = path_to_repo(project)
+        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(project)
+        end
         cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
 
         output, status = Gitlab::Popen.popen(cmd)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 7acf11e3c91..bbbe0111a6f 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -109,7 +109,7 @@ module Gitlab
       end
 
       def ==(other)
-        path == other.path
+        [storage, relative_path] == [other.storage, other.relative_path]
       end
 
       def path
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 695462c7dd2..0c224bd1971 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -26,10 +26,6 @@ module Gitlab
         @shared.error(e)
         false
       end
-
-      def path_to_repo
-        @project.repository.path_to_repo
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 5fa2e101e29..2fd62c0fc7b 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -22,12 +22,8 @@ module Gitlab
         "project.wiki.bundle"
       end
 
-      def path_to_repo
-        @wiki.repository.path_to_repo
-      end
-
       def wiki_repository_exists?
-        File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
+        @wiki.repository.exists? && !@wiki.repository.empty?
       end
     end
   end
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index 42be301fd9b..723e655c150 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -128,10 +128,12 @@ module Gitlab
     end
 
     def all_repos
-      Gitlab.config.repositories.storages.each_value do |repository_storage|
-        IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
-          find.each_line do |path|
-            yield path.chomp
+      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        Gitlab.config.repositories.storages.each_value do |repository_storage|
+          IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find|
+            find.each_line do |path|
+              yield path.chomp
+            end
           end
         end
       end
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index d6d15285489..52ae1330d7f 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
       namespaces  = Namespace.pluck(:path)
       namespaces << HASHED_REPOSITORY_NAME  # add so that it will be ignored
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        git_base_path = repository_storage.legacy_disk_path
+        git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path }
         all_dirs = Dir.glob(git_base_path + '/*')
 
         puts git_base_path.color(:yellow)
@@ -54,7 +54,8 @@ namespace :gitlab do
 
       move_suffix = "+orphaned+#{Time.now.to_i}"
       Gitlab.config.repositories.storages.each do |name, repository_storage|
-        repo_root = repository_storage.legacy_disk_path
+        repo_root = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path }
+
         # Look for global repos (legacy, depth 1) and normal repos (depth 2)
         IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find|
           find.each_line do |path|