Merge branch 'fix/importing-io-timing-issue' into 'master'

Fix timing problems running imports on production

Fixes https://gitlab.com/gitlab-com/infrastructure/issues/151

I've found out that in staging, the imported file is not copied fully by the time sidekiq runs the job, this should hopefully fix it. (Tested against staging).

See merge request !5523

1 files changed, 17 insertions, 1 deletions

diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 82d1e1805c5..eca6e5b6d51 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -3,6 +3,8 @@ module Gitlab
     class FileImporter
       include Gitlab::ImportExport::CommandLineUtil
 
+      MAX_RETRIES = 8
+
       def self.import(*args)
         new(*args).import
       end
@@ -14,7 +16,10 @@ module Gitlab
 
       def import
         FileUtils.mkdir_p(@shared.export_path)
-        decompress_archive
+
+        wait_for_archived_file do
+          decompress_archive
+        end
       rescue => e
         @shared.error(e)
         false
@@ -22,6 +27,17 @@ module Gitlab
 
       private
 
+      # Exponentially sleep until I/O finishes copying the file
+      def wait_for_archived_file
+        MAX_RETRIES.times do |retry_number|
+          break if File.exist?(@archive_file)
+
+          sleep(2**retry_number)
+        end
+
+        yield
+      end
+
       def decompress_archive
         result = untar_zxf(archive: @archive_file, dir: @shared.export_path)