Merge branch 'fix-data-integrity-issue-with-repository-downloads-path' into 'master'

Avoid data-integrity issue when cleaning up repository archive cache

## What does this MR do?
Sets the default value for `repository_downloads_path` if someone has it configured incorrectly, and it points to the path where repositories are stored. It's also replace invocation of `find` with Ruby code that matches old cached files in a better, and safe way to avoid data-integrity issues.

## Why was this MR needed?
The `repository_downloads_path` is used by the `RepositoryArchiveCacheWorker` to remove outdated repository archives, if it points to the wrong directory can cause some data-integrity issue.

## What are the relevant issue numbers?
Closes #14222

See merge request !5285

2 files changed, 81 insertions, 24 deletions

diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb
index 59c5732c075..38b0c345b48 100644
--- a/spec/models/repository_spec.rb
+++ b/spec/models/repository_spec.rb
@@ -1172,30 +1172,6 @@ describe Repository, models: true do
     end
   end
 
-  describe '.clean_old_archives' do
-    let(:path) { Gitlab.config.gitlab.repository_downloads_path }
-
-    context 'when the downloads directory does not exist' do
-      it 'does not remove any archives' do
-        expect(File).to receive(:directory?).with(path).and_return(false)
-
-        expect(Gitlab::Popen).not_to receive(:popen)
-
-        described_class.clean_old_archives
-      end
-    end
-
-    context 'when the downloads directory exists' do
-      it 'removes old archives' do
-        expect(File).to receive(:directory?).with(path).and_return(true)
-
-        expect(Gitlab::Popen).to receive(:popen)
-
-        described_class.clean_old_archives
-      end
-    end
-  end
-
   describe "#keep_around" do
     it "stores a reference to the specified commit sha so it isn't garbage collected" do
       repository.keep_around(sample_commit.id)
diff --git a/spec/services/repository_archive_clean_up_service_spec.rb b/spec/services/repository_archive_clean_up_service_spec.rb
new file mode 100644
index 00000000000..842585f9e54
--- /dev/null
+++ b/spec/services/repository_archive_clean_up_service_spec.rb
@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe RepositoryArchiveCleanUpService, services: true do
+  describe '#execute' do
+    subject(:service) { described_class.new }
+
+    context 'when the downloads directory does not exist' do
+      it 'does not remove any archives' do
+        path = '/invalid/path/'
+        stub_repository_downloads_path(path)
+
+        expect(File).to receive(:directory?).with(path).and_return(false)
+        expect(service).not_to receive(:clean_up_old_archives)
+        expect(service).not_to receive(:clean_up_empty_directories)
+
+        service.execute
+      end
+    end
+
+    context 'when the downloads directory exists' do
+      shared_examples 'invalid archive files' do |dirname, extensions, mtime|
+        it 'does not remove files and directoy' do
+          in_directory_with_files(dirname, extensions, mtime) do |dir, files|
+            service.execute
+
+            files.each { |file| expect(File.exist?(file)).to eq true }
+            expect(File.directory?(dir)).to eq true
+          end
+        end
+      end
+
+      it 'removes files older than 2 hours that matches valid archive extensions' do
+        in_directory_with_files('sample.git', %w[tar tar.bz2 tar.gz zip], 2.hours) do |dir, files|
+          service.execute
+
+          files.each { |file| expect(File.exist?(file)).to eq false }
+          expect(File.directory?(dir)).to eq false
+        end
+      end
+
+      context 'with files older than 2 hours that does not matches valid archive extensions' do
+        it_behaves_like 'invalid archive files', 'sample.git', %w[conf rb], 2.hours
+      end
+
+      context 'with files older than 2 hours inside invalid directories' do
+        it_behaves_like 'invalid archive files', 'john_doe/sample.git', %w[conf rb tar tar.gz], 2.hours
+      end
+
+      context 'with files newer than 2 hours that matches valid archive extensions' do
+        it_behaves_like 'invalid archive files', 'sample.git', %w[tar tar.bz2 tar.gz zip], 1.hour
+      end
+
+      context 'with files newer than 2 hours that does not matches valid archive extensions' do
+        it_behaves_like 'invalid archive files', 'sample.git', %w[conf rb], 1.hour
+      end
+
+      context 'with files newer than 2 hours inside invalid directories' do
+        it_behaves_like 'invalid archive files', 'sample.git', %w[conf rb tar tar.gz], 1.hour
+      end
+    end
+
+    def in_directory_with_files(dirname, extensions, mtime)
+      Dir.mktmpdir do |tmpdir|
+        stub_repository_downloads_path(tmpdir)
+        dir = File.join(tmpdir, dirname)
+        files = create_temporary_files(dir, extensions, mtime)
+
+        yield(dir, files)
+      end
+    end
+
+    def stub_repository_downloads_path(path)
+      allow(Gitlab.config.gitlab).to receive(:repository_downloads_path).and_return(path)
+    end
+
+    def create_temporary_files(dir, extensions, mtime)
+      FileUtils.mkdir_p(dir)
+      FileUtils.touch(extensions.map { |ext| File.join(dir, "sample.#{ext}") }, mtime: Time.now - mtime)
+    end
+  end
+end