Add latest changes from gitlab-org/gitlab@master

8 files changed, 353 insertions, 0 deletions

diff --git a/spec/lib/bulk_imports/file_downloads/filename_fetch_spec.rb b/spec/lib/bulk_imports/file_downloads/filename_fetch_spec.rb
new file mode 100644
index 00000000000..a77eba06027
--- /dev/null
+++ b/spec/lib/bulk_imports/file_downloads/filename_fetch_spec.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::FileDownloads::FilenameFetch do
+  let(:dummy_instance) { dummy_class.new }
+  let(:dummy_class) do
+    Class.new do
+      include BulkImports::FileDownloads::FilenameFetch
+    end
+  end
+
+  describe '#raise_error' do
+    it { expect { dummy_instance.raise_error('text') }.to raise_exception(NotImplementedError) }
+  end
+end
diff --git a/spec/lib/bulk_imports/file_downloads/validations_spec.rb b/spec/lib/bulk_imports/file_downloads/validations_spec.rb
new file mode 100644
index 00000000000..85f45c2a8f0
--- /dev/null
+++ b/spec/lib/bulk_imports/file_downloads/validations_spec.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::FileDownloads::Validations do
+  let(:dummy_instance) { dummy_class.new }
+  let(:dummy_class) do
+    Class.new do
+      include BulkImports::FileDownloads::Validations
+    end
+  end
+
+  describe '#raise_error' do
+    it { expect { dummy_instance.raise_error('text') }.to raise_exception(NotImplementedError) }
+  end
+
+  describe '#filepath' do
+    it { expect { dummy_instance.filepath }.to raise_exception(NotImplementedError) }
+  end
+
+  describe '#response_headers' do
+    it { expect { dummy_instance.response_headers }.to raise_exception(NotImplementedError) }
+  end
+
+  describe '#file_size_limit' do
+    it { expect { dummy_instance.file_size_limit }.to raise_exception(NotImplementedError) }
+  end
+end
diff --git a/spec/lib/gitlab/github_import/attachments_downloader_spec.rb b/spec/lib/gitlab/github_import/attachments_downloader_spec.rb
new file mode 100644
index 00000000000..57391e06192
--- /dev/null
+++ b/spec/lib/gitlab/github_import/attachments_downloader_spec.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::GithubImport::AttachmentsDownloader do
+  subject(:downloader) { described_class.new(file_url) }
+
+  let_it_be(:file_url) { 'https://example.com/avatar.png' }
+  let_it_be(:content_type) { 'application/octet-stream' }
+
+  let(:content_length) { 1000 }
+  let(:chunk_double) { instance_double(HTTParty::FragmentWithResponse, code: 200) }
+  let(:headers_double) do
+    instance_double(
+      HTTParty::Response,
+      code: 200,
+      success?: true,
+      parsed_response: {},
+      headers: {
+        'content-length' => content_length,
+        'content-type' => content_type
+      }
+    )
+  end
+
+  describe '#perform' do
+    before do
+      allow(Gitlab::HTTP).to receive(:perform_request)
+        .with(Net::HTTP::Get, file_url, stream_body: true).and_yield(chunk_double)
+      allow(Gitlab::HTTP).to receive(:perform_request)
+        .with(Net::HTTP::Head, file_url, {}).and_return(headers_double)
+    end
+
+    context 'when file valid' do
+      it 'downloads file' do
+        file = downloader.perform
+
+        expect(File.exist?(file.path)).to eq(true)
+      end
+    end
+
+    context 'when filename is malicious' do
+      let_it_be(:file_url) { 'https://example.com/ava%2F..%2Ftar.png' }
+
+      it 'raises expected exception' do
+        expect { downloader.perform }.to raise_exception(
+          Gitlab::Utils::PathTraversalAttackError,
+          'Invalid path'
+        )
+      end
+    end
+
+    context 'when file size exceeds limit' do
+      let(:content_length) { 26.megabytes }
+
+      it 'raises expected exception' do
+        expect { downloader.perform }.to raise_exception(
+          Gitlab::GithubImport::AttachmentsDownloader::DownloadError,
+          'File size 26 MB exceeds limit of 25 MB'
+        )
+      end
+    end
+
+    context 'when file name length exceeds limit' do
+      before do
+        stub_const('BulkImports::FileDownloads::FilenameFetch::FILENAME_SIZE_LIMIT', 2)
+      end
+
+      it 'chops filename' do
+        file = downloader.perform
+
+        expect(File.exist?(file.path)).to eq(true)
+        expect(File.basename(file)).to eq('av.png')
+      end
+    end
+  end
+
+  describe '#delete' do
+    let(:tmp_dir_path) { File.join(Dir.tmpdir, 'github_attachments_test') }
+    let(:file) do
+      downloader.mkdir_p(tmp_dir_path)
+      file = File.open("#{tmp_dir_path}/test.txt", 'wb')
+      file.write('foo')
+      file.close
+      file
+    end
+
+    before do
+      allow(downloader).to receive(:filepath).and_return(file.path)
+    end
+
+    it 'removes file with parent folder' do
+      downloader.delete
+      expect(Dir.exist?(tmp_dir_path)).to eq false
+    end
+  end
+end
diff --git a/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb
new file mode 100644
index 00000000000..4779f9c8982
--- /dev/null
+++ b/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter do
+  subject(:importer) { described_class.new(release_attachments, project, client) }
+
+  let_it_be(:project) { create(:project) }
+
+  let(:client) { instance_double('Gitlab::GithubImport::Client') }
+  let(:release) { create(:release, project: project, description: description) }
+  let(:release_attachments) do
+    Gitlab::GithubImport::Representation::ReleaseAttachments
+      .from_json_hash(release_db_id: release.id, description: release.description)
+  end
+
+  let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' }
+  let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' }
+  let(:description) do
+    <<-TEXT.strip
+      Some text...
+
+      [special-doc](#{doc_url})
+      ![image.jpeg](#{image_url})
+    TEXT
+  end
+
+  describe '#execute' do
+    let(:downloader_stub) { instance_double(Gitlab::GithubImport::AttachmentsDownloader) }
+    let(:tmp_stub_doc) { Tempfile.create('attachment_download_test.txt') }
+    let(:tmp_stub_image) { Tempfile.create('image.jpeg') }
+
+    context 'when importing doc attachment' do
+      before do
+        allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(doc_url)
+          .and_return(downloader_stub)
+        allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(image_url)
+          .and_return(downloader_stub)
+        allow(downloader_stub).to receive(:perform).and_return(tmp_stub_doc, tmp_stub_image)
+        allow(downloader_stub).to receive(:delete).twice
+
+        allow(UploadService).to receive(:new)
+          .with(project, tmp_stub_doc, FileUploader).and_call_original
+        allow(UploadService).to receive(:new)
+          .with(project, tmp_stub_image, FileUploader).and_call_original
+      end
+
+      it 'updates release description with new attachment url' do
+        importer.execute
+
+        release.reload
+        expect(release.description).to start_with("Some text...\n\n      [special-doc](/uploads/")
+        expect(release.description).to include('![image.jpeg](/uploads/')
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb
new file mode 100644
index 00000000000..1aeb3462cd5
--- /dev/null
+++ b/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter do
+  subject { described_class.new(project, client) }
+
+  let_it_be(:project) { create(:project) }
+
+  let(:client) { instance_double(Gitlab::GithubImport::Client) }
+
+  describe '#each_object_to_import', :clean_gitlab_redis_cache do
+    let!(:release_1) { create(:release, project: project) }
+    let!(:release_2) { create(:release, project: project) }
+
+    it 'iterates each project release' do
+      list = []
+      subject.each_object_to_import do |object|
+        list << object
+      end
+      expect(list).to contain_exactly(release_1, release_2)
+    end
+
+    context 'when release is already processed' do
+      it "doesn't process this release" do
+        subject.mark_as_imported(release_1)
+
+        list = []
+        subject.each_object_to_import do |object|
+          list << object
+        end
+        expect(list).to contain_exactly(release_2)
+      end
+    end
+  end
+
+  describe '#representation_class' do
+    it { expect(subject.representation_class).to eq(Gitlab::GithubImport::Representation::ReleaseAttachments) }
+  end
+
+  describe '#importer_class' do
+    it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter) }
+  end
+
+  describe '#sidekiq_worker_class' do
+    it { expect(subject.sidekiq_worker_class).to eq(Gitlab::GithubImport::ImportReleaseAttachmentsWorker) }
+  end
+
+  describe '#collection_method' do
+    it { expect(subject.collection_method).to eq(:release_attachments) }
+  end
+
+  describe '#object_type' do
+    it { expect(subject.object_type).to eq(:release_attachment) }
+  end
+
+  describe '#id_for_already_imported_cache' do
+    let(:release) { build_stubbed(:release) }
+
+    it { expect(subject.id_for_already_imported_cache(release)).to eq(release.id) }
+  end
+
+  describe '#object_representation' do
+    let(:release) { build_stubbed(:release) }
+
+    it 'returns release attachments representation' do
+      representation = subject.object_representation(release)
+
+      expect(representation.class).to eq subject.representation_class
+      expect(representation.release_db_id).to eq release.id
+      expect(representation.description).to eq release.description
+    end
+  end
+end
diff --git a/spec/lib/gitlab/github_import/markdown_text_spec.rb b/spec/lib/gitlab/github_import/markdown_text_spec.rb
index ad45469a4c3..1da6bb06403 100644
--- a/spec/lib/gitlab/github_import/markdown_text_spec.rb
+++ b/spec/lib/gitlab/github_import/markdown_text_spec.rb
@@ -60,6 +60,34 @@ RSpec.describe Gitlab::GithubImport::MarkdownText do
     end
   end
 
+  describe '.fetch_attachment_urls' do
+    let(:image_extension) { described_class::MEDIA_TYPES.sample }
+    let(:image_attachment) do
+      "![special-image](https://user-images.githubusercontent.com/6833862/"\
+        "176685788-e7a93168-7ded-406a-82b5-eb1c56685a93.#{image_extension})"
+    end
+
+    let(:doc_extension) { described_class::DOC_TYPES.sample }
+    let(:doc_attachment) do
+      "[some-doc](https://github.com/nickname/public-test-repo/"\
+      "files/9020437/git-cheat-sheet.#{doc_extension})"
+    end
+
+    let(:text) do
+      <<-TEXT
+        Comment with an attachment
+        #{image_attachment}
+        #{FFaker::Lorem.sentence}
+        #{doc_attachment}
+      TEXT
+    end
+
+    it 'fetches attachment urls' do
+      expect(described_class.fetch_attachment_urls(text))
+        .to contain_exactly(image_attachment, doc_attachment)
+    end
+  end
+
   describe '#to_s' do
     it 'returns the text when the author was found' do
       author = double(:author, login: 'Alice')
diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb
index 738e7c88d7d..860bb60f3ed 100644
--- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb
+++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb
@@ -15,6 +15,10 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do
         Class
       end
 
+      def sidekiq_worker_class
+        Class
+      end
+
       def object_type
         :dummy
       end
diff --git a/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb b/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb
new file mode 100644
index 00000000000..0ef9dad6a13
--- /dev/null
+++ b/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::GithubImport::Representation::ReleaseAttachments do
+  shared_examples 'a Release attachments data' do
+    it 'returns an instance of ReleaseAttachments' do
+      expect(representation).to be_an_instance_of(described_class)
+    end
+
+    it 'includes release DB id' do
+      expect(representation.release_db_id).to eq 42
+    end
+
+    it 'includes release description' do
+      expect(representation.description).to eq 'Some text here..'
+    end
+  end
+
+  describe '.from_db_record' do
+    let(:release) { build_stubbed(:release, id: 42, description: 'Some text here..') }
+
+    it_behaves_like 'a Release attachments data' do
+      let(:representation) { described_class.from_db_record(release) }
+    end
+  end
+
+  describe '.from_json_hash' do
+    it_behaves_like 'a Release attachments data' do
+      let(:hash) do
+        {
+          'release_db_id' => 42,
+          'description' => 'Some text here..'
+        }
+      end
+
+      let(:representation) { described_class.from_json_hash(hash) }
+    end
+  end
+
+  describe '#github_identifiers' do
+    it 'returns a hash with needed identifiers' do
+      release_id = rand(100)
+      representation = described_class.new(release_db_id: release_id, description: 'text')
+
+      expect(representation.github_identifiers).to eq({ db_id: release_id })
+    end
+  end
+end