Merge branch 'mk-improve-background-migration-specs' into 'master'

Improve background migration specs

See merge request gitlab-org/gitlab-ce!17162

7 files changed, 498 insertions, 365 deletions

diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb
new file mode 100644
index 00000000000..c76adcbe2f5
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_dependencies/untracked_file_spec.rb
@@ -0,0 +1,262 @@
+require 'spec_helper'
+
+# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
+describe Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile, :migration, schema: 20180208183958 do
+  include MigrationsHelpers::TrackUntrackedUploadsHelpers
+
+  let!(:appearances) { table(:appearances) }
+  let!(:namespaces) { table(:namespaces) }
+  let!(:projects) { table(:projects) }
+  let!(:routes) { table(:routes) }
+  let!(:uploads) { table(:uploads) }
+
+  before(:all) do
+    ensure_temporary_tracking_table_exists
+  end
+
+  describe '#upload_path' do
+    def assert_upload_path(file_path, expected_upload_path)
+      untracked_file = create_untracked_file(file_path)
+
+      expect(untracked_file.upload_path).to eq(expected_upload_path)
+    end
+
+    context 'for an appearance logo file path' do
+      it 'returns the file path relative to the CarrierWave root' do
+        assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg')
+      end
+    end
+
+    context 'for an appearance header_logo file path' do
+      it 'returns the file path relative to the CarrierWave root' do
+        assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg')
+      end
+    end
+
+    context 'for a pre-Markdown Note attachment file path' do
+      it 'returns the file path relative to the CarrierWave root' do
+        assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf')
+      end
+    end
+
+    context 'for a user avatar file path' do
+      it 'returns the file path relative to the CarrierWave root' do
+        assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg')
+      end
+    end
+
+    context 'for a group avatar file path' do
+      it 'returns the file path relative to the CarrierWave root' do
+        assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg')
+      end
+    end
+
+    context 'for a project avatar file path' do
+      it 'returns the file path relative to the CarrierWave root' do
+        assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg')
+      end
+    end
+
+    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
+      it 'returns the file path relative to the project directory in uploads' do
+        project = create_project
+        random_hex = SecureRandom.hex
+
+        assert_upload_path("/#{get_full_path(project)}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
+      end
+    end
+  end
+
+  describe '#uploader' do
+    def assert_uploader(file_path, expected_uploader)
+      untracked_file = create_untracked_file(file_path)
+
+      expect(untracked_file.uploader).to eq(expected_uploader)
+    end
+
+    context 'for an appearance logo file path' do
+      it 'returns AttachmentUploader as a string' do
+        assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader')
+      end
+    end
+
+    context 'for an appearance header_logo file path' do
+      it 'returns AttachmentUploader as a string' do
+        assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader')
+      end
+    end
+
+    context 'for a pre-Markdown Note attachment file path' do
+      it 'returns AttachmentUploader as a string' do
+        assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader')
+      end
+    end
+
+    context 'for a user avatar file path' do
+      it 'returns AvatarUploader as a string' do
+        assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader')
+      end
+    end
+
+    context 'for a group avatar file path' do
+      it 'returns AvatarUploader as a string' do
+        assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader')
+      end
+    end
+
+    context 'for a project avatar file path' do
+      it 'returns AvatarUploader as a string' do
+        assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader')
+      end
+    end
+
+    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
+      it 'returns FileUploader as a string' do
+        project = create_project
+
+        assert_uploader("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
+      end
+    end
+  end
+
+  describe '#model_type' do
+    def assert_model_type(file_path, expected_model_type)
+      untracked_file = create_untracked_file(file_path)
+
+      expect(untracked_file.model_type).to eq(expected_model_type)
+    end
+
+    context 'for an appearance logo file path' do
+      it 'returns Appearance as a string' do
+        assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance')
+      end
+    end
+
+    context 'for an appearance header_logo file path' do
+      it 'returns Appearance as a string' do
+        assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance')
+      end
+    end
+
+    context 'for a pre-Markdown Note attachment file path' do
+      it 'returns Note as a string' do
+        assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note')
+      end
+    end
+
+    context 'for a user avatar file path' do
+      it 'returns User as a string' do
+        assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User')
+      end
+    end
+
+    context 'for a group avatar file path' do
+      it 'returns Namespace as a string' do
+        assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace')
+      end
+    end
+
+    context 'for a project avatar file path' do
+      it 'returns Project as a string' do
+        assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project')
+      end
+    end
+
+    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
+      it 'returns Project as a string' do
+        project = create_project
+
+        assert_model_type("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'Project')
+      end
+    end
+  end
+
+  describe '#model_id' do
+    def assert_model_id(file_path, expected_model_id)
+      untracked_file = create_untracked_file(file_path)
+
+      expect(untracked_file.model_id).to eq(expected_model_id)
+    end
+
+    context 'for an appearance logo file path' do
+      it 'returns the ID as a string' do
+        assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1)
+      end
+    end
+
+    context 'for an appearance header_logo file path' do
+      it 'returns the ID as a string' do
+        assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1)
+      end
+    end
+
+    context 'for a pre-Markdown Note attachment file path' do
+      it 'returns the ID as a string' do
+        assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234)
+      end
+    end
+
+    context 'for a user avatar file path' do
+      it 'returns the ID as a string' do
+        assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234)
+      end
+    end
+
+    context 'for a group avatar file path' do
+      it 'returns the ID as a string' do
+        assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234)
+      end
+    end
+
+    context 'for a project avatar file path' do
+      it 'returns the ID as a string' do
+        assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234)
+      end
+    end
+
+    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
+      it 'returns the ID as a string' do
+        project = create_project
+
+        assert_model_id("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", project.id)
+      end
+    end
+  end
+
+  describe '#file_size' do
+    context 'for an appearance logo file path' do
+      let(:appearance) { create_or_update_appearance(logo: true) }
+      let(:untracked_file) { described_class.create!(path: get_uploads(appearance, 'Appearance').first.path) }
+
+      it 'returns the file size' do
+        expect(untracked_file.file_size).to eq(1062)
+      end
+    end
+
+    context 'for a project avatar file path' do
+      let(:project) { create_project(avatar: true) }
+      let(:untracked_file) { described_class.create!(path: get_uploads(project, 'Project').first.path) }
+
+      it 'returns the file size' do
+        expect(untracked_file.file_size).to eq(1062)
+      end
+    end
+
+    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
+      let(:project) { create_project }
+      let(:untracked_file) { create_untracked_file("/#{get_full_path(project)}/#{get_uploads(project, 'Project').first.path}") }
+
+      before do
+        add_markdown_attachment(project)
+      end
+
+      it 'returns the file size' do
+        expect(untracked_file.file_size).to eq(1062)
+      end
+    end
+  end
+
+  def create_untracked_file(path_relative_to_upload_dir)
+    described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}")
+  end
+end
diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb
index c8fa252439a..0d2074eed22 100644
--- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb
+++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb
@@ -1,17 +1,19 @@
 require 'spec_helper'
 
-# This migration is using UploadService, which sets uploads.secret that is only
-# added to the DB schema in 20180129193323. Since the test isn't isolated, we
-# just use the latest schema when testing this migration.
-# Ideally, the test should not use factories nor UploadService, and rely on the
-# `table` helper instead.
-describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180129193323 do
-  include TrackUntrackedUploadsHelpers
+# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
+describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180208183958 do
+  include MigrationsHelpers::TrackUntrackedUploadsHelpers
 
   subject { described_class.new }
 
-  let!(:untracked_files_for_uploads) { described_class::UntrackedFile }
-  let!(:uploads) { described_class::Upload }
+  let!(:appearances) { table(:appearances) }
+  let!(:namespaces) { table(:namespaces) }
+  let!(:notes) { table(:notes) }
+  let!(:projects) { table(:projects) }
+  let!(:routes) { table(:routes) }
+  let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
+  let!(:uploads) { table(:uploads) }
+  let!(:users) { table(:users) }
 
   before do
     ensure_temporary_tracking_table_exists
@@ -19,30 +21,30 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
   end
 
   context 'with untracked files and tracked files in untracked_files_for_uploads' do
-    let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
-    let!(:user1) { create(:user, :with_avatar) }
-    let!(:user2) { create(:user, :with_avatar) }
-    let!(:project1) { create(:project, :legacy_storage, :with_avatar) }
-    let!(:project2) { create(:project, :legacy_storage, :with_avatar) }
+    let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) }
+    let!(:user1) { create_user(avatar: true) }
+    let!(:user2) { create_user(avatar: true) }
+    let!(:project1) { create_project(avatar: true) }
+    let!(:project2) { create_project(avatar: true) }
 
     before do
-      UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
-      UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
+      add_markdown_attachment(project1)
+      add_markdown_attachment(project2)
 
       # File records created by PrepareUntrackedUploads
-      untracked_files_for_uploads.create!(path: appearance.uploads.first.path)
-      untracked_files_for_uploads.create!(path: appearance.uploads.last.path)
-      untracked_files_for_uploads.create!(path: user1.uploads.first.path)
-      untracked_files_for_uploads.create!(path: user2.uploads.first.path)
-      untracked_files_for_uploads.create!(path: project1.uploads.first.path)
-      untracked_files_for_uploads.create!(path: project2.uploads.first.path)
-      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}")
-      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}")
+      untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').first.path)
+      untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').last.path)
+      untracked_files_for_uploads.create!(path: get_uploads(user1, 'User').first.path)
+      untracked_files_for_uploads.create!(path: get_uploads(user2, 'User').first.path)
+      untracked_files_for_uploads.create!(path: get_uploads(project1, 'Project').first.path)
+      untracked_files_for_uploads.create!(path: get_uploads(project2, 'Project').first.path)
+      untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project1).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project1, 'Project').last.path}")
+      untracked_files_for_uploads.create!(path: "#{legacy_project_uploads_dir(project2).sub("#{MigrationsHelpers::TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project2, 'Project').last.path}")
 
       # Untrack 4 files
-      user2.uploads.delete_all
-      project2.uploads.delete_all # 2 files: avatar and a Markdown upload
-      appearance.uploads.where("path like '%header_logo%'").delete_all
+      get_uploads(user2, 'User').delete_all
+      get_uploads(project2, 'Project').delete_all # 2 files: avatar and a Markdown upload
+      get_uploads(appearance, 'Appearance').where("path like '%header_logo%'").delete_all
     end
 
     it 'adds untracked files to the uploads table' do
@@ -50,9 +52,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
         subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id)
       end.to change { uploads.count }.from(4).to(8)
 
-      expect(user2.uploads.count).to eq(1)
-      expect(project2.uploads.count).to eq(2)
-      expect(appearance.uploads.count).to eq(2)
+      expect(get_uploads(user2, 'User').count).to eq(1)
+      expect(get_uploads(project2, 'Project').count).to eq(2)
+      expect(get_uploads(appearance, 'Appearance').count).to eq(2)
     end
 
     it 'deletes rows after processing them' do
@@ -66,9 +68,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
     it 'does not create duplicate uploads of already tracked files' do
       subject.perform(1, untracked_files_for_uploads.last.id)
 
-      expect(user1.uploads.count).to eq(1)
-      expect(project1.uploads.count).to eq(2)
-      expect(appearance.uploads.count).to eq(2)
+      expect(get_uploads(user1, 'User').count).to eq(1)
+      expect(get_uploads(project1, 'Project').count).to eq(2)
+      expect(get_uploads(appearance, 'Appearance').count).to eq(2)
     end
 
     it 'uses the start and end batch ids [only 1st half]' do
@@ -80,11 +82,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
         subject.perform(start_id, end_id)
       end.to change { uploads.count }.from(4).to(6)
 
-      expect(user1.uploads.count).to eq(1)
-      expect(user2.uploads.count).to eq(1)
-      expect(appearance.uploads.count).to eq(2)
-      expect(project1.uploads.count).to eq(2)
-      expect(project2.uploads.count).to eq(0)
+      expect(get_uploads(user1, 'User').count).to eq(1)
+      expect(get_uploads(user2, 'User').count).to eq(1)
+      expect(get_uploads(appearance, 'Appearance').count).to eq(2)
+      expect(get_uploads(project1, 'Project').count).to eq(2)
+      expect(get_uploads(project2, 'Project').count).to eq(0)
 
       # Only 4 have been either confirmed or added to uploads
       expect(untracked_files_for_uploads.count).to eq(4)
@@ -99,11 +101,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
         subject.perform(start_id, end_id)
       end.to change { uploads.count }.from(4).to(6)
 
-      expect(user1.uploads.count).to eq(1)
-      expect(user2.uploads.count).to eq(0)
-      expect(appearance.uploads.count).to eq(1)
-      expect(project1.uploads.count).to eq(2)
-      expect(project2.uploads.count).to eq(2)
+      expect(get_uploads(user1, 'User').count).to eq(1)
+      expect(get_uploads(user2, 'User').count).to eq(0)
+      expect(get_uploads(appearance, 'Appearance').count).to eq(1)
+      expect(get_uploads(project1, 'Project').count).to eq(2)
+      expect(get_uploads(project2, 'Project').count).to eq(2)
 
       # Only 4 have been either confirmed or added to uploads
       expect(untracked_files_for_uploads.count).to eq(4)
@@ -122,7 +124,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
     end
 
     it 'does not block a whole batch because of one bad path' do
-      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a")
+      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a")
       expect(untracked_files_for_uploads.count).to eq(9)
       expect(uploads.count).to eq(4)
 
@@ -133,7 +135,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
     end
 
     it 'an unparseable path is shown in error output' do
-      bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a"
+      bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a"
       untracked_files_for_uploads.create!(path: bad_path)
 
       expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/)
@@ -152,363 +154,100 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
 
   describe 'upload outcomes for each path pattern' do
     shared_examples_for 'non_markdown_file' do
-      let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
+      let!(:expected_upload_attrs) { model_uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
       let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
 
       before do
-        model.uploads.delete_all
+        model_uploads.delete_all
       end
 
       it 'creates an Upload record' do
         expect do
           subject.perform(1, untracked_files_for_uploads.last.id)
-        end.to change { model.reload.uploads.count }.from(0).to(1)
+        end.to change { model_uploads.count }.from(0).to(1)
 
-        expect(model.uploads.first.attributes).to include(expected_upload_attrs)
+        expect(model_uploads.first.attributes).to include(expected_upload_attrs)
       end
     end
 
     context 'for an appearance logo file path' do
-      let(:model) { create_or_update_appearance(logo: uploaded_file) }
+      let(:model) { create_or_update_appearance(logo: true) }
+      let(:model_uploads) { get_uploads(model, 'Appearance') }
 
       it_behaves_like 'non_markdown_file'
     end
 
     context 'for an appearance header_logo file path' do
-      let(:model) { create_or_update_appearance(header_logo: uploaded_file) }
+      let(:model) { create_or_update_appearance(header_logo: true) }
+      let(:model_uploads) { get_uploads(model, 'Appearance') }
 
       it_behaves_like 'non_markdown_file'
     end
 
     context 'for a pre-Markdown Note attachment file path' do
-      let(:model) { create(:note, :with_attachment) }
-      let!(:expected_upload_attrs) { Upload.where(model_type: 'Note', model_id: model.id).first.attributes.slice('path', 'uploader', 'size', 'checksum') }
+      let(:model) { create_note(attachment: true) }
+      let!(:expected_upload_attrs) { get_uploads(model, 'Note').first.attributes.slice('path', 'uploader', 'size', 'checksum') }
       let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
 
       before do
-        Upload.where(model_type: 'Note', model_id: model.id).delete_all
+        get_uploads(model, 'Note').delete_all
       end
 
       # Can't use the shared example because Note doesn't have an `uploads` association
       it 'creates an Upload record' do
         expect do
           subject.perform(1, untracked_files_for_uploads.last.id)
-        end.to change { Upload.where(model_type: 'Note', model_id: model.id).count }.from(0).to(1)
+        end.to change { get_uploads(model, 'Note').count }.from(0).to(1)
 
-        expect(Upload.where(model_type: 'Note', model_id: model.id).first.attributes).to include(expected_upload_attrs)
+        expect(get_uploads(model, 'Note').first.attributes).to include(expected_upload_attrs)
       end
     end
 
     context 'for a user avatar file path' do
-      let(:model) { create(:user, :with_avatar) }
+      let(:model) { create_user(avatar: true) }
+      let(:model_uploads) { get_uploads(model, 'User') }
 
       it_behaves_like 'non_markdown_file'
     end
 
     context 'for a group avatar file path' do
-      let(:model) { create(:group, :with_avatar) }
+      let(:model) { create_group(avatar: true) }
+      let(:model_uploads) { get_uploads(model, 'Namespace') }
 
       it_behaves_like 'non_markdown_file'
     end
 
     context 'for a project avatar file path' do
-      let(:model) { create(:project, :legacy_storage, :with_avatar) }
+      let(:model) { create_project(avatar: true) }
+      let(:model_uploads) { get_uploads(model, 'Project') }
 
       it_behaves_like 'non_markdown_file'
     end
 
     context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      let(:model) { create(:project, :legacy_storage) }
+      let(:model) { create_project }
 
       before do
         # Upload the file
-        UploadService.new(model, uploaded_file, FileUploader).execute
+        add_markdown_attachment(model)
 
         # Create the untracked_files_for_uploads record
-        untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}")
+        untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(model)}/#{get_uploads(model, 'Project').first.path}")
 
         # Save the expected upload attributes
-        @expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum')
+        @expected_upload_attrs = get_uploads(model, 'Project').first.attributes.slice('path', 'uploader', 'size', 'checksum')
 
         # Untrack the file
-        model.reload.uploads.delete_all
+        get_uploads(model, 'Project').delete_all
       end
 
       it 'creates an Upload record' do
         expect do
           subject.perform(1, untracked_files_for_uploads.last.id)
-        end.to change { model.reload.uploads.count }.from(0).to(1)
+        end.to change { get_uploads(model, 'Project').count }.from(0).to(1)
 
-        expect(model.uploads.first.attributes).to include(@expected_upload_attrs)
+        expect(get_uploads(model, 'Project').first.attributes).to include(@expected_upload_attrs)
       end
     end
   end
 end
-
-describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
-  include TrackUntrackedUploadsHelpers
-
-  let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload }
-
-  before(:all) do
-    ensure_temporary_tracking_table_exists
-  end
-
-  describe '#upload_path' do
-    def assert_upload_path(file_path, expected_upload_path)
-      untracked_file = create_untracked_file(file_path)
-
-      expect(untracked_file.upload_path).to eq(expected_upload_path)
-    end
-
-    context 'for an appearance logo file path' do
-      it 'returns the file path relative to the CarrierWave root' do
-        assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg')
-      end
-    end
-
-    context 'for an appearance header_logo file path' do
-      it 'returns the file path relative to the CarrierWave root' do
-        assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg')
-      end
-    end
-
-    context 'for a pre-Markdown Note attachment file path' do
-      it 'returns the file path relative to the CarrierWave root' do
-        assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf')
-      end
-    end
-
-    context 'for a user avatar file path' do
-      it 'returns the file path relative to the CarrierWave root' do
-        assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg')
-      end
-    end
-
-    context 'for a group avatar file path' do
-      it 'returns the file path relative to the CarrierWave root' do
-        assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg')
-      end
-    end
-
-    context 'for a project avatar file path' do
-      it 'returns the file path relative to the CarrierWave root' do
-        assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg')
-      end
-    end
-
-    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      it 'returns the file path relative to the project directory in uploads' do
-        project = create(:project, :legacy_storage)
-        random_hex = SecureRandom.hex
-
-        assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
-      end
-    end
-  end
-
-  describe '#uploader' do
-    def assert_uploader(file_path, expected_uploader)
-      untracked_file = create_untracked_file(file_path)
-
-      expect(untracked_file.uploader).to eq(expected_uploader)
-    end
-
-    context 'for an appearance logo file path' do
-      it 'returns AttachmentUploader as a string' do
-        assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader')
-      end
-    end
-
-    context 'for an appearance header_logo file path' do
-      it 'returns AttachmentUploader as a string' do
-        assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader')
-      end
-    end
-
-    context 'for a pre-Markdown Note attachment file path' do
-      it 'returns AttachmentUploader as a string' do
-        assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader')
-      end
-    end
-
-    context 'for a user avatar file path' do
-      it 'returns AvatarUploader as a string' do
-        assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader')
-      end
-    end
-
-    context 'for a group avatar file path' do
-      it 'returns AvatarUploader as a string' do
-        assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader')
-      end
-    end
-
-    context 'for a project avatar file path' do
-      it 'returns AvatarUploader as a string' do
-        assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader')
-      end
-    end
-
-    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      it 'returns FileUploader as a string' do
-        project = create(:project, :legacy_storage)
-
-        assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
-      end
-    end
-  end
-
-  describe '#model_type' do
-    def assert_model_type(file_path, expected_model_type)
-      untracked_file = create_untracked_file(file_path)
-
-      expect(untracked_file.model_type).to eq(expected_model_type)
-    end
-
-    context 'for an appearance logo file path' do
-      it 'returns Appearance as a string' do
-        assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance')
-      end
-    end
-
-    context 'for an appearance header_logo file path' do
-      it 'returns Appearance as a string' do
-        assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance')
-      end
-    end
-
-    context 'for a pre-Markdown Note attachment file path' do
-      it 'returns Note as a string' do
-        assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note')
-      end
-    end
-
-    context 'for a user avatar file path' do
-      it 'returns User as a string' do
-        assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User')
-      end
-    end
-
-    context 'for a group avatar file path' do
-      it 'returns Namespace as a string' do
-        assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace')
-      end
-    end
-
-    context 'for a project avatar file path' do
-      it 'returns Project as a string' do
-        assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project')
-      end
-    end
-
-    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      it 'returns Project as a string' do
-        project = create(:project, :legacy_storage)
-
-        assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project')
-      end
-    end
-  end
-
-  describe '#model_id' do
-    def assert_model_id(file_path, expected_model_id)
-      untracked_file = create_untracked_file(file_path)
-
-      expect(untracked_file.model_id).to eq(expected_model_id)
-    end
-
-    context 'for an appearance logo file path' do
-      it 'returns the ID as a string' do
-        assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1)
-      end
-    end
-
-    context 'for an appearance header_logo file path' do
-      it 'returns the ID as a string' do
-        assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1)
-      end
-    end
-
-    context 'for a pre-Markdown Note attachment file path' do
-      it 'returns the ID as a string' do
-        assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234)
-      end
-    end
-
-    context 'for a user avatar file path' do
-      it 'returns the ID as a string' do
-        assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234)
-      end
-    end
-
-    context 'for a group avatar file path' do
-      it 'returns the ID as a string' do
-        assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234)
-      end
-    end
-
-    context 'for a project avatar file path' do
-      it 'returns the ID as a string' do
-        assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234)
-      end
-    end
-
-    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      it 'returns the ID as a string' do
-        project = create(:project, :legacy_storage)
-
-        assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id)
-      end
-    end
-  end
-
-  describe '#file_size' do
-    context 'for an appearance logo file path' do
-      let(:appearance) { create_or_update_appearance(logo: uploaded_file) }
-      let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) }
-
-      it 'returns the file size' do
-        expect(untracked_file.file_size).to eq(35255)
-      end
-
-      it 'returns the same thing that CarrierWave would return' do
-        expect(untracked_file.file_size).to eq(appearance.logo.size)
-      end
-    end
-
-    context 'for a project avatar file path' do
-      let(:project) { create(:project, :legacy_storage, avatar: uploaded_file) }
-      let(:untracked_file) { described_class.create!(path: project.uploads.first.path) }
-
-      it 'returns the file size' do
-        expect(untracked_file.file_size).to eq(35255)
-      end
-
-      it 'returns the same thing that CarrierWave would return' do
-        expect(untracked_file.file_size).to eq(project.avatar.size)
-      end
-    end
-
-    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
-      let(:project) { create(:project, :legacy_storage) }
-      let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") }
-
-      before do
-        UploadService.new(project, uploaded_file, FileUploader).execute
-      end
-
-      it 'returns the file size' do
-        expect(untracked_file.file_size).to eq(35255)
-      end
-
-      it 'returns the same thing that CarrierWave would return' do
-        expect(untracked_file.file_size).to eq(project.uploads.first.size)
-      end
-    end
-  end
-
-  def create_untracked_file(path_relative_to_upload_dir)
-    described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}")
-  end
-end
diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
index ca77e64ae40..35750d89c35 100644
--- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
+++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
@@ -1,10 +1,16 @@
 require 'spec_helper'
 
-describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migration, schema: 20180129193323 do
-  include TrackUntrackedUploadsHelpers
-  include MigrationsHelpers
-
-  let!(:untracked_files_for_uploads) { described_class::UntrackedFile }
+# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
+describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migration, schema: 20180208183958 do
+  include MigrationsHelpers::TrackUntrackedUploadsHelpers
+
+  let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
+  let!(:appearances) { table(:appearances) }
+  let!(:namespaces) { table(:namespaces) }
+  let!(:projects) { table(:projects) }
+  let!(:routes) { table(:routes) }
+  let!(:uploads) { table(:uploads) }
+  let!(:users) { table(:users) }
 
   around do |example|
     # Especially important so the follow-up migration does not get run
@@ -15,19 +21,17 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migrat
 
   shared_examples 'prepares the untracked_files_for_uploads table' do
     context 'when files were uploaded before and after hashed storage was enabled' do
-      let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
-      let!(:user) { create(:user, :with_avatar) }
-      let!(:project1) { create(:project, :with_avatar, :legacy_storage) }
-      let(:project2) { create(:project) } # instantiate after enabling hashed_storage
+      let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) }
+      let!(:user) { create_user(avatar: true) }
+      let!(:project1) { create_project(avatar: true) }
+      let(:project2) { create_project } # instantiate after enabling hashed_storage
 
       before do
         # Markdown upload before enabling hashed_storage
-        UploadService.new(project1, uploaded_file, FileUploader).execute
-
-        stub_application_setting(hashed_storage_enabled: true)
+        add_markdown_attachment(project1)
 
         # Markdown upload after enabling hashed_storage
-        UploadService.new(project2, uploaded_file, FileUploader).execute
+        add_markdown_attachment(project2, hashed_storage: true)
       end
 
       it 'has a path field long enough for really long paths' do
@@ -61,7 +65,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq, :migrat
       it 'does not add hashed files to the untracked_files_for_uploads table' do
         described_class.new.perform
 
-        hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
+        hashed_file_path = get_uploads(project2, 'Project').where(uploader: 'FileUploader').first.path
         expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
       end
 
diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb
index fe4d5b8a279..2fccfb3f12c 100644
--- a/spec/migrations/track_untracked_uploads_spec.rb
+++ b/spec/migrations/track_untracked_uploads_spec.rb
@@ -2,7 +2,7 @@ require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads')
 
 describe TrackUntrackedUploads, :migration, :sidekiq do
-  include TrackUntrackedUploadsHelpers
+  include MigrationsHelpers::TrackUntrackedUploadsHelpers
 
   it 'correctly schedules the follow-up background migration' do
     Sidekiq::Testing.fake! do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 85de0a14631..5600c9c6ad5 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -154,6 +154,22 @@ RSpec.configure do |config|
     Sidekiq.redis(&:flushall)
   end
 
+  # The :each scope runs "inside" the example, so this hook ensures the DB is in the
+  # correct state before any examples' before hooks are called. This prevents a
+  # problem where `ScheduleIssuesClosedAtTypeChange` (or any migration that depends
+  # on background migrations being run inline during test setup) can be broken by
+  # altering Sidekiq behavior in an unrelated spec like so:
+  #
+  # around do |example|
+  #   Sidekiq::Testing.fake! do
+  #     example.run
+  #   end
+  # end
+  config.before(:context, :migration) do
+    schema_migrate_down!
+  end
+
+  # Each example may call `migrate!`, so we must ensure we are migrated down every time
   config.before(:each, :migration) do
     schema_migrate_down!
   end
diff --git a/spec/support/migrations_helpers/track_untracked_uploads_helpers.rb b/spec/support/migrations_helpers/track_untracked_uploads_helpers.rb
new file mode 100644
index 00000000000..016bcfa9b1b
--- /dev/null
+++ b/spec/support/migrations_helpers/track_untracked_uploads_helpers.rb
@@ -0,0 +1,128 @@
+module MigrationsHelpers
+  module TrackUntrackedUploadsHelpers
+    PUBLIC_DIR = File.join(Rails.root, 'tmp', 'tests', 'public')
+    UPLOADS_DIR = File.join(PUBLIC_DIR, 'uploads')
+    SYSTEM_DIR = File.join(UPLOADS_DIR, '-', 'system')
+    UPLOAD_FILENAME = 'image.png'.freeze
+    FIXTURE_FILE_PATH = File.join(Rails.root, 'spec', 'fixtures', 'dk.png')
+    FIXTURE_CHECKSUM = 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'.freeze
+
+    def create_or_update_appearance(logo: false, header_logo: false)
+      appearance = appearances.first_or_create(title: 'foo', description: 'bar', logo: (UPLOAD_FILENAME if logo), header_logo: (UPLOAD_FILENAME if header_logo))
+
+      add_upload(appearance, 'Appearance', 'logo', 'AttachmentUploader') if logo
+      add_upload(appearance, 'Appearance', 'header_logo', 'AttachmentUploader') if header_logo
+
+      appearance
+    end
+
+    def create_group(avatar: false)
+      index = unique_index(:group)
+      group = namespaces.create(name: "group#{index}", path: "group#{index}", avatar: (UPLOAD_FILENAME if avatar))
+
+      add_upload(group, 'Group', 'avatar', 'AvatarUploader') if avatar
+
+      group
+    end
+
+    def create_note(attachment: false)
+      note = notes.create(attachment: (UPLOAD_FILENAME if attachment))
+
+      add_upload(note, 'Note', 'attachment', 'AttachmentUploader') if attachment
+
+      note
+    end
+
+    def create_project(avatar: false)
+      group = create_group
+      project = projects.create(namespace_id: group.id, path: "project#{unique_index(:project)}", avatar: (UPLOAD_FILENAME if avatar))
+      routes.create(path: "#{group.path}/#{project.path}", source_id: project.id, source_type: 'Project') # so Project.find_by_full_path works
+
+      add_upload(project, 'Project', 'avatar', 'AvatarUploader') if avatar
+
+      project
+    end
+
+    def create_user(avatar: false)
+      user = users.create(email: "foo#{unique_index(:user)}@bar.com", avatar: (UPLOAD_FILENAME if avatar), projects_limit: 100)
+
+      add_upload(user, 'User', 'avatar', 'AvatarUploader') if avatar
+
+      user
+    end
+
+    def unique_index(name = :unnamed)
+      @unique_index ||= {}
+      @unique_index[name] ||= 0
+      @unique_index[name] += 1
+    end
+
+    def add_upload(model, model_type, attachment_type, uploader)
+      file_path = upload_file_path(model, model_type, attachment_type)
+      path_relative_to_public = file_path.sub("#{PUBLIC_DIR}/", '')
+      create_file(file_path)
+
+      uploads.create!(
+        size: 1062,
+        path: path_relative_to_public,
+        model_id: model.id,
+        model_type: model_type == 'Group' ? 'Namespace' : model_type,
+        uploader: uploader,
+        checksum: FIXTURE_CHECKSUM
+      )
+    end
+
+    def add_markdown_attachment(project, hashed_storage: false)
+      project_dir = hashed_storage ? hashed_project_uploads_dir(project) : legacy_project_uploads_dir(project)
+      attachment_dir = File.join(project_dir, SecureRandom.hex)
+      attachment_file_path = File.join(attachment_dir, UPLOAD_FILENAME)
+      project_attachment_path_relative_to_project = attachment_file_path.sub("#{project_dir}/", '')
+      create_file(attachment_file_path)
+
+      uploads.create!(
+        size: 1062,
+        path: project_attachment_path_relative_to_project,
+        model_id: project.id,
+        model_type: 'Project',
+        uploader: 'FileUploader',
+        checksum: FIXTURE_CHECKSUM
+      )
+    end
+
+    def legacy_project_uploads_dir(project)
+      namespace = namespaces.find_by(id: project.namespace_id)
+      File.join(UPLOADS_DIR, namespace.path, project.path)
+    end
+
+    def hashed_project_uploads_dir(project)
+      File.join(UPLOADS_DIR, '@hashed', 'aa', 'aaaaaaaaaaaa')
+    end
+
+    def upload_file_path(model, model_type, attachment_type)
+      dir = File.join(upload_dir(model_type.downcase, attachment_type.to_s), model.id.to_s)
+      File.join(dir, UPLOAD_FILENAME)
+    end
+
+    def upload_dir(model_type, attachment_type)
+      File.join(SYSTEM_DIR, model_type, attachment_type)
+    end
+
+    def create_file(path)
+      File.delete(path) if File.exist?(path)
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.cp(FIXTURE_FILE_PATH, path)
+    end
+
+    def get_uploads(model, model_type)
+      uploads.where(model_type: model_type, model_id: model.id)
+    end
+
+    def get_full_path(project)
+      routes.find_by(source_id: project.id, source_type: 'Project').path
+    end
+
+    def ensure_temporary_tracking_table_exists
+      Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
+    end
+  end
+end
diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb
deleted file mode 100644
index a8b3ed1f41c..00000000000
--- a/spec/support/track_untracked_uploads_helpers.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-module TrackUntrackedUploadsHelpers
-  def uploaded_file
-    fixture_path = Rails.root.join('spec/fixtures/rails_sample.jpg')
-    fixture_file_upload(fixture_path)
-  end
-
-  def ensure_temporary_tracking_table_exists
-    Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
-  end
-
-  def create_or_update_appearance(attrs)
-    a = Appearance.first_or_initialize(title: 'foo', description: 'bar')
-    a.update!(attrs)
-    a
-  end
-end