Add support to migrate existing projects to Hashed Storage asynchashed-storage-migration-path

5 files changed, 149 insertions, 2 deletions

diff --git a/app/models/project.rb b/app/models/project.rb
index f7221e4f3b2..9b10401f5b8 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -245,6 +245,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
+  scope :with_hashed_storage, -> { where('storage_version >= 1') }
+  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
 
@@ -1550,18 +1553,44 @@ class Project < ActiveRecord::Base
   end
 
   def legacy_storage?
-    self.storage_version.nil?
+    [nil, 0].include?(self.storage_version)
+  end
+
+  def hashed_storage?
+    self.storage_version && self.storage_version >= 1
   end
 
   def renamed?
     persisted? && path_changed?
   end
 
+  def migrate_to_hashed_storage!
+    return if hashed_storage?
+
+    update!(repository_read_only: true)
+
+    if repo_reference_count > 0 || wiki_reference_count > 0
+      ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
+    else
+      ProjectMigrateHashedStorageWorker.perform_async(id)
+    end
+  end
+
+  def storage_version=(value)
+    super
+
+    @storage = nil if storage_version_changed?
+  end
+
+  def gl_repository(is_wiki:)
+    Gitlab::GlRepository.gl_repository(self, is_wiki)
+  end
+
   private
 
   def storage
     @storage ||=
-      if self.storage_version && self.storage_version >= 1
+      if hashed_storage?
         Storage::HashedProject.new(self)
       else
         Storage::LegacyProject.new(self)
@@ -1574,6 +1603,14 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def repo_reference_count
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+  end
+
+  def wiki_reference_count
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+  end
+
   # set last_activity_at to the same as created_at
   def set_last_activity_at
     update_column(:last_activity_at, self.created_at)
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index fae1b64961a..f025f40994e 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,6 +4,7 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
+    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb
new file mode 100644
index 00000000000..41259de3a16
--- /dev/null
+++ b/app/services/projects/hashed_storage_migration_service.rb
@@ -0,0 +1,68 @@
+module Projects
+  class HashedStorageMigrationService < BaseService
+    include Gitlab::ShellAdapter
+
+    attr_reader :old_disk_path, :new_disk_path
+
+    def initialize(project, logger = nil)
+      @project = project
+      @logger ||= Rails.logger
+    end
+
+    def execute
+      return if project.hashed_storage?
+
+      @old_disk_path = project.disk_path
+      has_wiki = project.wiki.repository_exists?
+
+      project.storage_version = Storage::HashedProject::STORAGE_VERSION
+      project.ensure_storage_path_exists
+
+      @new_disk_path = project.disk_path
+
+      result = move_repository(@old_disk_path, @new_disk_path)
+
+      if has_wiki
+        result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki")
+      end
+
+      unless result
+        rollback_folder_move
+        return
+      end
+
+      project.repository_read_only = false
+      project.save!
+
+      block_given? ? yield : result
+    end
+
+    private
+
+    def move_repository(from_name, to_name)
+      from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
+      to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
+
+      # If we don't find the repository on either original or target we should log that as it could be an issue if the
+      # project was not originally empty.
+      if !from_exists && !to_exists
+        logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
+        return false
+      elsif !from_exists
+        # Repository have been moved already.
+        return true
+      end
+
+      gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
+    end
+
+    def rollback_folder_move
+      move_repository(@new_disk_path, @old_disk_path)
+      move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
+    end
+
+    def logger
+      @logger
+    end
+  end
+end
diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb
new file mode 100644
index 00000000000..ca276d7801c
--- /dev/null
+++ b/app/workers/project_migrate_hashed_storage_worker.rb
@@ -0,0 +1,11 @@
+class ProjectMigrateHashedStorageWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  def perform(project_id)
+    project = Project.find_by(id: project_id)
+    return if project.nil? || project.pending_delete?
+
+    ::Projects::HashedStorageMigrationService.new(project, logger).execute
+  end
+end
diff --git a/app/workers/storage_migrator_worker.rb b/app/workers/storage_migrator_worker.rb
new file mode 100644
index 00000000000..b48ead799b9
--- /dev/null
+++ b/app/workers/storage_migrator_worker.rb
@@ -0,0 +1,30 @@
+class StorageMigratorWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  BATCH_SIZE = 100
+
+  def perform(start, finish)
+    projects = build_relation(start, finish)
+
+    projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
+      Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
+
+      begin
+        project.migrate_to_hashed_storage!
+      rescue => err
+        Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
+      end
+    end
+  end
+
+  def build_relation(start, finish)
+    relation = Project
+    table = Project.arel_table
+
+    relation = relation.where(table[:id].gteq(start)) if start
+    relation = relation.where(table[:id].lteq(finish)) if finish
+
+    relation
+  end
+end