From f4de14d71f425dc14ee5837d96f4e9f42c7cc239 Mon Sep 17 00:00:00 2001 From: Gabriel Mazetto Date: Wed, 6 Sep 2017 07:16:26 +0200 Subject: Add support to migrate existing projects to Hashed Storage async --- app/models/project.rb | 41 ++++++++++++- app/models/storage/hashed_project.rb | 1 + .../projects/hashed_storage_migration_service.rb | 68 ++++++++++++++++++++++ .../project_migrate_hashed_storage_worker.rb | 11 ++++ app/workers/storage_migrator_worker.rb | 30 ++++++++++ 5 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 app/services/projects/hashed_storage_migration_service.rb create mode 100644 app/workers/project_migrate_hashed_storage_worker.rb create mode 100644 app/workers/storage_migrator_worker.rb (limited to 'app') diff --git a/app/models/project.rb b/app/models/project.rb index f7221e4f3b2..9b10401f5b8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -245,6 +245,9 @@ class Project < ActiveRecord::Base scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } + scope :with_hashed_storage, -> { where('storage_version >= 1') } + scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) } + scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } @@ -1550,18 +1553,44 @@ class Project < ActiveRecord::Base end def legacy_storage? - self.storage_version.nil? + [nil, 0].include?(self.storage_version) + end + + def hashed_storage? + self.storage_version && self.storage_version >= 1 end def renamed? persisted? && path_changed? end + def migrate_to_hashed_storage! + return if hashed_storage? + + update!(repository_read_only: true) + + if repo_reference_count > 0 || wiki_reference_count > 0 + ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id) + else + ProjectMigrateHashedStorageWorker.perform_async(id) + end + end + + def storage_version=(value) + super + + @storage = nil if storage_version_changed? + end + + def gl_repository(is_wiki:) + Gitlab::GlRepository.gl_repository(self, is_wiki) + end + private def storage @storage ||= - if self.storage_version && self.storage_version >= 1 + if hashed_storage? Storage::HashedProject.new(self) else Storage::LegacyProject.new(self) @@ -1574,6 +1603,14 @@ class Project < ActiveRecord::Base end end + def repo_reference_count + Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value + end + + def wiki_reference_count + Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value + end + # set last_activity_at to the same as created_at def set_last_activity_at update_column(:last_activity_at, self.created_at) diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index fae1b64961a..f025f40994e 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -4,6 +4,7 @@ module Storage delegate :gitlab_shell, :repository_storage_path, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze + STORAGE_VERSION = 1 def initialize(project) @project = project diff --git a/app/services/projects/hashed_storage_migration_service.rb b/app/services/projects/hashed_storage_migration_service.rb new file mode 100644 index 00000000000..41259de3a16 --- /dev/null +++ b/app/services/projects/hashed_storage_migration_service.rb @@ -0,0 +1,68 @@ +module Projects + class HashedStorageMigrationService < BaseService + include Gitlab::ShellAdapter + + attr_reader :old_disk_path, :new_disk_path + + def initialize(project, logger = nil) + @project = project + @logger ||= Rails.logger + end + + def execute + return if project.hashed_storage? + + @old_disk_path = project.disk_path + has_wiki = project.wiki.repository_exists? + + project.storage_version = Storage::HashedProject::STORAGE_VERSION + project.ensure_storage_path_exists + + @new_disk_path = project.disk_path + + result = move_repository(@old_disk_path, @new_disk_path) + + if has_wiki + result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki") + end + + unless result + rollback_folder_move + return + end + + project.repository_read_only = false + project.save! + + block_given? ? yield : result + end + + private + + def move_repository(from_name, to_name) + from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git") + to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git") + + # If we don't find the repository on either original or target we should log that as it could be an issue if the + # project was not originally empty. + if !from_exists && !to_exists + logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..." + return false + elsif !from_exists + # Repository have been moved already. + return true + end + + gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name) + end + + def rollback_folder_move + move_repository(@new_disk_path, @old_disk_path) + move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki") + end + + def logger + @logger + end + end +end diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb new file mode 100644 index 00000000000..ca276d7801c --- /dev/null +++ b/app/workers/project_migrate_hashed_storage_worker.rb @@ -0,0 +1,11 @@ +class ProjectMigrateHashedStorageWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def perform(project_id) + project = Project.find_by(id: project_id) + return if project.nil? || project.pending_delete? + + ::Projects::HashedStorageMigrationService.new(project, logger).execute + end +end diff --git a/app/workers/storage_migrator_worker.rb b/app/workers/storage_migrator_worker.rb new file mode 100644 index 00000000000..b48ead799b9 --- /dev/null +++ b/app/workers/storage_migrator_worker.rb @@ -0,0 +1,30 @@ +class StorageMigratorWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + BATCH_SIZE = 100 + + def perform(start, finish) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + + begin + project.migrate_to_hashed_storage! + rescue => err + Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + end + end + + def build_relation(start, finish) + relation = Project + table = Project.arel_table + + relation = relation.where(table[:id].gteq(start)) if start + relation = relation.where(table[:id].lteq(finish)) if finish + + relation + end +end -- cgit v1.2.1