diff options
| -rw-r--r-- | app/models/concerns/storage/legacy_namespace.rb | 109 | ||||
| -rw-r--r-- | app/models/concerns/storage/legacy_project.rb | 76 | ||||
| -rw-r--r-- | app/models/namespace.rb | 104 | ||||
| -rw-r--r-- | app/models/project.rb | 73 |
4 files changed, 194 insertions, 168 deletions
diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb new file mode 100644 index 00000000000..5cbd52e5c75 --- /dev/null +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -0,0 +1,109 @@ +module Storage + module LegacyNamespace + extend ActiveSupport::Concern + + def move_dir + if any_project_has_container_registry_tags? + raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') + end + + # Move the namespace directory in all storages paths used by member projects + repository_storage_paths.each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, full_path_was) + + unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path) + Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}" + + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise Gitlab::UpdatePathError.new('namespace directory cannot be moved') + end + end + + Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path) + Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path) + + remove_exports! + + # If repositories moved successfully we need to + # send update instructions to users. + # However we cannot allow rollback since we moved namespace dir + # So we basically we mute exceptions in next actions + begin + send_update_instructions + true + rescue + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false + end + end + + # Hooks + + # Save the storage paths before the projects are destroyed to use them on after destroy + def prepare_for_destroy + old_repository_storage_paths + end + + private + + def send_update_instructions + projects.each do |project| + project.send_move_instructions("#{full_path_was}/#{project.path}") + end + end + + def old_repository_storage_paths + @old_repository_storage_paths ||= repository_storage_paths + end + + def repository_storage_paths + # We need to get the storage paths for all the projects, even the ones that are + # pending delete. Unscoping also get rids of the default order, which causes + # problems with SELECT DISTINCT. + Project.unscoped do + all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) + end + end + + def rm_dir + # Remove the namespace directory in all storages paths used by member projects + old_repository_storage_paths.each do |repository_storage_path| + # Move namespace directory into trash. + # We will remove it later async + new_path = "#{full_path}+#{id}+deleted" + + if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path) + message = "Namespace directory \"#{full_path}\" moved to \"#{new_path}\"" + Gitlab::AppLogger.info message + + # Remove namespace directroy async with delay so + # GitLab has time to remove all projects first + run_after_commit do + GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) + end + end + end + + remove_exports! + end + + def remove_exports! + Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) + end + + def export_path + File.join(Gitlab::ImportExport.storage_path, full_path_was) + end + + def full_path_was + if parent + parent.full_path + '/' + path_was + else + path_was + end + end + end +end diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb new file mode 100644 index 00000000000..b537f40548e --- /dev/null +++ b/app/models/concerns/storage/legacy_project.rb @@ -0,0 +1,76 @@ +module Storage + module LegacyProject + extend ActiveSupport::Concern + + def disk_path + full_path + end + + def ensure_dir_exist + gitlab_shell.add_namespace(repository_storage_path, namespace.full_path) + end + + def rename_repo + path_was = previous_changes['path'].first + old_path_with_namespace = File.join(namespace.full_path, path_was) + new_path_with_namespace = File.join(namespace.full_path, path) + + Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" + + if has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" + + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') + end + + expire_caches_before_rename(old_path_with_namespace) + + if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) + # If repository moved successfully we need to send update instructions to users. + # However we cannot allow rollback since we moved repository + # So we basically we mute exceptions in next actions + begin + gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") + send_move_instructions(old_path_with_namespace) + expires_full_path_cache + + @old_path_with_namespace = old_path_with_namespace + + SystemHooksService.new.execute_hooks_for(self, :rename) + + @repository = nil + rescue => e + Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false + end + else + Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise StandardError.new('repository cannot be renamed') + end + + Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" + + Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) + Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) + end + + def create_repository(force: false) + # Forked import is handled asynchronously + return if forked? && !force + + if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) + repository.after_create + true + else + errors.add(:base, 'Failed to create repository via gitlab-shell') + false + end + end + end +end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 0bb04194bdb..010c97c507e 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -8,6 +8,7 @@ class Namespace < ActiveRecord::Base include Gitlab::VisibilityLevel include Routable include AfterCommitQueue + include Storage::LegacyNamespace # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of @@ -41,10 +42,11 @@ class Namespace < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true - after_update :move_dir, if: :path_changed? after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } - # Save the storage paths before the projects are destroyed to use them on after destroy + # Legacy Storage specific hooks + + after_update :move_dir, if: :path_changed? before_destroy(prepend: true) { prepare_for_destroy } after_destroy :rm_dir @@ -118,53 +120,10 @@ class Namespace < ActiveRecord::Base owner_name end - def move_dir - if any_project_has_container_registry_tags? - raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') - end - - # Move the namespace directory in all storages paths used by member projects - repository_storage_paths.each do |repository_storage_path| - # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, full_path_was) - - unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}" - - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise Gitlab::UpdatePathError.new('namespace directory cannot be moved') - end - end - - Gitlab::UploadsTransfer.new.rename_namespace(full_path_was, full_path) - Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path) - - remove_exports! - - # If repositories moved successfully we need to - # send update instructions to users. - # However we cannot allow rollback since we moved namespace dir - # So we basically we mute exceptions in next actions - begin - send_update_instructions - rescue - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - end - def any_project_has_container_registry_tags? all_projects.any?(&:has_container_registry_tags?) end - def send_update_instructions - projects.each do |project| - project.send_move_instructions("#{full_path_was}/#{project.path}") - end - end - def kind type == 'Group' ? 'group' : 'user' end @@ -206,14 +165,6 @@ class Namespace < ActiveRecord::Base parent_id_changed? end - def prepare_for_destroy - old_repository_storage_paths - end - - def old_repository_storage_paths - @old_repository_storage_paths ||= repository_storage_paths - end - # Includes projects from this namespace and projects from all subgroups # that belongs to this namespace def all_projects @@ -232,37 +183,6 @@ class Namespace < ActiveRecord::Base private - def repository_storage_paths - # We need to get the storage paths for all the projects, even the ones that are - # pending delete. Unscoping also get rids of the default order, which causes - # problems with SELECT DISTINCT. - Project.unscoped do - all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) - end - end - - def rm_dir - # Remove the namespace directory in all storages paths used by member projects - old_repository_storage_paths.each do |repository_storage_path| - # Move namespace directory into trash. - # We will remove it later async - new_path = "#{full_path}+#{id}+deleted" - - if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path) - message = "Namespace directory \"#{full_path}\" moved to \"#{new_path}\"" - Gitlab::AppLogger.info message - - # Remove namespace directroy async with delay so - # GitLab has time to remove all projects first - run_after_commit do - GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) - end - end - end - - remove_exports! - end - def refresh_access_of_projects_invited_groups Group .joins(project_group_links: :project) @@ -270,22 +190,6 @@ class Namespace < ActiveRecord::Base .find_each(&:refresh_members_authorized_projects) end - def remove_exports! - Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) - end - - def export_path - File.join(Gitlab::ImportExport.storage_path, full_path_was) - end - - def full_path_was - if parent - parent.full_path + '/' + path_was - else - path_was - end - end - def nesting_level_allowed if ancestors.count > Group::NUMBER_OF_ANCESTORS_ALLOWED errors.add(:parent_id, "has too deep level of nesting") diff --git a/app/models/project.rb b/app/models/project.rb index d827bfaa806..39949e6dbfd 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -17,6 +17,7 @@ class Project < ActiveRecord::Base include ProjectFeaturesCompatibility include SelectForProjectAuthorization include Routable + include Storage::LegacyProject extend Gitlab::ConfigHelper @@ -45,7 +46,6 @@ class Project < ActiveRecord::Base after_create :ensure_dir_exist after_create :create_project_feature, unless: :project_feature - after_save :ensure_dir_exist, if: :namespace_id_changed? after_save :update_project_statistics, if: :namespace_id_changed? # set last_activity_at to the same as created_at @@ -67,6 +67,10 @@ class Project < ActiveRecord::Base after_validation :check_pending_delete + # Legacy Storage specific hooks + + after_save :ensure_dir_exist, if: :namespace_id_changed? + acts_as_taggable attr_accessor :new_default_branch @@ -974,56 +978,6 @@ class Project < ActiveRecord::Base !group end - def rename_repo - path_was = previous_changes['path'].first - old_path_with_namespace = File.join(namespace.full_path, path_was) - new_path_with_namespace = File.join(namespace.full_path, path) - - Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}" - - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" - - # we currently doesn't support renaming repository if it contains images in container registry - raise StandardError.new('Project cannot be renamed, because images are present in its container registry') - end - - expire_caches_before_rename(old_path_with_namespace) - - if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) - # If repository moved successfully we need to send update instructions to users. - # However we cannot allow rollback since we moved repository - # So we basically we mute exceptions in next actions - begin - gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") - send_move_instructions(old_path_with_namespace) - expires_full_path_cache - - @old_path_with_namespace = old_path_with_namespace - - SystemHooksService.new.execute_hooks_for(self, :rename) - - @repository = nil - rescue => e - Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}" - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false - end - else - Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise StandardError.new('repository cannot be renamed') - end - - Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}" - - Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) - Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) - end - # Expires various caches before a project is renamed. def expire_caches_before_rename(old_path) repo = Repository.new(old_path, self) @@ -1109,19 +1063,6 @@ class Project < ActiveRecord::Base merge_requests.where(source_project_id: self.id) end - def create_repository(force: false) - # Forked import is handled asynchronously - return if forked? && !force - - if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) - repository.after_create - true - else - errors.add(:base, 'Failed to create repository via gitlab-shell') - false - end - end - def ensure_repository create_repository(force: true) unless repository_exists? end @@ -1327,10 +1268,6 @@ class Project < ActiveRecord::Base status.zero? end - def ensure_dir_exist - gitlab_shell.add_namespace(repository_storage_path, namespace.full_path) - end - def predefined_variables [ { key: 'CI_PROJECT_ID', value: id.to_s, public: true }, |