diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/groups.rb | 4 | ||||
| -rw-r--r-- | lib/banzai/filter/emoji_filter.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/diff/file.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 329 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/commit_service.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/operation_service.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/import_export/group_project_object_builder.rb | 90 | ||||
| -rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 29 | ||||
| -rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 74 | ||||
| -rw-r--r-- | lib/gitlab/middleware/read_only/controller.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/repository_cache_adapter.rb | 10 |
11 files changed, 209 insertions, 372 deletions
diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c7f41aba854..f633dd88d06 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -56,6 +56,8 @@ module API def find_group_projects(params) group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = reorder_projects(projects) paginate(projects) end @@ -191,6 +193,8 @@ module API desc: 'Return only the ID, URL, name, and path of each project' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' use :pagination use :with_custom_attributes diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index e1261e7bbbe..4eccd9d5ed5 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -3,10 +3,6 @@ module Banzai # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter - # - # Context options: - # :asset_root - # :asset_host class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 40bcfa20e7d..a9e209d5b71 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -250,7 +250,7 @@ module Gitlab last_line = lines.last - if last_line.new_pos < total_blob_lines(blob) + if last_line.new_pos < total_blob_lines(blob) && !deleted_file? match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) lines.push(match_line) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7c3b91f6efb..706aa7343be 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -648,18 +648,14 @@ module Gitlab end def add_branch(branch_name, user:, target:) - gitaly_operation_client.user_create_branch(branch_name, user, target) - rescue GRPC::FailedPrecondition => ex - raise InvalidRef, ex + wrapped_gitaly_errors do + gitaly_operation_client.user_create_branch(branch_name, user, target) + end end def add_tag(tag_name, user:, target:, message: nil) - gitaly_migrate(:operation_user_add_tag, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_add_tag(tag_name, user: user, target: target, message: message) - else - rugged_add_tag(tag_name, user: user, target: target, message: message) - end + wrapped_gitaly_errors do + gitaly_operation_client.add_tag(tag_name, user, target, message) end end @@ -668,22 +664,14 @@ module Gitlab end def rm_branch(branch_name, user:) - gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_operations_client.user_delete_branch(branch_name, user) - else - OperationService.new(user, self).rm_branch(find_branch(branch_name)) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_delete_branch(branch_name, user) end end def rm_tag(tag_name, user:) - gitaly_migrate(:operation_user_delete_tag) do |is_enabled| - if is_enabled - gitaly_operations_client.rm_tag(tag_name, user) - else - Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) - end + wrapped_gitaly_errors do + gitaly_operation_client.rm_tag(tag_name, user) end end @@ -692,72 +680,29 @@ module Gitlab end def merge(user, source_sha, target_branch, message, &block) - gitaly_migrate(:operation_user_merge_branch) do |is_enabled| - if is_enabled - gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) - else - rugged_merge(user, source_sha, target_branch, message, &block) - end - end - end - - def rugged_merge(user, source_sha, target_branch, message) - committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) - - OperationService.new(user, self).with_branch(target_branch) do |start_commit| - our_commit = start_commit.sha - their_commit = source_sha - - raise 'Invalid merge target' unless our_commit - raise 'Invalid merge source' unless their_commit - - merge_index = rugged.merge_commits(our_commit, their_commit) - break if merge_index.conflicts? - - options = { - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - message: message, - author: committer, - committer: committer - } - - commit_id = create_commit(options) - - yield commit_id - - commit_id + wrapped_gitaly_errors do + gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) end - rescue Gitlab::Git::CommitError # when merge_index.conflicts? - nil end def ff_merge(user, source_sha, target_branch) - gitaly_migrate(:operation_user_ff_branch) do |is_enabled| - if is_enabled - gitaly_ff_merge(user, source_sha, target_branch) - else - rugged_ff_merge(user, source_sha, target_branch) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_ff_branch(user, source_sha, target_branch) end end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:revert, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - if is_enabled - gitaly_operations_client.user_revert(args) - else - rugged_revert(args) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_revert(args) end end @@ -775,21 +720,17 @@ module Gitlab end def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:cherry_pick) do |is_enabled| - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - if is_enabled - gitaly_operations_client.user_cherry_pick(args) - else - rugged_cherry_pick(args) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_cherry_pick(args) end end @@ -1113,20 +1054,12 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - gitaly_migrate(:rebase) do |is_enabled| - if is_enabled - gitaly_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - else - git_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) end end @@ -1137,13 +1070,9 @@ module Gitlab end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - gitaly_migrate(:squash) do |is_enabled| - if is_enabled - gitaly_operation_client.user_squash(user, squash_id, branch, + wrapped_gitaly_errors do + gitaly_operation_client.user_squash(user, squash_id, branch, start_sha, end_sha, author, message) - else - git_squash(user, squash_id, branch, start_sha, end_sha, author, message) - end end end @@ -1189,15 +1118,10 @@ module Gitlab author_email: nil, author_name: nil, start_branch_name: nil, start_repository: self) - gitaly_migrate(:operation_user_commit_files) do |is_enabled| - if is_enabled - gitaly_operation_client.user_commit_files(user, branch_name, + wrapped_gitaly_errors do + gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, start_branch_name, start_repository) - else - rugged_multi_action(user, branch_name, message, actions, - author_email, author_name, start_branch_name, start_repository) - end end end # rubocop:enable Metrics/ParameterLists @@ -1217,10 +1141,6 @@ module Gitlab Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end - def gitaly_operations_client - @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) - end - def gitaly_ref_client @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) end @@ -1760,33 +1680,6 @@ module Gitlab false end - def gitaly_add_tag(tag_name, user:, target:, message: nil) - gitaly_operations_client.add_tag(tag_name, user, target, message) - end - - def rugged_add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } - end - - Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex - rescue Rugged::TagError - raise TagExistsError - end - def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -1831,28 +1724,6 @@ module Gitlab end end - def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - revert_tree_id = check_revert_content(commit, start_commit.sha) - raise CreateTreeError unless revert_tree_id - - committer = user_to_committer(user) - - create_commit(message: message, - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) - end - end - def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) OperationService.new(user, self).with_branch( branch_name, @@ -1892,71 +1763,6 @@ module Gitlab tree_id end - def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - gitaly_operation_client.user_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end - - def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - if remote_repository.is_a?(RemoteRepository) - env.merge!(remote_repository.fetch_env) - remote_repo_path = GITALY_INTERNAL_URL - else - remote_repo_path = remote_repository.path - end - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repo_path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - update_branch(branch, user: user, newrev: rebase_sha, oldrev: branch_sha) - - rebase_sha - end - end - - def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=ar --binary #{diff_range}) - ).chomp - - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin| - stdin.binmode - stdin.write(diff) - end - - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) - - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp - end - end - def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -1968,22 +1774,6 @@ module Gitlab run_git(args, env: source_repository.fetch_env) end - def gitaly_ff_merge(user, source_sha, target_branch) - gitaly_operations_client.user_ff_branch(user, source_sha, target_branch) - rescue GRPC::FailedPrecondition => e - raise CommitError, e - end - - def rugged_ff_merge(user, source_sha, target_branch) - OperationService.new(user, self).with_branch(target_branch) do |our_commit| - raise ArgumentError, 'Invalid merge target' unless our_commit - - source_sha - end - rescue Rugged::ReferenceError, InvalidRef - raise ArgumentError, 'Invalid merge source' - end - def rugged_add_remote(remote_name, url, mirror_refmap) rugged.remotes.create(remote_name, url) @@ -2035,39 +1825,6 @@ module Gitlab remove_remote(remote_name) end - def rugged_multi_action( - user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository) - - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - index = Gitlab::Git::Index.new(self) - parents = [] - - if start_commit - index.read_tree(start_commit.rugged_commit.tree) - parents = [start_commit.sha] - end - - actions.each { |opts| index.apply(opts.delete(:action), opts) } - - committer = user_to_committer(user) - author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer - options = { - tree: index.write_tree, - message: message, - parents: parents, - author: author, - committer: committer - } - - create_commit(options) - end - end - def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index c9c414e5d33..d979ba0eb14 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -368,7 +368,7 @@ module Gitlab def call_commit_diff(request_params, options = {}) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) - request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params[:collapse_diffs] = !options.fetch(:expanded, true) request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index e9d4bb4c4b6..c04183a348f 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -64,6 +64,8 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + rescue GRPC::FailedPrecondition => ex + raise Gitlab::Git::Repository::InvalidRef, ex end def user_delete_branch(branch_name, user) @@ -133,6 +135,8 @@ module Gitlab request ).branch_update Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::CommitError, e end def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb new file mode 100644 index 00000000000..6c2af770119 --- /dev/null +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -0,0 +1,90 @@ +module Gitlab + module ImportExport + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `GroupProjectObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class GroupProjectObjectBuilder + def self.build(*args) + Project.transaction do + new(*args).find + end + end + + def initialize(klass, attributes) + @klass = klass < Label ? Label : klass + @attributes = attributes + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + find_object || @klass.create(project_attributes) + end + + private + + def find_object + @klass.where(where_clause).first + end + + def where_clause + @attributes.slice('title').map do |key, value| + scope_clause = table[:project_id].eq(@project.id) + scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group + + table[key].eq(value).and(scope_clause) + end.reduce(:or) + end + + def table + @table ||= @klass.arel_table + end + + def project_attributes + @attributes.except('group').tap do |atts| + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + end + end + + def label? + @klass == Label + end + + def milestone? + @klass == Milestone + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = @project.milestones.find_by(iid: @attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4eb67fbe11e..76b99b1de16 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,8 +1,8 @@ module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @@ -70,12 +70,23 @@ module Gitlab def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) + remove_group_models(relation_hash) if relation_hash.is_a?(Array) + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Remove project models that became group models as we found them at group level. + # This no longer required saving them at the root project level. + # For example, in the case of an existing group label that matched the title. + def remove_group_models(relation_hash) + relation_hash.reject! do |value| + GROUP_MODELS.include?(value.class) && value.group_id + end + end + def default_relation_list reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] @@ -170,7 +181,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), + relation_hash: relation_hash, members_mapper: members_mapper, user: @user, project: @restored_project, @@ -180,18 +191,6 @@ module Gitlab relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash, relation_type) - if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) - params = {} - params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] - params['project_id'] = restored_project.id if relation_hash['project_id'] - else - params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } - end - - relation_hash.merge(params) - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index c5cf290f191..091e81028bb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -54,6 +54,8 @@ module Gitlab @project = project @imported_object_retries = 0 + @relation_hash['project_id'] = @project.id + # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, @@ -80,15 +82,12 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when :project_label, :project_labels then setup_label - when :milestone, :milestones then setup_milestone when 'Ci::Pipeline' then setup_pipeline - else - @relation_hash['project_id'] = @project.id end update_user_references update_project_references + update_group_references remove_duplicate_assignees reset_tokens! @@ -151,39 +150,23 @@ module Gitlab end def update_project_references - project_id = @relation_hash.delete('project_id') - # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID end - # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id - @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] end def same_source_and_target? @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def setup_label - # If there's no group, move the label to a project label - if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] - @relation_hash['project_id'] = nil - @relation_name = :group_label - else - @relation_hash['group_id'] = nil - @relation_hash['type'] = 'ProjectLabel' - end - end + def update_group_references + return unless EXISTING_OBJECT_CHECK.include?(@relation_name) + return unless @relation_hash['group_id'] - def setup_milestone - if @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group.id - else - @relation_hash['project_id'] = @project.id - end + @relation_hash['group_id'] = @project.group&.id end def reset_tokens! @@ -271,15 +254,7 @@ module Gitlab end def existing_object - @existing_object ||= - begin - existing_object = find_or_create_object! - - # Done in two steps, as MySQL behaves differently than PostgreSQL using - # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update!(parsed_relation_hash) - existing_object - end + @existing_object ||= find_or_create_object! end def unknown_service? @@ -288,29 +263,16 @@ module Gitlab end def find_or_create_object! - finder_attributes = if @relation_name == :group_label - %w[title group_id] - elsif parsed_relation_hash['project_id'] - %w[title project_id] - else - %w[title group_id] - end - - finder_hash = parsed_relation_hash.slice(*finder_attributes) - - if label? - label = relation_class.find_or_initialize_by(finder_hash) - parsed_relation_hash.delete('priorities') if label.persisted? - - label.save! - label - else - relation_class.find_or_create_by(finder_hash) + return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + hash['group'] = @project.group if relation_class.attribute_method?('group_id') + hash['project'] = @project + hash.delete('project_id') end - end - def label? - @relation_name.to_s.include?('label') + GroupProjectObjectBuilder.build(relation_class, finder_hash) end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 45b644e6510..4a99b7cca5c 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -4,8 +4,18 @@ module Gitlab class Controller DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze APPLICATION_JSON = 'application/json'.freeze + APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + WHITELISTED_GIT_ROUTES = { + 'projects/git_http' => %w{git_upload_pack git_receive_pack} + }.freeze + + WHITELISTED_GIT_LFS_ROUTES = { + 'projects/lfs_api' => %w{batch}, + 'projects/lfs_locks_api' => %w{verify create unlock} + }.freeze + def initialize(app, env) @app = app @env = env @@ -36,7 +46,7 @@ module Gitlab end def json_request? - request.media_type == APPLICATION_JSON + APPLICATION_JSON_TYPES.include?(request.media_type) end def rack_flash @@ -63,22 +73,27 @@ module Gitlab grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - def grack_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') + return false unless + request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def lfs_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') + unless request.path.end_with?('/info/lfs/objects/batch', + '/info/lfs/locks', '/info/lfs/locks/verify') || + %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) + return false + end + + WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + end - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + def sidekiq_route + request.path.start_with?('/admin/sidekiq') end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 7f64a8c9e46..2ec871f0754 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -25,6 +25,11 @@ module Gitlab raise NotImplementedError end + # List of cached methods. Should be overridden by the including class + def cached_methods + raise NotImplementedError + end + # Caches the supplied block both in a cache and in an instance variable. # # The cache key and instance variable are named the same way as the value of @@ -67,6 +72,11 @@ module Gitlab # Expires the caches of a specific set of methods def expire_method_caches(methods) methods.each do |key| + unless cached_methods.include?(key.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + next + end + cache.expire(key) ivar = cache_instance_variable_name(key) |