diff options
Diffstat (limited to 'lib')
29 files changed, 426 insertions, 486 deletions
diff --git a/lib/api/files.rb b/lib/api/files.rb index 1598d3c00b8..29d7489bd7c 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,6 +5,8 @@ module API # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } + helpers ::API::Helpers::HeadersHelpers + helpers do def commit_params(attrs) { @@ -40,6 +42,20 @@ module API } end + def blob_data + { + file_name: @blob.name, + file_path: @blob.path, + size: @blob.size, + encoding: "base64", + content_sha256: Digest::SHA256.hexdigest(@blob.data), + ref: params[:ref], + blob_id: @blob.id, + commit_id: @commit.id, + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) + } + end + params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' @@ -61,6 +77,17 @@ module API requires :id, type: String, desc: 'The project ID' end resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do + desc 'Get raw file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' @@ -69,9 +96,22 @@ module API get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! + set_http_headers(blob_data) + send_git_blob @repo, @blob end + desc 'Get file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get a file from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' @@ -80,17 +120,11 @@ module API get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! - { - file_name: @blob.name, - file_path: @blob.path, - size: @blob.size, - encoding: "base64", - content: Base64.strict_encode64(@blob.data), - ref: params[:ref], - blob_id: @blob.id, - commit_id: @commit.id, - last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) - } + data = blob_data + + set_http_headers(data) + + data.merge(content: Base64.strict_encode64(@blob.data)) end desc 'Create new file in repository' diff --git a/lib/api/groups.rb b/lib/api/groups.rb index c7f41aba854..f633dd88d06 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -56,6 +56,8 @@ module API def find_group_projects(params) group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = reorder_projects(projects) paginate(projects) end @@ -191,6 +193,8 @@ module API desc: 'Return only the ID, URL, name, and path of each project' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' use :pagination use :with_custom_attributes diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb new file mode 100644 index 00000000000..cde51fccc62 --- /dev/null +++ b/lib/api/helpers/headers_helpers.rb @@ -0,0 +1,11 @@ +module API + module Helpers + module HeadersHelpers + def set_http_headers(header_data) + header_data.each do |key, value| + header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value + end + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index af7d2471b34..0f46bc4c98e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,8 +72,8 @@ module API end params :merge_requests_params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' + optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', + desc: 'Return opened, closed, locked, merged, or all merge requests' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3ef3680c5d9..b83da00502d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -459,6 +459,23 @@ module API conflict!(error.message) end end + + desc 'Transfer a project to a new namespace' + params do + requires :namespace, type: String, desc: 'The ID or path of the new namespace' + end + put ":id/transfer" do + authorize! :change_namespace, user_project + + namespace = find_namespace!(params[:namespace]) + result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) + + if result + present user_project, with: Entities::Project + else + render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) + end + end end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index bb3fa99af38..33a9646ac3b 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -100,9 +100,10 @@ module API params do requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare' do - compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) + compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to], straight: params[:straight]) present compare, with: Entities::Compare end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index e1261e7bbbe..4eccd9d5ed5 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -3,10 +3,6 @@ module Banzai # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter - # - # Context options: - # :asset_root - # :asset_host class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index a1f24e8b093..0d9b874ef85 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -44,11 +44,7 @@ module Banzai def self.transform_context(context) context[:only_path] = true unless context.key?(:only_path) - context.merge( - # EmojiFilter - asset_host: Gitlab::Application.config.asset_host, - asset_root: Gitlab.config.gitlab.base_url - ) + context end end end diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 605e93022e7..2760211fee8 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -22,6 +22,18 @@ module Gitaly server_version == Gitlab::GitalyClient.expected_server_version end + def read_writeable? + readable? && writeable? + end + + def readable? + storage_status&.readable + end + + def writeable? + storage_status&.writeable + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e @@ -30,13 +42,17 @@ module Gitaly private + def storage_status + @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage } + end + def info @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded # This will show the server as being out of date - Gitaly::ServerInfoResponse.new(git_version: '', server_version: '') + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end end end diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb new file mode 100644 index 00000000000..0b785e1b056 --- /dev/null +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class DeleteDiffFiles + def perform(merge_request_diff_id) + merge_request_diff = MergeRequestDiff.find_by(id: merge_request_diff_id) + + return unless merge_request_diff + return unless should_delete_diff_files?(merge_request_diff) + + MergeRequestDiff.transaction do + merge_request_diff.update_column(:state, 'without_files') + + # explain (analyze, buffers) when deleting 453 diff files: + # + # Delete on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actual time=43.265..43.265 rows=0 loops=1) + # Buffers: shared hit=2043 read=259 dirtied=254 + # -> Index Scan using index_merge_request_diff_files_on_mr_diff_id_and_order on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actu + # al time=0.466..26.317 rows=453 loops=1) + # Index Cond: (merge_request_diff_id = 463448) + # Buffers: shared hit=17 read=84 + # Planning time: 0.107 ms + # Execution time: 43.287 ms + # + MergeRequestDiffFile.where(merge_request_diff_id: merge_request_diff.id).delete_all + end + end + + private + + def should_delete_diff_files?(merge_request_diff) + return false if merge_request_diff.state == 'without_files' + + merge_request = merge_request_diff.merge_request + + return false unless merge_request.state == 'merged' + return false if merge_request_diff.id == merge_request.latest_merge_request_diff_id + + true + end + end + end +end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index d512fc58e46..4850a6c0430 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -38,7 +38,8 @@ module Gitlab # we only want to create full urls when there's a different asset_host # configured. def host - if Gitlab::Application.config.asset_host.nil? || Gitlab::Application.config.asset_host == Gitlab.config.gitlab.base_url + asset_host = ActionController::Base.asset_host + if asset_host.nil? || asset_host == Gitlab.config.gitlab.base_url nil else Gitlab.config.gitlab.base_url diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 341768752dc..c67826da1d2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -116,15 +116,9 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.between(base, head) - else - repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) } - end + repo.wrapped_gitaly_errors do + repo.gitaly_commit_client.between(base, head) end - rescue Rugged::ReferenceError - [] end # Returns commits collection @@ -149,56 +143,9 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) - Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| - if is_enabled - find_all_by_gitaly(repo, options) - else - find_all_by_rugged(repo, options) - end - end - end - - def find_all_by_rugged(repo, options = {}) - actual_options = options.dup - - allowed_options = [:ref, :max_count, :skip, :order] - - actual_options.keep_if do |key| - allowed_options.include?(key) - end - - default_options = { skip: 0 } - actual_options = default_options.merge(actual_options) - - rugged = repo.rugged - walker = Rugged::Walker.new(rugged) - - if actual_options[:ref] - walker.push(rugged.rev_parse_oid(actual_options[:ref])) - else - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end + repo.wrapped_gitaly_errors do + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end - - walker.sorting(rugged_sort_type(actual_options[:order])) - - commits = [] - offset = actual_options[:skip] - limit = actual_options[:max_count] - walker.each(offset: offset, limit: limit) do |commit| - commits.push(decorate(repo, commit)) - end - - walker.reset - - commits - rescue Rugged::OdbError - [] - end - - def find_all_by_gitaly(repo, options = {}) - Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end def decorate(repository, commit, ref = nil) @@ -220,19 +167,7 @@ module Gitlab end def shas_with_signatures(repository, shas) - GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) - else - shas.select do |sha| - begin - Rugged::Commit.extract_signature(repository.rugged, sha) - rescue Rugged::OdbError - false - end - end - end - end + Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) end # Only to be used when the object ids will not necessarily have a @@ -250,13 +185,7 @@ module Gitlab end def extract_signature(repository, commit_id) - repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.extract_signature(commit_id) - else - rugged_extract_signature(repository, commit_id) - end - end + repository.gitaly_commit_client.extract_signature(commit_id) end def extract_signature_lazily(repository, commit_id) @@ -276,36 +205,9 @@ module Gitlab end def batch_signature_extraction(repository, commit_ids) - repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled| - if is_enabled - gitaly_batch_signature_extraction(repository, commit_ids) - else - rugged_batch_signature_extraction(repository, commit_ids) - end - end - end - - def gitaly_batch_signature_extraction(repository, commit_ids) repository.gitaly_commit_client.get_commit_signatures(commit_ids) end - def rugged_batch_signature_extraction(repository, commit_ids) - commit_ids.each_with_object({}) do |commit_id, signatures| - signature_data = rugged_extract_signature(repository, commit_id) - next unless signature_data - - signatures[commit_id] = signature_data - end - end - - def rugged_extract_signature(repository, commit_id) - begin - Rugged::Commit.extract_signature(repository.rugged, commit_id) - rescue Rugged::OdbError - nil - end - end - def get_message(repository, commit_id) BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| items_by_repo = items.group_by { |i| i[:repository] } @@ -323,13 +225,7 @@ module Gitlab end def get_messages(repository, commit_ids) - repository.gitaly_migrate(:commit_messages) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.get_commit_messages(commit_ids) - else - commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h - end - end + repository.gitaly_commit_client.get_commit_messages(commit_ids) end end @@ -493,13 +389,18 @@ module Gitlab def tree_entry(path) return unless path.present? - @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| - if is_migrated - gitaly_tree_entry(path) - else - rugged_tree_entry(path) - end - end + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry end def to_gitaly_commit @@ -562,28 +463,6 @@ module Gitlab SERIALIZE_KEYS end - def gitaly_tree_entry(path) - # We're only interested in metadata, so limit actual data to 1 byte - # since Gitaly doesn't support "send no data" option. - entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) - return unless entry - - # To be compatible with the rugged format - entry = entry.to_h - entry.delete(:data) - entry[:name] = File.basename(path) - entry[:type] = entry[:type].downcase - - entry - end - - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - rescue Rugged::TreeError - nil - end - def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 88944cd62ea..7c3b91f6efb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -492,27 +492,6 @@ module Gitlab Ref.dereference_object(obj) end - # Return a collection of Rugged::Commits between the two revspec arguments. - # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for - # a detailed list of valid arguments. - # - # Gitaly note: JV: to be deprecated in favor of Commit.between - def rugged_commits_between(from, to) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) - - sha_from = sha_from_ref(from) - sha_to = sha_from_ref(to) - - walker.push(sha_to) - walker.hide(sha_from) - - commits = walker.to_a - walker.reset - - commits - end - # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to, options = {}) count_commits(from: from, to: to, **options) @@ -1316,16 +1295,7 @@ module Gitlab safe_query = Regexp.escape(query) ref ||= root_ref - gitaly_migrate(:search_files_by_content) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_content(ref, safe_query) - else - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) - - run_git(args).first.scrub.split(/^--\n/) - end - end + gitaly_repository_client.search_files_by_content(ref, safe_query) end def can_be_merged?(source_sha, target_branch) @@ -1342,15 +1312,7 @@ module Gitlab return [] if empty? || safe_query.blank? - gitaly_migrate(:search_files_by_name) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_name(ref, safe_query) - else - args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - - run_git(args).first.lines.map(&:strip) - end - end + gitaly_repository_client.search_files_by_name(ref, safe_query) end def find_commits_by_message(query, ref, path, limit, offset) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index e44284572fd..bbf2ecdb1fa 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -28,18 +28,7 @@ module Gitlab end def get_messages(repository, tag_ids) - repository.gitaly_migrate(:tag_messages) do |is_enabled| - if is_enabled - repository.gitaly_ref_client.get_tag_messages(tag_ids) - else - tag_ids.map do |id| - tag = repository.rugged.lookup(id) - message = tag.is_a?(Rugged::Commit) ? "" : tag.message - - [id, message] - end.to_h - end - end + repository.gitaly_ref_client.get_tag_messages(tag_ids) end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 7f2e6441f16..c9c414e5d33 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -76,6 +76,13 @@ module Gitlab end def tree_entry(ref, path, limit = nil) + if Pathname.new(path).cleanpath.to_s.start_with?('../') + # The TreeEntry RPC should return an empty reponse in this case but in + # Gitaly 0.107.0 and earlier we get an exception instead. This early return + # saves us a Gitaly roundtrip while also avoiding the exception. + return + end + request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: encode_binary(ref), @@ -317,6 +324,8 @@ module Gitlab return if signature.blank? && signed_text.blank? [signature, signed_text] + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_signatures(commit_ids) @@ -334,6 +343,8 @@ module Gitlab end signatures + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_messages(commit_ids) diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb new file mode 100644 index 00000000000..e3779995406 --- /dev/null +++ b/lib/gitlab/graphql/expose_permissions.rb @@ -0,0 +1,15 @@ +module Gitlab + module Graphql + module ExposePermissions + extend ActiveSupport::Concern + prepended do + def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource') + field :user_permissions, permission_type, + description: description, + null: false, + resolve: -> (obj, _, _) { obj } + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index 1688262974b..6f2b26c9676 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -10,9 +10,18 @@ module Gitlab old_resolver = field.resolve_proc resolve_with_presenter = -> (presented_type, args, context) do + # We need to wrap the original presentation type into a type that + # uses the presenter as an object. object = presented_type.object + + if object.is_a?(presented_in.presenter_class) + next old_resolver.call(presented_type, args, context) + end + presenter = presented_in.presenter_class.new(object, **context.to_h) - old_resolver.call(presenter, args, context) + wrapped = presented_type.class.new(presenter, context) + + old_resolver.call(wrapped, args, context) end field.redefine do diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb deleted file mode 100644 index 050fe7a5173..00000000000 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ /dev/null @@ -1,169 +0,0 @@ -module Gitlab - module HealthChecks - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1218 - class FsShardsCheck - extend BaseAbstractCheck - RANDOM_STRING = SecureRandom.hex(1000).freeze - COMMAND_TIMEOUT = '1'.freeze - TIMEOUT_EXECUTABLE = 'timeout'.freeze - - class << self - def readiness - repository_storages.map do |storage_name| - begin - if !storage_circuitbreaker_test(storage_name) - HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) - elsif !storage_stat_test(storage_name) - HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - else - with_temp_file(storage_name) do |tmp_file_path| - if !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) - else - HealthChecks::Result.new(true, nil, shard: storage_name) - end - end - end - rescue RuntimeError => ex - message = "unexpected error #{ex} when checking storage #{storage_name}" - Rails.logger.error(message) - HealthChecks::Result.new(false, message, shard: storage_name) - end - end - end - - def metrics - repository_storages.flat_map do |storage_name| - [ - storage_stat_metrics(storage_name), - storage_write_metrics(storage_name), - storage_read_metrics(storage_name), - storage_circuitbreaker_metrics(storage_name) - ].flatten - end - end - - private - - def operation_metrics(ok_metric, latency_metric, **labels) - result, elapsed = yield - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - rescue RuntimeError => ex - Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") - [metric(ok_metric, 0, **labels)] - end - - def repository_storages - storages_paths.keys - end - - def storages_paths - Gitlab.config.repositories.storages - end - - def exec_with_timeout(cmd_args, *args, &block) - Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) - end - - def with_temp_file(storage_name) - temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } - yield temp_file_path - ensure - delete_test_file(temp_file_path) - end - - def storage_path(storage_name) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - storages_paths[storage_name]&.legacy_disk_path - end - end - - # All below test methods use shell commands to perform actions on storage volumes. - # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, - # we can rely on shell commands to be terminated once `timeout` kills them. - # - # However we also fallback to pure Ruby file operations in case a specific shell command is missing - # so we are still able to perform healthchecks and gather metrics from such system. - - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status.zero? - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT - end - - def storage_stat_test(storage_name) - stat_path = File.join(storage_path(storage_name), '.') - begin - _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status.zero? - rescue Errno::ENOENT - File.exist?(stat_path) && File::Stat.new(stat_path).readable? - end - end - - def storage_write_test(tmp_path) - _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT - written_bytes == RANDOM_STRING.length - end - - def storage_read_test(tmp_path) - _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - file_contents = File.read(tmp_path) rescue Errno::ENOENT - file_contents == RANDOM_STRING - end - - def storage_circuitbreaker_test(storage_name) - Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" } - rescue Gitlab::Git::Storage::Inaccessible - nil - end - - def storage_stat_metrics(storage_name) - operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do - with_timing { storage_stat_test(storage_name) } - end - end - - def storage_write_metrics(storage_name) - operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - with_timing { storage_write_test(tmp_file_path) } - end - end - end - - def storage_read_metrics(storage_name) - operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - storage_write_test(tmp_file_path) # writes data used by read test - with_timing { storage_read_test(tmp_file_path) } - end - end - end - - def storage_circuitbreaker_metrics(storage_name) - operation_metrics(:filesystem_circuitbreaker, - :filesystem_circuitbreaker_latency_seconds, - shard: storage_name) do - with_timing { storage_circuitbreaker_test(storage_name) } - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index 11416c002e3..1f623e0b6ec 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -13,14 +13,14 @@ module Gitlab end def metrics - repository_storages.flat_map do |storage_name| - result, elapsed = with_timing { check(storage_name) } - labels = { shard: storage_name } + Gitaly::Server.all.flat_map do |server| + result, elapsed = with_timing { server.read_writeable? } + labels = { shard: server.storage } [ - metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels), + metric("#{metric_prefix}_success", result ? 1 : 0, **labels), metric("#{metric_prefix}_latency_seconds", elapsed, **labels) - ].flatten + ] end end @@ -36,10 +36,6 @@ module Gitlab METRIC_PREFIX end - def successful?(result) - result[:success] - end - def repository_storages storages.keys end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b713fa7e1cd..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.3'.freeze + VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb new file mode 100644 index 00000000000..6c2af770119 --- /dev/null +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -0,0 +1,90 @@ +module Gitlab + module ImportExport + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `GroupProjectObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class GroupProjectObjectBuilder + def self.build(*args) + Project.transaction do + new(*args).find + end + end + + def initialize(klass, attributes) + @klass = klass < Label ? Label : klass + @attributes = attributes + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + find_object || @klass.create(project_attributes) + end + + private + + def find_object + @klass.where(where_clause).first + end + + def where_clause + @attributes.slice('title').map do |key, value| + scope_clause = table[:project_id].eq(@project.id) + scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group + + table[key].eq(value).and(scope_clause) + end.reduce(:or) + end + + def table + @table ||= @klass.arel_table + end + + def project_attributes + @attributes.except('group').tap do |atts| + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + end + end + + def label? + @klass == Label + end + + def milestone? + @klass == Milestone + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = @project.milestones.find_by(iid: @attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4eb67fbe11e..76b99b1de16 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,8 +1,8 @@ module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @@ -70,12 +70,23 @@ module Gitlab def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) + remove_group_models(relation_hash) if relation_hash.is_a?(Array) + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Remove project models that became group models as we found them at group level. + # This no longer required saving them at the root project level. + # For example, in the case of an existing group label that matched the title. + def remove_group_models(relation_hash) + relation_hash.reject! do |value| + GROUP_MODELS.include?(value.class) && value.group_id + end + end + def default_relation_list reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] @@ -170,7 +181,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), + relation_hash: relation_hash, members_mapper: members_mapper, user: @user, project: @restored_project, @@ -180,18 +191,6 @@ module Gitlab relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash, relation_type) - if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) - params = {} - params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] - params['project_id'] = restored_project.id if relation_hash['project_id'] - else - params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } - end - - relation_hash.merge(params) - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index c5cf290f191..091e81028bb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -54,6 +54,8 @@ module Gitlab @project = project @imported_object_retries = 0 + @relation_hash['project_id'] = @project.id + # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, @@ -80,15 +82,12 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when :project_label, :project_labels then setup_label - when :milestone, :milestones then setup_milestone when 'Ci::Pipeline' then setup_pipeline - else - @relation_hash['project_id'] = @project.id end update_user_references update_project_references + update_group_references remove_duplicate_assignees reset_tokens! @@ -151,39 +150,23 @@ module Gitlab end def update_project_references - project_id = @relation_hash.delete('project_id') - # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID end - # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id - @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] end def same_source_and_target? @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def setup_label - # If there's no group, move the label to a project label - if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] - @relation_hash['project_id'] = nil - @relation_name = :group_label - else - @relation_hash['group_id'] = nil - @relation_hash['type'] = 'ProjectLabel' - end - end + def update_group_references + return unless EXISTING_OBJECT_CHECK.include?(@relation_name) + return unless @relation_hash['group_id'] - def setup_milestone - if @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group.id - else - @relation_hash['project_id'] = @project.id - end + @relation_hash['group_id'] = @project.group&.id end def reset_tokens! @@ -271,15 +254,7 @@ module Gitlab end def existing_object - @existing_object ||= - begin - existing_object = find_or_create_object! - - # Done in two steps, as MySQL behaves differently than PostgreSQL using - # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update!(parsed_relation_hash) - existing_object - end + @existing_object ||= find_or_create_object! end def unknown_service? @@ -288,29 +263,16 @@ module Gitlab end def find_or_create_object! - finder_attributes = if @relation_name == :group_label - %w[title group_id] - elsif parsed_relation_hash['project_id'] - %w[title project_id] - else - %w[title group_id] - end - - finder_hash = parsed_relation_hash.slice(*finder_attributes) - - if label? - label = relation_class.find_or_initialize_by(finder_hash) - parsed_relation_hash.delete('priorities') if label.persisted? - - label.save! - label - else - relation_class.find_or_create_by(finder_hash) + return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + hash['group'] = @project.group if relation_class.attribute_method?('group_id') + hash['project'] = @project + hash.delete('project_id') end - end - def label? - @relation_name.to_s.include?('label') + GroupProjectObjectBuilder.build(relation_class, finder_hash) end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 45b644e6510..4a99b7cca5c 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -4,8 +4,18 @@ module Gitlab class Controller DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze APPLICATION_JSON = 'application/json'.freeze + APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + WHITELISTED_GIT_ROUTES = { + 'projects/git_http' => %w{git_upload_pack git_receive_pack} + }.freeze + + WHITELISTED_GIT_LFS_ROUTES = { + 'projects/lfs_api' => %w{batch}, + 'projects/lfs_locks_api' => %w{verify create unlock} + }.freeze + def initialize(app, env) @app = app @env = env @@ -36,7 +46,7 @@ module Gitlab end def json_request? - request.media_type == APPLICATION_JSON + APPLICATION_JSON_TYPES.include?(request.media_type) end def rack_flash @@ -63,22 +73,27 @@ module Gitlab grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - def grack_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') + return false unless + request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def lfs_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') + unless request.path.end_with?('/info/lfs/objects/batch', + '/info/lfs/locks', '/info/lfs/locks/verify') || + %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) + return false + end + + WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + end - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + def sidekiq_route + request.path.start_with?('/admin/sidekiq') end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 7f64a8c9e46..2ec871f0754 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -25,6 +25,11 @@ module Gitlab raise NotImplementedError end + # List of cached methods. Should be overridden by the including class + def cached_methods + raise NotImplementedError + end + # Caches the supplied block both in a cache and in an instance variable. # # The cache key and instance variable are named the same way as the value of @@ -67,6 +72,11 @@ module Gitlab # Expires the caches of a specific set of methods def expire_method_caches(methods) methods.each do |key| + unless cached_methods.include?(key.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + next + end + cache.expire(key) ivar = cache_instance_variable_name(key) diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb new file mode 100644 index 00000000000..3f03f46d4b1 --- /dev/null +++ b/lib/gitlab/shard_health_cache.rb @@ -0,0 +1,41 @@ +module Gitlab + class ShardHealthCache + HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards'.freeze + HEALTHY_SHARDS_TIMEOUT = 300 + + # Clears the Redis set storing the list of healthy shards + def self.clear + Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) } + end + + # Updates the list of healthy shards using a Redis set + # + # shards - An array of shard names to store + def self.update(shards) + Gitlab::Redis::Cache.with do |redis| + redis.multi do |m| + m.del(HEALTHY_SHARDS_KEY) + shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) } + m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT) + end + end + end + + # Returns an array of strings of healthy shards + def self.cached_healthy_shards + Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } + end + + # Checks whether the given shard name is in the list of healthy shards. + # + # shard_name - The string to check + def self.healthy_shard?(shard_name) + Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } + end + + # Returns the number of healthy shards in the Redis set + def self.healthy_shard_count + Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 1a817eb735b..81f7cd3ffe8 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -15,7 +15,7 @@ module Gitlab if @resource ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." else - ":sweat_smile: Couldn't identify you, nor can I autorize you!" + ":sweat_smile: Couldn't identify you, nor can I authorize you!" end ephemeral_response(text: message) diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 139ab70e125..69166851816 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -46,7 +46,9 @@ namespace :gitlab do desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' task configure: :environment do - if ActiveRecord::Base.connection.tables.any? + # Check if we have existing db tables + # The schema_migrations table will still exist if drop_tables was called + if ActiveRecord::Base.connection.tables.count > 1 Rake::Task['db:migrate'].invoke else Rake::Task['db:schema:load'].invoke diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 44074397c05..900dbf7be24 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -10,15 +10,22 @@ namespace :gitlab do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end - desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' - task bump_test_version: :environment do - Dir.mktmpdir do |tmp_dir| - system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") - File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') - system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' + task bump_version: :environment do + archives = Dir['vendor/project_templates/*.tar.gz'] + archives.push('spec/features/projects/import_export/test_project_export.tar.gz') + + archives.each do |archive| + raise ArgumentError unless File.exist?(archive) + + Dir.mktmpdir do |tmp_dir| + system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null") + end end - puts "Updated to #{Gitlab::ImportExport.version}" + puts "Updated #{archives} to #{Gitlab::ImportExport.version}." end end end |