diff options
Diffstat (limited to 'lib')
26 files changed, 465 insertions, 238 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 1794207e29b..13cfba728fa 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -16,6 +16,10 @@ module API render_api_error!('The branch refname is invalid', 400) end end + + params :filter_params do + optional :search, type: String, desc: 'Return list of branches matching the search criteria' + end end params do @@ -27,15 +31,23 @@ module API end params do use :pagination + use :filter_params end get ':id/repository/branches' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') repository = user_project.repository - branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) + + branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - present paginate(branches), with: Entities::Branch, project: user_project, merged_branch_names: merged_branch_names + present( + paginate(::Kaminari.paginate_array(branches)), + with: Entities::Branch, + project: user_project, + merged_branch_names: merged_branch_names + ) end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b6c278c89d0..f74b3b26802 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -32,6 +32,8 @@ module API optional :search, type: String, desc: 'Search issues for text present in the title or description' optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me all], diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 16d0f005f21..8c02972b421 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -42,6 +42,8 @@ module API optional :labels, type: String, desc: 'Comma-separated list of label names' optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time' optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time' + optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time' + optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index b8d2673c1a6..75b64ae9af2 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -25,8 +25,8 @@ module Banzai # period or comma for punctuation without those characters being included # in the generated link. # - # Rubular: http://rubular.com/r/cxjPyZc7Sb - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://\S+)(?<!,|\.)} + # Rubular: http://rubular.com/r/JzPhi6DCZp + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!,|\.)} # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set @@ -35,53 +35,19 @@ module Banzai TEXT_QUERY = %Q(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., '://') - and not(starts-with(., 'http')) - and not(starts-with(., 'ftp')) ]).freeze + PUNCTUATION_PAIRS = { + "'" => "'", + '"' => '"', + ')' => '(', + ']' => '[', + '}' => '{' + }.freeze + def call return doc if context[:autolink] == false - rinku_parse - text_parse - end - - private - - # Run the text through Rinku as a first pass - # - # This will quickly autolink http(s) and ftp links. - # - # `@doc` will be re-parsed with the HTML String from Rinku. - def rinku_parse - # Convert the options from a Hash to a String that Rinku expects - options = tag_options(link_options) - - # NOTE: We don't parse email links because it will erroneously match - # external Commit and CommitRange references. - # - # The final argument tells Rinku to link short URLs that don't include a - # period (e.g., http://localhost:3000/) - rinku = Rinku.auto_link(html, :urls, options, IGNORE_PARENTS.to_a, 1) - - return if rinku == html - - # Rinku returns a String, so parse it back to a Nokogiri::XML::Document - # for further processing. - @doc = parse_html(rinku) - end - - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - - # Autolinks any text matching LINK_PATTERN that Rinku didn't already - # replace - def text_parse doc.xpath(TEXT_QUERY).each do |node| content = node.to_html @@ -97,6 +63,16 @@ module Banzai doc end + private + + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + def autolink_match(match) # start by stripping out dangerous links begin @@ -112,12 +88,30 @@ module Banzai match.gsub!(/((?:&[\w#]+;)+)\z/, '') dropped = ($1 || '').html_safe + # To match the behaviour of Rinku, if the matched link ends with a + # closing part of a matched pair of punctuation, we remove that trailing + # character unless there are an equal number of closing and opening + # characters in the link. + if match.end_with?(*PUNCTUATION_PAIRS.keys) + close_character = match[-1] + close_count = match.count(close_character) + open_character = PUNCTUATION_PAIRS[close_character] + open_count = match.count(open_character) + + if open_count != close_count || open_character == close_character + dropped += close_character + match = match[0..-2] + end + end + options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + content_tag(:a, match.html_safe, options) + dropped end def autolink_filter(text) - text.gsub(LINK_PATTERN) { |match| autolink_match(match) } + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + autolink_match(link) + end end def link_options diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d19a2519803..d5e17a123df 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -17,27 +17,11 @@ module Gitlab end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") - ensure - if pipeline.builds.where(stage_id: nil).any? - invalid_builds_counter.increment(node: hostname) - end end def break? !pipeline.persisted? end - - private - - def invalid_builds_counter - @counter ||= Gitlab::Metrics - .counter(:gitlab_ci_invalid_builds_total, - 'Invalid builds without stage assigned counter') - end - - def hostname - @hostname ||= Socket.gethostname - end end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9576d5a3fd8..02d3763514e 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -23,7 +23,7 @@ module Gitlab mr_events = event_counts(date_from, :merge_requests) .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") note_events = event_counts(date_from, :merge_requests) - .having(action: [Event::COMMENTED], target_type: "Note") + .having(action: [Event::COMMENTED], target_type: %w(Note DiffNote)) union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b2fca2c16de..eabcf46cf58 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -238,9 +238,9 @@ module Gitlab self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end - @loaded_all_data = false # Retain the actual size before it is encoded @loaded_size = @data.bytesize if @data + @loaded_all_data = @loaded_size == size end def binary? @@ -255,10 +255,15 @@ module Gitlab # memory as a Ruby string. def load_all_data!(repository) return if @data == '' # don't mess with submodule blobs - return @data if @loaded_all_data - Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| - @data = begin + # Even if we return early, recalculate wether this blob is binary in + # case a blob was initialized as text but the full data isn't + @binary = nil + + return if @loaded_all_data + + @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| + begin if is_enabled repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data else @@ -269,7 +274,6 @@ module Gitlab @loaded_all_data = true @loaded_size = @data.bytesize - @binary = nil end def name diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 48434047fce..b9e5cf258f4 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,6 +7,28 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) + @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) + else + git_new_pointers(object_limit, not_in) + end + end + end + + def all_pointers + @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| + if is_enabled + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) + else + git_all_pointers + end + end + end + + private + + def git_new_pointers(object_limit, not_in) @new_pointers ||= begin rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| object_ids = object_ids.take(object_limit) if object_limit @@ -16,14 +38,12 @@ module Gitlab end end - def all_pointers + def git_all_pointers rev_list.all_objects(require_path: true) do |object_ids| Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) end end - private - def rev_list Gitlab::Git::RevList.new(@repository, newrev: @newrev) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7c373ccd6f..21c79a7a550 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -479,9 +479,8 @@ module Gitlab raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") end - # TODO support options[:all] in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/1049 gitaly_migrate(:find_commits) do |is_enabled| - if is_enabled && !options[:all] + if is_enabled gitaly_commit_client.find_commits(options) else raw_log(options).map { |c| Commit.decorate(self, c) } @@ -508,9 +507,8 @@ module Gitlab def count_commits(options) count_commits_options = process_count_commits_options(options) - # TODO add support for options[:all] in Gitaly https://gitlab.com/gitlab-org/gitaly/issues/1050 gitaly_migrate(:count_commits) do |is_enabled| - if is_enabled && !options[:all] + if is_enabled count_commits_by_gitaly(count_commits_options) else count_commits_by_shelling_out(count_commits_options) diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index dfa0fa43b0f..28554208984 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -45,16 +45,7 @@ module Gitlab response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) - response.flat_map do |message| - message.lfs_pointers.map do |lfs_pointer| - Gitlab::Git::Blob.new( - id: lfs_pointer.oid, - size: lfs_pointer.size, - data: lfs_pointer.data, - binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) - ) - end - end + map_lfs_pointers(response) end def get_blobs(revision_paths, limit = -1) @@ -80,6 +71,50 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end + + def get_new_lfs_pointers(revision, limit, not_in) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + limit: limit || 0 + ) + + if not_in.nil? || not_in == :all + request.not_in_all = true + else + request.not_in_refs += not_in + end + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request) + + map_lfs_pointers(response) + end + + def get_all_lfs_pointers(revision) + request = Gitaly::GetNewLFSPointersRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request) + + map_lfs_pointers(response) + end + + private + + def map_lfs_pointers(response) + response.flat_map do |message| + message.lfs_pointers.map do |lfs_pointer| + Gitlab::Git::Blob.new( + id: lfs_pointer.oid, + size: lfs_pointer.size, + data: lfs_pointer.data, + binary: Gitlab::Git::Blob.binary?(lfs_pointer.data) + ) + end + end + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1ad0bf1d060..456a8a1a2d6 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -134,7 +134,8 @@ module Gitlab def commit_count(ref, options = {}) request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, - revision: encode_binary(ref) + revision: encode_binary(ref), + all: !!options[:all] ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? @@ -269,6 +270,7 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], + all: !!options[:all], disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 92f0e0402a8..a7e055ac444 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,6 +20,7 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.test_env = Rails.env.test? + gon.suggested_label_colors = LabelsHelper.suggested_colors if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index c26656704d7..d9d5f90596f 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -1,90 +1,19 @@ module Gitlab module Middleware class ReadOnly - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze - APPLICATION_JSON = 'application/json'.freeze API_VERSIONS = (3..4) + def self.internal_routes + @internal_routes ||= + API_VERSIONS.map { |version| "api/v#{version}/internal" } + end + def initialize(app) @app = app - @whitelisted = internal_routes end def call(env) - @env = env - @route_hash = nil - - if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') - error_message = 'You cannot do writing operations on a read-only GitLab instance' - - if json_request? - return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] - else - rack_flash.alert = error_message - rack_session['flash'] = rack_flash.to_session_value - - return [301, { 'Location' => last_visited_url }, []] - end - end - - @app.call(env) - end - - private - - def internal_routes - API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } - end - - def disallowed_request? - DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes - end - - def json_request? - request.media_type == APPLICATION_JSON - end - - def rack_flash - @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) - end - - def rack_session - @env['rack.session'] - end - - def request - @env['rack.request'] ||= Rack::Request.new(@env) - end - - def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url - end - - def route_hash - @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} - end - - def whitelisted_routes - grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - - def grack_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') - - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' - end - - def lfs_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') - - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + ReadOnly::Controller.new(@app, env).call end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb new file mode 100644 index 00000000000..45b644e6510 --- /dev/null +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -0,0 +1,86 @@ +module Gitlab + module Middleware + class ReadOnly + class Controller + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + + def initialize(app, env) + @app = app + @env = env + end + + def call + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + + if json_request? + return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] + else + rack_flash.alert = ERROR_MESSAGE + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(@env) + end + + private + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && + !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + end + + def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + end + end + end + end +end diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb new file mode 100644 index 00000000000..f8d0a135965 --- /dev/null +++ b/lib/gitlab/middleware/release_env.rb @@ -0,0 +1,14 @@ +module Gitlab + module Middleware + # Some of middleware would hold env for no good reason even after the + # request had already been processed, and we could not garbage collect + # them due to this. Put this middleware as the first middleware so that + # it would clear the env after the request is done, allowing GC gets a + # chance to release memory for the last request. + ReleaseEnv = Struct.new(:app) do + def call(env) + app.call(env).tap { env.clear } + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index cf0935dbd9a..29277ec6481 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,8 +29,18 @@ module Gitlab @blobs_count ||= blobs.count end - def notes_count - @notes_count ||= notes.count + def limited_notes_count + return @limited_notes_count if defined?(@limited_notes_count) + + types = %w(issue merge_request commit snippet) + @limited_notes_count = 0 + + types.each do |type| + @limited_notes_count += notes_finder(type).limit(count_limit).count + break if @limited_notes_count >= count_limit + end + + @limited_notes_count end def wiki_blobs_count @@ -72,11 +82,12 @@ module Gitlab end def single_commit_result? - commits_count == 1 && total_result_count == 1 - end + return false if commits_count != 1 - def total_result_count - issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + counts = %i(limited_milestones_count limited_notes_count + limited_merge_requests_count limited_issues_count + blobs_count wiki_blobs_count) + counts.all? { |count_method| public_send(count_method).zero? } # rubocop:disable GitlabSecurity/PublicSend end private @@ -106,7 +117,11 @@ module Gitlab end def notes - @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') + @notes ||= notes_finder(nil) + end + + def notes_finder(type) + NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 781783f4d97..757ef71b95a 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -62,22 +62,6 @@ module Gitlab without_count ? collection.without_count : collection end - def projects_count - @projects_count ||= projects.count - end - - def issues_count - @issues_count ||= issues.count - end - - def merge_requests_count - @merge_requests_count ||= merge_requests.count - end - - def milestones_count - @milestones_count ||= milestones.count - end - def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index f9faa134206..c6ad997a4d4 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -14,7 +14,7 @@ module Gitlab end def mark(marker_ranges) - return rich_line unless marker_ranges + return rich_line unless marker_ranges&.any? if html_escaped rich_marker_ranges = [] diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index 7ebf1c0428c..b19aa6dea35 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,13 +1,15 @@ module Gitlab class StringRegexMarker < StringRangeMarker def mark(regex, group: 0, &block) - regex_match = raw_line.match(regex) - return rich_line unless regex_match + ranges = [] - begin_index, end_index = regex_match.offset(group) - name_range = begin_index..(end_index - 1) + raw_line.scan(regex) do + begin_index, end_index = Regexp.last_match.offset(group) - super([name_range], &block) + ranges << (begin_index..(end_index - 1)) + end + + super(ranges, &block) end end end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb new file mode 100644 index 00000000000..1ef369a4b67 --- /dev/null +++ b/lib/gitlab/verify/batch_verifier.rb @@ -0,0 +1,64 @@ +module Gitlab + module Verify + class BatchVerifier + attr_reader :batch_size, :start, :finish + + def initialize(batch_size:, start: nil, finish: nil) + @batch_size = batch_size + @start = start + @finish = finish + end + + # Yields a Range of IDs and a Hash of failed verifications (object => error) + def run_batches(&blk) + relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches + range = relation.first.id..relation.last.id + failures = run_batch(relation) + + yield(range, failures) + end + end + + def name + raise NotImplementedError.new + end + + def describe(_object) + raise NotImplementedError.new + end + + private + + def run_batch(relation) + relation.map { |upload| verify(upload) }.compact.to_h + end + + def verify(object) + expected = expected_checksum(object) + actual = actual_checksum(object) + + raise 'Checksum missing' unless expected.present? + raise 'Checksum mismatch' unless expected == actual + + nil + rescue => err + [object, err] + end + + # This should return an ActiveRecord::Relation suitable for calling #in_batches on + def relation + raise NotImplementedError.new + end + + # The checksum we expect the object to have + def expected_checksum(_object) + raise NotImplementedError.new + end + + # The freshly-recalculated checksum of the object + def actual_checksum(_object) + raise NotImplementedError.new + end + end + end +end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb new file mode 100644 index 00000000000..fe51edbdeeb --- /dev/null +++ b/lib/gitlab/verify/lfs_objects.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class LfsObjects < BatchVerifier + def name + 'LFS objects' + end + + def describe(object) + "LFS object: #{object.oid}" + end + + private + + def relation + LfsObject.all + end + + def expected_checksum(lfs_object) + lfs_object.oid + end + + def actual_checksum(lfs_object) + LfsObject.calculate_oid(lfs_object.file.path) + end + end + end +end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb new file mode 100644 index 00000000000..dd138e6b92b --- /dev/null +++ b/lib/gitlab/verify/rake_task.rb @@ -0,0 +1,53 @@ +module Gitlab + module Verify + class RakeTask + def self.run!(verify_kls) + verifier = verify_kls.new( + batch_size: ENV.fetch('BATCH', 200).to_i, + start: ENV['ID_FROM'], + finish: ENV['ID_TO'] + ) + + verbose = Gitlab::Utils.to_boolean(ENV['VERBOSE']) + + new(verifier, verbose).run! + end + + attr_reader :verifier, :output + + def initialize(verifier, verbose) + @verifier = verifier + @verbose = verbose + end + + def run! + say "Checking integrity of #{verifier.name}" + + verifier.run_batches { |*args| run_batch(*args) } + + say 'Done!' + end + + def verbose? + !!@verbose + end + + private + + def say(text) + puts(text) # rubocop:disable Rails/Output + end + + def run_batch(range, failures) + status_color = failures.empty? ? :green : :red + say "- #{range}: Failures: #{failures.count}".color(status_color) + + return unless verbose? + + failures.each do |object, error| + say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + end + end + end + end +end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb new file mode 100644 index 00000000000..6972e517ea5 --- /dev/null +++ b/lib/gitlab/verify/uploads.rb @@ -0,0 +1,27 @@ +module Gitlab + module Verify + class Uploads < BatchVerifier + def name + 'Uploads' + end + + def describe(object) + "Upload: #{object.id}" + end + + private + + def relation + Upload.all + end + + def expected_checksum(upload) + upload.checksum + end + + def actual_checksum(upload) + Upload.hexdigest(upload.absolute_path) + end + end + end +end diff --git a/lib/tasks/gitlab/lfs/check.rake b/lib/tasks/gitlab/lfs/check.rake new file mode 100644 index 00000000000..869463d4e5d --- /dev/null +++ b/lib/tasks/gitlab/lfs/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :lfs do + desc 'GitLab | LFS | Check integrity of uploaded LFS objects' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::LfsObjects) + end + end +end diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake deleted file mode 100644 index df31567ce64..00000000000 --- a/lib/tasks/gitlab/uploads.rake +++ /dev/null @@ -1,44 +0,0 @@ -namespace :gitlab do - namespace :uploads do - desc 'GitLab | Uploads | Check integrity of uploaded files' - task check: :environment do - puts 'Checking integrity of uploaded files' - - uploads_batches do |batch| - batch.each do |upload| - puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green) - - if upload.exist? - check_checksum(upload) - else - puts " * File does not exist on the file system".color(:red) - end - end - end - - puts 'Done!' - end - - def batch_size - ENV.fetch('BATCH', 200).to_i - end - - def calculate_checksum(absolute_path) - Digest::SHA256.file(absolute_path).hexdigest - end - - def check_checksum(upload) - checksum = calculate_checksum(upload.absolute_path) - - if checksum != upload.checksum - puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red) - end - end - - def uploads_batches(&block) - Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches - yield relation - end - end - end -end diff --git a/lib/tasks/gitlab/uploads/check.rake b/lib/tasks/gitlab/uploads/check.rake new file mode 100644 index 00000000000..2be2ec7f9c9 --- /dev/null +++ b/lib/tasks/gitlab/uploads/check.rake @@ -0,0 +1,8 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Check integrity of uploaded files' + task check: :environment do + Gitlab::Verify::RakeTask.run!(Gitlab::Verify::Uploads) + end + end +end |