diff options
Diffstat (limited to 'lib')
129 files changed, 905 insertions, 279 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 18063fb20a2..cecff6d3b81 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class AccessRequests < Grape::API include PaginationParams diff --git a/lib/api/api.rb b/lib/api/api.rb index e89d9337853..06c8b48b8cc 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class API < Grape::API include APIGuard diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8ee7987cfff..61357b3f1d6 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Guard API with OAuth 2.0 Access Token require 'rack/oauth2' diff --git a/lib/api/applications.rb b/lib/api/applications.rb index b122cdefe4e..f29cd7fc003 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # External applications API class Applications < Grape::API diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index 70219bc8ea0..0f14d003065 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Avatar < Grape::API resource :avatar do diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index e334af22183..c2abf9155f3 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class AwardEmoji < Grape::API include PaginationParams diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 8ceffe9c5ef..ab670988f47 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Badges < Grape::API include PaginationParams diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 0f89414148b..c80e1c57864 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Boards < Grape::API include BoardsResponses diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 3322b37c6ff..86d9b24802f 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module BoardsResponses extend ActiveSupport::Concern diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 5d106ed93a0..2735d410c8e 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index d7138b2f2fe..19148758fc5 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class BroadcastMessages < Grape::API include PaginationParams diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index c13154dc0ec..6eddc5e5b61 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class CircuitBreakers < Grape::API before { authenticated_as_admin! } diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 8e6f706afd4..99553d993ca 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API diff --git a/lib/api/commits.rb b/lib/api/commits.rb index fcaff35459e..f0db1318146 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -73,7 +75,26 @@ module API params do requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + requires :actions, type: Array, desc: 'Actions to perform in commit' do + requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze + requires :file_path, type: String, desc: 'Full path to the file. Ex. `lib/class.rb`' + given action: ->(action) { action == 'move' } do + requires :previous_path, type: String, desc: 'Original full path to the file being moved. Ex. `lib/class1.rb`' + end + given action: ->(action) { %w[create move].include? action } do + optional :content, type: String, desc: 'File content' + end + given action: ->(action) { action == 'update' } do + requires :content, type: String, desc: 'File content' + end + optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64] + given action: ->(action) { %w[update move delete].include? action } do + optional :last_commit_id, type: String, desc: 'Last known file commit id' + end + given action: ->(action) { action == 'chmod' } do + requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' + end + end optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb index b5864665cc3..2149e04451e 100644 --- a/lib/api/custom_attributes_endpoints.rb +++ b/lib/api/custom_attributes_endpoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module CustomAttributesEndpoints extend ActiveSupport::Concern diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 501e9f64db0..ce35720d408 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class DeployKeys < Grape::API include PaginationParams diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index b7892599295..6747e2e5005 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Deployments RESTful API endpoints class Deployments < Grape::API diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 88668992215..39c6d28391d 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Discussions < Grape::API include PaginationParams diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0fec3dc3dc4..a78a93cbfd9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Entities class WikiPageBasic < Grape::Entity @@ -53,7 +55,7 @@ module API class User < UserBasic expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization end class UserActivity < Grape::Entity diff --git a/lib/api/environments.rb b/lib/api/environments.rb index fa828f43001..c64217a6977 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Environments RESTfull API endpoints class Environments < Grape::API diff --git a/lib/api/events.rb b/lib/api/events.rb index dfe0e81af26..6e0b508be19 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Events < Grape::API include PaginationParams @@ -16,12 +18,27 @@ module API desc: 'Return events sorted in ascending and descending order' end + RedactedEvent = OpenStruct.new(target_title: 'Confidential event').freeze + + def redact_events(events) + events.map do |event| + if event.visible_to_user?(current_user) + event + else + RedactedEvent + end + end + end + # rubocop: disable CodeReuse/ActiveRecord - def present_events(events) + def present_events(events, redact: true) events = events.reorder(created_at: params[:sort]) .with_associations - present paginate(events), with: Entities::Event + events = paginate(events) + events = redact_events(events) if redact + + present events, with: Entities::Event end # rubocop: enable CodeReuse/ActiveRecord end @@ -44,7 +61,8 @@ module API events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) - present_events(events) + # Since we're viewing our own events, redaction is unnecessary + present_events(events, redact: false) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/features.rb b/lib/api/features.rb index 79be8c1903e..6f2422af13a 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Features < Grape::API before { authenticated_as_admin! } diff --git a/lib/api/files.rb b/lib/api/files.rb index ac02488d30c..bcd2cd48a45 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Files < Grape::API FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 3832cdc10a8..dc30e868e2e 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupBoards < Grape::API include BoardsResponses diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 4b4352c2b27..b36436dbf43 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupMilestones < Grape::API include MilestoneResponses diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index b6610dd04b3..ae7241e9a30 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupVariables < Grape::API include PaginationParams diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 018ca72c32a..64b998ab455 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Groups < Grape::API include PaginationParams diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 85e3e06e4fd..a7ba8066233 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers include Gitlab::Utils @@ -381,9 +383,10 @@ module API # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 trace = exception.backtrace - message = "\n#{exception.class} (#{exception.message}):\n" + message = ["\n#{exception.class} (#{exception.message}):\n"] message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) message << " " << trace.join("\n ") + message = message.join API.logger.add Logger::FATAL, message diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb index 1f8afbf3c90..46ce5b4e7b5 100644 --- a/lib/api/helpers/badges_helpers.rb +++ b/lib/api/helpers/badges_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module BadgesHelpers diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 9993caa5249..7551ca50a7f 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CommonHelpers diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb index 3bbe827967e..88208226c40 100644 --- a/lib/api/helpers/custom_attributes.rb +++ b/lib/api/helpers/custom_attributes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CustomAttributes diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index dd4f6c41131..23b1cd1ad45 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CustomValidators diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb index c9c44e3c218..7553af9d156 100644 --- a/lib/api/helpers/headers_helpers.rb +++ b/lib/api/helpers/headers_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module HeadersHelpers diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 83151be82ad..4eaaca96b49 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module InternalHelpers diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 518aaa62aef..73d58ee7f37 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable GitlabSecurity/PublicSend module API diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 7b1f5c2584b..216b2c45741 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module NotesHelpers diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 50bcd4e0437..d311cbb5f7e 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module Pagination diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb index 94798a8cb51..1b5dc281e38 100644 --- a/lib/api/helpers/project_snapshots_helpers.rb +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module ProjectSnapshotsHelpers diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 98672f2f765..e6a72b949f9 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module ProjectsHelpers diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index bc7333ca4b3..793ae11b41d 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module RelatedResourcesHelpers diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 61eb88d3331..45d0343bc89 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module Runner diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 71b87f60bf6..6a264c4cc6d 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Internal access API class Internal < Grape::API diff --git a/lib/api/issues.rb b/lib/api/issues.rb index bcb03a0b540..25d78053c88 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Issues < Grape::API include PaginationParams diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index ab4203c4e25..2229cbcd9d4 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class JobArtifacts < Grape::API before { authenticate_non_get! } diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 27ffd42834c..63fab6b0abb 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Jobs < Grape::API include PaginationParams diff --git a/lib/api/keys.rb b/lib/api/keys.rb index fd93f797f72..d5280a0035d 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Keys API class Keys < Grape::API diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 98c9818db39..28555454307 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Labels < Grape::API include PaginationParams diff --git a/lib/api/lint.rb b/lib/api/lint.rb index d202eaa4c49..0342a4b6654 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Lint < Grape::API namespace :ci do diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index 5d55224c1a7..50d8a1ac596 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Markdown < Grape::API params do diff --git a/lib/api/members.rb b/lib/api/members.rb index 4d8e23dee91..a8f67be3463 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Members < Grape::API include PaginationParams diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 95ef8f42954..e4fb890960a 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # MergeRequestDiff API class MergeRequestDiffs < Grape::API diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index cad38271cbb..764905ca00f 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class MergeRequests < Grape::API include PaginationParams diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index a8eb137e46a..a0ca39b69d4 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module MilestoneResponses extend ActiveSupport::Concern diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 32b77aedba8..76639fbb031 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Namespaces < Grape::API include PaginationParams diff --git a/lib/api/notes.rb b/lib/api/notes.rb index dc9373bb3c2..9f323b87baf 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Notes < Grape::API include PaginationParams diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bf0d6b9e434..4d9a4629268 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # notification_settings API class NotificationSettings < Grape::API diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 8730c91b426..c9ad47e0f0d 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class PagesDomains < Grape::API include PaginationParams diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb index f566eb3ed2b..ae03595eb25 100644 --- a/lib/api/pagination_params.rb +++ b/lib/api/pagination_params.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Concern for declare pagination params. # diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 5bd1ce8c5e1..ed0a38b9d70 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class PipelineSchedules < Grape::API include PaginationParams diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 5cce96d5ae7..1cfb982c04b 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Pipelines < Grape::API include PaginationParams diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 8562ae6d737..e34ed0bdb44 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectExport < Grape::API before do diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 0fb454bc22e..4af4c6ac593 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectHooks < Grape::API include PaginationParams diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index bc5152e539f..cbfa0c5bc1c 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectImport < Grape::API include PaginationParams diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 72cf32d7717..c7137ba5217 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectMilestones < Grape::API include PaginationParams diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb index 71005acc587..175fbb2ce92 100644 --- a/lib/api/project_snapshots.rb +++ b/lib/api/project_snapshots.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectSnapshots < Grape::API helpers ::API::Helpers::ProjectSnapshotsHelpers diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 1ef176b1320..f3a1b73b153 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectSnippets < Grape::API include PaginationParams diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ee426f39523..00bad49ebdc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'declarative_policy' module API diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 9fd79c491c2..8edcfea7c93 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module ProjectsRelationBuilder extend ActiveSupport::Concern diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 804f6fa9b73..47752f40e58 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProtectedBranches < Grape::API include PaginationParams diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index e406344e42d..ed1c5f0cc05 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProtectedTags < Grape::API include PaginationParams diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 79736107bbb..5125f302fbb 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API diff --git a/lib/api/runner.rb b/lib/api/runner.rb index b2d46cef23c..d8768a54986 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Runner < Grape::API helpers ::API::Helpers::Runner diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 9bcdfc8cb15..60868821810 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Runners < Grape::API include PaginationParams diff --git a/lib/api/scope.rb b/lib/api/scope.rb index d5165b2e482..707775e5d15 100644 --- a/lib/api/scope.rb +++ b/lib/api/scope.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Encapsulate a scope used for authorization, such as `api`, or `read_user` module API class Scope diff --git a/lib/api/search.rb b/lib/api/search.rb index 37fbabe419c..12d97dcfe7f 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Search < Grape::API include PaginationParams diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 8d71bd9dff1..edbd134822c 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Settings < Grape::API before { authenticated_as_admin! } diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index 11f2b40269a..daa9598a204 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'sidekiq/api' module API diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 6352a9c8742..f1786c15f4f 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Snippets API class Snippets < Grape::API diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index b3e1e23031a..077e9373ac4 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Subscriptions < Grape::API before { authenticate! } diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 07552aa18e8..51fae0e54aa 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class SystemHooks < Grape::API include PaginationParams diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 5e0afc6a7e4..f739eacf9ba 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Tags < Grape::API include PaginationParams diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 7bf0e0f5934..8ff3b2ac33c 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Templates < Grape::API include PaginationParams diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 2bb451dea89..93fe06bec27 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module TimeTrackingEndpoints extend ActiveSupport::Concern diff --git a/lib/api/todos.rb b/lib/api/todos.rb index c6dbcf84e3a..ed2cf2cc31b 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Todos < Grape::API include PaginationParams diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 2339505b05b..f784c857883 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Triggers < Grape::API include PaginationParams diff --git a/lib/api/users.rb b/lib/api/users.rb index ac09ca7f7b7..11a7f4ef64d 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Users < Grape::API include PaginationParams @@ -42,12 +44,12 @@ module API optional :provider, type: String, desc: 'The external provider' optional :bio, type: String, desc: 'The biography of the user' optional :location, type: String, desc: 'The location of the user' + optional :public_email, type: String, desc: 'The public email of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' - optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' all_or_none_of :extern_uid, :provider end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 50e6fa6bcdf..c844ba321ed 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Variables < Grape::API include PaginationParams diff --git a/lib/api/version.rb b/lib/api/version.rb index 3b10bfa6a7d..74cd857f447 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Version < Grape::API before { authenticate! } diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index e86ebc573f2..6e1d4eb335f 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Wikis < Grape::API helpers do diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 45a935ab352..33658ae225f 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index adf85ca4719..5e795a449de 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 086ca5986bd..e6bf3d1856f 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'yaml' module Backup diff --git a/lib/backup/files.rb b/lib/backup/files.rb index e287aa1e392..0032ae8f84b 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'open3' require_relative 'helper' diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb index 54b9ce10b4d..22f00aef569 100644 --- a/lib/backup/helper.rb +++ b/lib/backup/helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Backup module Helper def access_denied_error(path) diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 185ff8ae6bd..0dfe56e214f 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index a3641505196..5d4a7efc456 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Backup class Manager ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 542e35a7c7c..a4be728df08 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 35821805797..d16ed2facf1 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 906ed498026..c8a5377bfa0 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'yaml' module Backup diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 49b117a7ee3..9577df2634a 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 8e838d04bad..7acbc933adc 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -60,7 +60,11 @@ module Banzai path_parts.unshift(relative_url_root, project.full_path) end - path = Addressable::URI.escape(File.join(*path_parts)) + begin + path = Addressable::URI.escape(File.join(*path_parts)) + rescue Addressable::URI::InvalidURIError + return + end html_attr.value = if context[:only_path] diff --git a/lib/event_filter.rb b/lib/event_filter.rb index f756a211a12..24fdcd6fbb1 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -1,76 +1,42 @@ -class EventFilter - attr_accessor :params - - class << self - def all - 'all' - end - - def push - 'push' - end - - def merged - 'merged' - end +# frozen_string_literal: true - def issue - 'issue' - end - - def comments - 'comments' - end - - def team - 'team' - end +class EventFilter + attr_accessor :filter + + ALL = 'all' + PUSH = 'push' + MERGED = 'merged' + ISSUE = 'issue' + COMMENTS = 'comments' + TEAM = 'team' + FILTERS = [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM].freeze + + def initialize(filter) + # Split using comma to maintain backward compatibility Ex/ "filter1,filter2" + filter = filter.to_s.split(',')[0].to_s + @filter = FILTERS.include?(filter) ? filter : ALL end - def initialize(params) - @params = if params - params.dup - else - [] # EventFilter.default_filter - end + def active?(key) + filter == key.to_s end # rubocop: disable CodeReuse/ActiveRecord def apply_filter(events) - return events if params.blank? || params == EventFilter.all - - case params - when EventFilter.push + case filter + when PUSH events.where(action: Event::PUSHED) - when EventFilter.merged + when MERGED events.where(action: Event::MERGED) - when EventFilter.comments + when COMMENTS events.where(action: Event::COMMENTED) - when EventFilter.team + when TEAM events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) - when EventFilter.issue + when ISSUE events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def options(key) - filter = params.dup - - if filter.include? key - filter.delete key else - filter << key - end - - filter - end - - def active?(key) - if params.present? - params.include? key - else - key == EventFilter.all + events end end + # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index ab6b609d099..7790534d5d7 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -47,4 +47,8 @@ module Gitlab def self.dev_env_or_com? Rails.env.development? || org? || com? end + + def self.pre_release? + VERSION.include?('pre') + end end diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb new file mode 100644 index 00000000000..0d333e47e7b --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # For this background migration to work, the table that is migrated _has_ to + # have an `id` column as the primary key. Additionally, the encrypted column + # should be managed by attr_encrypted, and map to an attribute with the same + # name as the unencrypted column (i.e., the unencrypted column should be + # shadowed). + # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + class EncryptColumns + def perform(model, attributes, from, to) + model = model.constantize if model.is_a?(String) + attributes = expand_attributes(model, Array(attributes).map(&:to_sym)) + + model.transaction do + # Use SELECT ... FOR UPDATE to prevent the value being changed while + # we are encrypting it + relation = model.where(id: from..to).lock + + relation.each do |instance| + encrypt!(instance, attributes) + end + end + end + + private + + # Build a hash of { attribute => encrypted column name } + def expand_attributes(klass, attributes) + expanded = attributes.flat_map do |attribute| + attr_config = klass.encrypted_attributes[attribute] + crypt_column_name = attr_config&.fetch(:attribute) + + raise "Couldn't determine encrypted column for #{klass}##{attribute}" if + crypt_column_name.nil? + + [attribute, crypt_column_name] + end + + Hash[*expanded] + end + + # Generate ciphertext for each column and update the database + def encrypt!(instance, attributes) + to_clear = attributes + .map { |plain, crypt| apply_attribute!(instance, plain, crypt) } + .compact + .flat_map { |plain| [plain, nil] } + + to_clear = Hash[*to_clear] + + if instance.changed? + instance.save! + instance.update_columns(to_clear) + end + end + + def apply_attribute!(instance, plain_column, crypt_column) + plaintext = instance[plain_column] + ciphertext = instance[crypt_column] + + # No need to do anything if the plaintext is nil, or an encrypted + # value already exists + return nil unless plaintext.present? && !ciphertext.present? + + # attr_encrypted will calculate and set the expected value for us + instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend + + plain_column + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb new file mode 100644 index 00000000000..bb76eb8ed48 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` and `url` columns + class WebHook < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'web_hooks' + self.inheritance_column = :_type_disabled + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated + + attr_encrypted :url, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb new file mode 100644 index 00000000000..036fe641757 --- /dev/null +++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateExternalPipelineSource + module Migratable + class Pipeline < ActiveRecord::Base + self.table_name = 'ci_pipelines' + + def self.sources + { + unknown: nil, + push: 1, + web: 2, + trigger: 3, + schedule: 4, + api: 5, + external: 6 + } + end + end + + class CommitStatus < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } + scope :of_type, -> (type) { where('type=?', type) } + end + end + + def perform(start_id, stop_id) + external_pipelines(start_id, stop_id) + .update_all(source: Migratable::Pipeline.sources[:external]) + end + + private + + def external_pipelines(start_id, stop_id) + Migratable::Pipeline.where(id: (start_id..stop_id)) + .where( + 'EXISTS (?) AND NOT EXISTS (?)', + Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), + Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) + ) + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 5963f3eb90c..98f12c226b3 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -9,7 +11,7 @@ module Gitlab include Validatable include Attributable - ALLOWED_KEYS = %i[junit].freeze + ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast].freeze attributes ALLOWED_KEYS @@ -19,6 +21,11 @@ module Gitlab with_options allow_nil: true do validates :junit, array_of_strings_or_string: true + validates :codequality, array_of_strings_or_string: true + validates :sast, array_of_strings_or_string: true + validates :dependency_scanning, array_of_strings_or_string: true + validates :container_scanning, array_of_strings_or_string: true + validates :dast, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb deleted file mode 100644 index a4eccc08dfc..00000000000 --- a/lib/gitlab/ci/parsers.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Gitlab - module Ci - module Parsers - def self.fabricate!(file_type) - "Gitlab::Ci::Parsers::#{file_type.classify}".constantize.new - end - end - end -end diff --git a/lib/gitlab/ci/parsers/junit.rb b/lib/gitlab/ci/parsers/junit.rb deleted file mode 100644 index d1c136f2009..00000000000 --- a/lib/gitlab/ci/parsers/junit.rb +++ /dev/null @@ -1,66 +0,0 @@ -module Gitlab - module Ci - module Parsers - class Junit - JunitParserError = Class.new(StandardError) - - def parse!(xml_data, test_suite) - root = Hash.from_xml(xml_data) - - all_cases(root) do |test_case| - test_case = create_test_case(test_case) - test_suite.add_test_case(test_case) - end - rescue REXML::ParseException => e - raise JunitParserError, "XML parsing failed: #{e.message}" - rescue => e - raise JunitParserError, "JUnit parsing failed: #{e.message}" - end - - private - - def all_cases(root, parent = nil, &blk) - return unless root.present? - - [root].flatten.compact.map do |node| - next unless node.is_a?(Hash) - - # we allow only one top-level 'testsuites' - all_cases(node['testsuites'], root, &blk) unless parent - - # we require at least one level of testsuites or testsuite - each_case(node['testcase'], &blk) if parent - - # we allow multiple nested 'testsuite' (eg. PHPUnit) - all_cases(node['testsuite'], root, &blk) - end - end - - def each_case(testcase, &blk) - return unless testcase.present? - - [testcase].flatten.compact.map(&blk) - end - - def create_test_case(data) - if data['failure'] - status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED - system_output = data['failure'] - else - status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS - system_output = nil - end - - ::Gitlab::Ci::Reports::TestCase.new( - classname: data['classname'], - name: data['name'], - file: data['file'], - execution_time: data['time'], - status: status, - system_output: system_output - ) - end - end - end - end -end diff --git a/lib/gitlab/ci/parsers/test.rb b/lib/gitlab/ci/parsers/test.rb new file mode 100644 index 00000000000..c6bc9662b07 --- /dev/null +++ b/lib/gitlab/ci/parsers/test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + ParserNotFoundError = Class.new(StandardError) + + PARSERS = { + junit: ::Gitlab::Ci::Parsers::Test::Junit + }.freeze + + def self.fabricate!(file_type) + PARSERS.fetch(file_type.to_sym).new + rescue KeyError + raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb new file mode 100644 index 00000000000..5d7d9a751d8 --- /dev/null +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + class Junit + JunitParserError = Class.new(StandardError) + + def parse!(xml_data, test_suite) + root = Hash.from_xml(xml_data) + + all_cases(root) do |test_case| + test_case = create_test_case(test_case) + test_suite.add_test_case(test_case) + end + rescue REXML::ParseException => e + raise JunitParserError, "XML parsing failed: #{e.message}" + rescue => e + raise JunitParserError, "JUnit parsing failed: #{e.message}" + end + + private + + def all_cases(root, parent = nil, &blk) + return unless root.present? + + [root].flatten.compact.map do |node| + next unless node.is_a?(Hash) + + # we allow only one top-level 'testsuites' + all_cases(node['testsuites'], root, &blk) unless parent + + # we require at least one level of testsuites or testsuite + each_case(node['testcase'], &blk) if parent + + # we allow multiple nested 'testsuite' (eg. PHPUnit) + all_cases(node['testsuite'], root, &blk) + end + end + + def each_case(testcase, &blk) + return unless testcase.present? + + [testcase].flatten.compact.map(&blk) + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 33ae9c6ad7e..9d01bcb7ca6 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -49,7 +49,7 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG KUBERNETES_VERSION: 1.8.6 - HELM_VERSION: 2.6.1 + HELM_VERSION: 2.11.0 DOCKER_DRIVER: overlay2 @@ -122,6 +122,9 @@ license_management: paths: [gl-license-management-report.json] only: - branches + only: + variables: + - $GITLAB_FEATURES =~ /\blicense_management\b/ except: variables: - $LICENSE_MANAGEMENT_DISABLED @@ -159,7 +162,10 @@ sast: artifacts: paths: [gl-sast-report.json] only: - - branches + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ except: variables: - $SAST_DISABLED @@ -176,7 +182,10 @@ dependency_scanning: artifacts: paths: [gl-dependency-scanning-report.json] only: - - branches + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ except: variables: - $DEPENDENCY_SCANNING_DISABLED @@ -193,7 +202,10 @@ container_scanning: artifacts: paths: [gl-container-scanning-report.json] only: - - branches + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast_container\b/ except: variables: - $CONTAINER_SCANNING_DISABLED @@ -212,6 +224,8 @@ dast: refs: - branches kubernetes: active + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ except: refs: - master @@ -225,7 +239,7 @@ review: - install_dependencies - download_chart - ensure_namespace - - install_tiller + - initialize_tiller - create_secret - deploy - persist_environment_url @@ -251,6 +265,7 @@ stop_review: GIT_STRATEGY: none script: - install_dependencies + - initialize_tiller - delete environment: name: review/$CI_COMMIT_REF_NAME @@ -285,7 +300,7 @@ staging: - install_dependencies - download_chart - ensure_namespace - - install_tiller + - initialize_tiller - create_secret - deploy environment: @@ -309,7 +324,7 @@ canary: - install_dependencies - download_chart - ensure_namespace - - install_tiller + - initialize_tiller - create_secret - deploy canary environment: @@ -330,7 +345,7 @@ canary: - install_dependencies - download_chart - ensure_namespace - - install_tiller + - initialize_tiller - create_secret - deploy - delete canary @@ -378,7 +393,7 @@ production_manual: - install_dependencies - download_chart - ensure_namespace - - install_tiller + - initialize_tiller - create_secret - deploy rollout $ROLLOUT_PERCENTAGE - scale stable $((100-ROLLOUT_PERCENTAGE)) @@ -484,15 +499,11 @@ rollout 100%: } function license_management() { - if echo $GITLAB_FEATURES |grep license_management > /dev/null ; then - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" - LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" + LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - docker run --volume "$PWD:/code" \ - "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code - else - echo "License management is not available in your subscription" - fi + docker run --volume "$PWD:/code" \ + "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code } function sast() { @@ -605,7 +616,6 @@ rollout 100%: --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ --set postgresql.postgresDatabase="$POSTGRES_DB" \ --namespace="$KUBE_NAMESPACE" \ - --version="$CI_PIPELINE_ID-$CI_JOB_ID" \ "$name" \ chart/ @@ -642,7 +652,12 @@ rollout 100%: curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx mv linux-amd64/helm /usr/bin/ + mv linux-amd64/tiller /usr/bin/ helm version --client + tiller -version + + helm init --client-only + helm plugin install https://github.com/adamreese/helm-local curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" chmod +x /usr/bin/kubectl @@ -749,10 +764,13 @@ rollout 100%: echo "" } - function install_tiller() { + function initialize_tiller() { echo "Checking Tiller..." - helm init --upgrade - kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy" + + helm local start + helm local status + export HELM_HOST=":44134" + if ! helm version --debug; then echo "Failed to init Tiller." return 1 diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 140cb4635f3..04c21b4725d 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -1,4 +1,4 @@ -# An example .gitlab-ci.yml file to test (and optionally report the coverage +# This is an example .gitlab-ci.yml file to test (and optionally report the coverage # results of) your [Julia][1] packages. Please refer to the [documentation][2] # for more information about package development in Julia. # @@ -6,7 +6,7 @@ # whatever name you have given to your package. # # [1]: http://julialang.org/ -# [2]: http://julia.readthedocs.org/ +# [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html # Below is the template to run your tests in Julia .test_template: &test_definition @@ -18,25 +18,30 @@ script: # Let's run the tests. Substitute `coverage = false` below, if you do not # want coverage results. - - /opt/julia/bin/julia -e 'Pkg.clone(pwd()); Pkg.test("MyPackage", - coverage = true)' + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)' # Comment out below if you do not want coverage results. - - /opt/julia/bin/julia -e 'Pkg.add("Coverage"); cd(Pkg.dir("MyPackage")); + - julia -e 'using Pkg; Pkg.add("Coverage"); + import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), "..")); using Coverage; cl, tl = get_summary(process_folder()); println("(", cl/tl*100, "%) covered")' # Name a test and select an appropriate image. -test:0.4.6: - image: julialang/julia:v0.4.6 +# images comes from Docker hub +test:0.7: + image: julia:0.7 + <<: *test_definition + +test:1.0: + image: julia:1.0 <<: *test_definition # Maybe you would like to test your package against the development branch: -test:0.5.0-dev: - image: julialang/julia:v0.5.0-dev - # ... allowing for failures, since we are testing against the development - # branch: - allow_failure: true - <<: *test_definition +# test:1.1-dev (not sure there is such an image in docker, so not tested yet): +# image: julia:v1.1-dev +# # ... allowing for failures, since we are testing against the development +# # branch: +# allow_failure: true +# <<: *test_definition # REMARK: Do not forget to enable the coverage feature for your project, if you # are using code coverage reporting above. This can be done by @@ -44,11 +49,28 @@ test:0.5.0-dev: # - Navigating to the `CI/CD Pipelines` settings of your project, # - Copying and pasting the default `Simplecov` regex example provided, i.e., # `\(\d+.\d+\%\) covered` in the `test coverage parsing` textfield. -# -# WARNING: This template is using the `julialang/julia` images from [Docker + +# Example documentation deployment +pages: + image: julia:0.7 + stage: deploy + script: + - apt-get update -qq && apt-get install -y git # needed by Documenter + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure + - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter + - julia --color=yes docs/make.jl # make documentation + - mv docs/build public # move to the directory picked up by Gitlab pages + artifacts: + paths: + - public + only: + - master + + +# WARNING: This template is using the `julia` images from [Docker # Hub][3]. One can use custom Julia images and/or the official ones found # in the same place. However, care must be taken to correctly locate the binary # file (`/opt/julia/bin/julia` above), which is usually given on the image's # description page. # -# [3]: http://hub.docker.com/ +# [3]: https://hub.docker.com/_/julia/ diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml new file mode 100644 index 00000000000..0e5fb410a4e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -0,0 +1,42 @@ +# Jigsaw is a simple static sites generator with Laravel's Blade. +# +# Full project: https://github.com/tightenco/jigsaw + +image: php:7.2 + +# These folders are cached between builds +cache: + paths: + - vendor/ + - node_modules/ + +before_script: + # Update packages + - apt-get update -yqq + + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + + # Install php extensions + - docker-php-ext-install zip + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + - npm install + +pages: + script: + - npm run production + - mv build_production public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index b97bfd460f2..ba8a802ba4f 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -1,5 +1,5 @@ # Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ -# This file assumes an own GitLab CI runner, set up on a macOS system. +# This file assumes an own GitLab CI runner, setup on a macOS system. stages: - build - archive diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index eb246d393a1..f382992cb0a 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -26,7 +26,8 @@ module Gitlab stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, - duration: pipeline.duration + duration: pipeline.duration, + variables: pipeline.variables.map(&:hook_attrs) } end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7f012312819..30541ee3553 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1073,6 +1073,10 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end + + def mysql_compatible_index_length + Gitlab::Database.mysql? ? 20 : nil + end end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 1f012043e56..a605ddb5c33 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,7 +24,7 @@ module Gitlab # ignore highlighting for "match" lines next diff_line if diff_line.meta? - rich_line = highlight_line(diff_line) || diff_line.text + rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) if line_inline_diffs = inline_diffs[i] begin diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index f6b51dc3982..0d96211f4d4 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -19,13 +19,17 @@ module Gitlab alias_method :expanded?, :expanded - SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze + # The default maximum content size to display a diff patch. + # + # If this value ever changes, make sure to create a migration to update + # current records, and default of `ApplicationSettings#diff_max_patch_bytes`. + DEFAULT_MAX_PATCH_BYTES = 100.kilobytes - # The maximum size of a diff to display. - SIZE_LIMIT = 100.kilobytes + # This is a limitation applied on the source (Gitaly), therefore we don't allow + # persisting limits over that. + MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - # The maximum size before a diff is collapsed. - COLLAPSE_LIMIT = 10.kilobytes + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self def between(repo, head, base, options = {}, *paths) @@ -105,6 +109,26 @@ module Gitlab def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end + + # Returns the limit of bytes a single diff file can reach before it + # appears as 'collapsed' for end-users. + # By convention, it's 10% of the persisted `diff_max_patch_bytes`. + # + # Example: If we have 100k for the `diff_max_patch_bytes`, it will be 10k by + # default. + # + # Patches surpassing this limit should still be persisted in the database. + def patch_safe_limit_bytes + patch_hard_limit_bytes / 10 + end + + # Returns the limit for a single diff file (patch). + # + # Patches surpassing this limit shouldn't be persisted in the database + # and will be presented as 'too large' for end-users. + def patch_hard_limit_bytes + Gitlab::CurrentSettings.diff_max_patch_bytes + end end def initialize(raw_diff, expanded: true) @@ -150,7 +174,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -168,7 +192,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT + @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes end def collapse! @@ -219,30 +243,6 @@ module Gitlab collapse! end end - - # If the patch surpasses any of the diff limits it calls the appropiate - # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch) - size = 0 - - patch.each_hunk do |hunk| - hunk.each_line do |line| - size += line.content.bytesize - - if size >= SIZE_LIMIT - too_large! - return true # rubocop:disable Cop/AvoidReturnFromBlocks - end - end - end - - if !expanded && size >= COLLAPSE_LIMIT - collapse! - return true - end - - false - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 20dce8d0e06..47ebca7c4a2 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -19,7 +19,7 @@ module Gitlab limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file - limits[:max_patch_bytes] = Gitlab::Git::Diff::SIZE_LIMIT + limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes OpenStruct.new(limits) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3d5a63bdbac..45d42c7078f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -953,6 +953,12 @@ module Gitlab end end + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25) + wrapped_gitaly_errors do + gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit) + end + end + def last_commit_for_path(sha, path) wrapped_gitaly_errors do gitaly_commit_client.last_commit_for_path(sha, path) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 07e5e204b68..085b2a127a5 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -148,6 +148,24 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) + request = Gitaly::ListLastCommitsForTreeRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + path: encode_binary(path.to_s), + offset: offset, + limit: limit + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + + response.each_with_object({}) do |gitaly_response, hsh| + gitaly_response.commits.each do |commit_for_tree| + hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + end + end + end + def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 54c78fdb680..0f148614b20 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -333,7 +333,8 @@ module Gitlab action: action[:action].upcase.to_sym, file_path: encode_binary(action[:file_path]), previous_path: encode_binary(action[:previous_path]), - base64_content: action[:encoding] == 'base64' + base64_content: action[:encoding] == 'base64', + execute_filemode: !!action[:execute_filemode] ) rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 5408a1a6838..0b6cc893db1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,5 +1,8 @@ module Gitlab class Highlight + TIMEOUT_BACKGROUND = 30.seconds + TIMEOUT_FOREGROUND = 3.seconds + def self.highlight(blob_name, blob_content, repository: nil, plain: false) new(blob_name, blob_content, repository: repository) .highlight(blob_content, continue: false, plain: plain) @@ -51,11 +54,20 @@ module Gitlab end def highlight_rich(text, continue: true) - @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + tag = lexer.tag + tokens = lexer.lex(text, continue: continue) + Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } + rescue Timeout::Error => e + Gitlab::Sentry.track_exception(e) + highlight_plain(text) rescue highlight_plain(text) end + def timeout_time + Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND + end + def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index a19b3c88627..2bed470514b 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -147,6 +147,12 @@ excluded_attributes: - :reference - :reference_html - :epic_id + hooks: + - :token + - :encrypted_token + - :encrypted_token_iv + - :encrypted_url + - :encrypted_url_iv methods: labels: diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index b1bf3ca4143..a03ce07b6a1 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -29,5 +29,21 @@ module Gitlab def read(key) backend.read(cache_key(key)) end + + def write(key, value) + backend.write(cache_key(key), value) + end + + def fetch_without_caching_false(key, &block) + value = read(key) + return value if value + + value = yield + + # Don't cache false values + write(key, value) if value + + value + end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 2ec871f0754..d95024fccf7 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -1,23 +1,80 @@ module Gitlab module RepositoryCacheAdapter extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize class_methods do - # Wraps around the given method and caches its output in Redis and an instance - # variable. + # Caches and strongly memoizes the method. # # This only works for methods that do not take any arguments. - def cache_method(name, fallback: nil, memoize_only: false) - original = :"_uncached_#{name}" + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end - alias_method(original, name) + # Caches truthy values from the method. All values are strongly memoized, + # and cached in RequestStore. + # + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be cached. + def cache_method_asymmetrically(name) + uncached_name = alias_uncached_method(name) define_method(name) do - cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do - __send__(original) # rubocop:disable GitlabSecurity/PublicSend + cache_method_output_asymmetrically(name) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend end end end + + # Strongly memoizes the method. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value + # is not memoized. + def memoize_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + memoize_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # Prepends "_uncached_" to the target method name + # + # Returns the uncached method name + def alias_uncached_method(name) + uncached_name = :"_uncached_#{name}" + + alias_method(uncached_name, name) + + uncached_name + end + end + + # RequestStore-backed RepositoryCache to be used. Should be overridden by + # the including class + def request_store_cache + raise NotImplementedError end # RepositoryCache to be used. Should be overridden by the including class @@ -30,65 +87,93 @@ module Gitlab raise NotImplementedError end - # Caches the supplied block both in a cache and in an instance variable. + # Caches and strongly memoizes the supplied block. # - # The cache key and instance variable are named the same way as the value of - # the `key` argument. + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + cache.fetch(name, &block) + end + end + + # Caches truthy values from the supplied block. All values are strongly + # memoized, and cached in RequestStore. # - # This method will return `nil` if the corresponding instance variable is also - # set to `nil`. This ensures we don't keep yielding the block when it returns - # `nil`. + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. # - # key - The name of the key to cache the data in. - # fallback - A value to fall back to in the event of a Git error. - def cache_method_output(key, fallback: nil, memoize_only: false, &block) - ivar = cache_instance_variable_name(key) - - if instance_variable_defined?(ivar) - instance_variable_get(ivar) - else - # If the repository doesn't exist and a fallback was specified we return - # that value inmediately. This saves us Rugged/gRPC invocations. - return fallback unless fallback.nil? || cache.repository.exists? - - begin - value = - if memoize_only - yield - else - cache.fetch(key, &block) - end - - instance_variable_set(ivar, value) - rescue Gitlab::Git::Repository::NoRepository - # Even if the above `#exists?` check passes these errors might still - # occur (for example because of a non-existing HEAD). We want to - # gracefully handle this and not cache anything - fallback + # name - The name of the method to be cached. + def cache_method_output_asymmetrically(name, &block) + memoize_method_output(name) do + request_store_cache.fetch(name) do + cache.fetch_without_caching_false(name, &block) end end end + # Strongly memoizes the supplied block. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value is + # not memoized. + def memoize_method_output(name, fallback: nil, &block) + no_repository_fallback(name, fallback: fallback) do + strong_memoize(memoizable_name(name), &block) + end + end + + # Returns the fallback value if the repository does not exist + def no_repository_fallback(name, fallback: nil, &block) + # Avoid unnecessary gRPC invocations + return fallback if fallback && fallback_early?(name) + + yield + rescue Gitlab::Git::Repository::NoRepository + # Even if the `#exists?` check in `fallback_early?` passes, these errors + # might still occur (for example because of a non-existing HEAD). We + # want to gracefully handle this and not memoize anything. + fallback + end + # Expires the caches of a specific set of methods def expire_method_caches(methods) - methods.each do |key| - unless cached_methods.include?(key.to_sym) - Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + methods.each do |name| + unless cached_methods.include?(name.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" next end - cache.expire(key) + cache.expire(name) - ivar = cache_instance_variable_name(key) - - remove_instance_variable(ivar) if instance_variable_defined?(ivar) + clear_memoization(memoizable_name(name)) end + + expire_request_store_method_caches(methods) end private - def cache_instance_variable_name(key) - :"@#{key.to_s.tr('?!', '')}" + def memoizable_name(name) + "#{name.to_s.tr('?!', '')}" + end + + def expire_request_store_method_caches(methods) + methods.each do |name| + request_store_cache.expire(name) + end + end + + # All cached repository methods depend on the existence of a Git repository, + # so if the repository doesn't exist, we already know not to call it. + def fallback_early?(method_name) + # Avoid infinite loop + return false if method_name == :exists? + + !exists? end end end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index b05d408b1c0..c2955cd374c 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -73,25 +73,29 @@ module Gitlab end def fill_last_commits!(entries) - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37433 - Gitlab::GitalyClient.allow_n_plus_1_calls do - entries.each do |entry| - raw_commit = repository.last_commit_for_path(commit.id, entry_path(entry)) + # Ensure the path is in "path/" format + ensured_path = + if path + File.join(*[path, ""]) + end + + commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit) - if raw_commit - commit = resolve_commit(raw_commit) + entries.each do |entry| + path_key = entry_path(entry) + commit = cache_commit(commits_hsh[path_key]) - entry[:commit] = commit - entry[:commit_path] = commit_path(commit) - end + if commit + entry[:commit] = commit + entry[:commit_path] = commit_path(commit) end end end - def resolve_commit(raw_commit) - return nil unless raw_commit.present? + def cache_commit(commit) + return nil unless commit.present? - resolved_commits[raw_commit.id] ||= ::Commit.new(raw_commit, project) + resolved_commits[commit.id] ||= commit end def commit_path(commit) diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index bfca4bfb3f7..e7634d2ed4f 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -15,7 +15,7 @@ namespace :gitlab do build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) - logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") rescue => e logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") end |