diff options
Diffstat (limited to 'lib')
279 files changed, 4217 insertions, 2462 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 223ae13bd2d..219ed45eff6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -18,7 +18,7 @@ module API formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ GrapeLogging::Loggers::FilterParameters.new(LOG_FILTERS), - GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::ClientEnvLogger.new, Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, @@ -104,7 +104,6 @@ module API mount ::API::BroadcastMessages mount ::API::Commits mount ::API::CommitStatuses - mount ::API::ContainerRegistry mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -116,6 +115,7 @@ module API mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups + mount ::API::GroupContainerRepositories mount ::API::GroupVariables mount ::API::ImportGithub mount ::API::Internal @@ -138,6 +138,7 @@ module API mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectClusters + mount ::API::ProjectContainerRepositories mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectImport diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index a1851ba3627..89b7e5c5e4b 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -69,12 +69,12 @@ module API post endpoint do not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? - award = awardable.create_award_emoji(params[:name], current_user) + service = AwardEmojis::AddService.new(awardable, params[:name], current_user).execute - if award.persisted? - present award, with: Entities::AwardEmoji + if service[:status] == :success + present service[:award], with: Entities::AwardEmoji else - not_found!("Award Emoji #{award.errors.messages}") + not_found!("Award Emoji #{service[:message]}") end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 08b4f8db8b0..d58a5e214ed 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -52,6 +52,7 @@ module API optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' + optional :pipeline_id, type: Integer, desc: 'An existing pipeline ID, when multiple pipelines on the same commit SHA have been triggered' end # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do @@ -73,7 +74,8 @@ module API name = params[:name] || params[:context] || 'default' - pipeline = @project.pipeline_for(ref, commit.sha) + pipeline = @project.pipeline_for(ref, commit.sha, params[:pipeline_id]) + unless pipeline pipeline = @project.ci_pipelines.create!( source: :external, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index c414ad75d9d..a2f3e87ebd2 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -43,7 +43,7 @@ module API path = params[:path] before = params[:until] after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] + ref = params[:ref_name].presence || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] all = params[:all] with_stats = params[:with_stats] @@ -76,7 +76,7 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array, desc: 'Actions to perform in commit' do requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze @@ -98,12 +98,16 @@ module API requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' end end - optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' - optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the commit from' + + optional :start_branch, type: String, desc: 'Name of the branch to start the new branch from' + optional :start_sha, type: String, desc: 'SHA of the commit to start the new branch from' + mutually_exclusive :start_branch, :start_sha + + optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the new branch from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' - optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' + optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha`' end post ':id/repository/commits' do if params[:start_project] @@ -118,7 +122,7 @@ module API attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) - attrs[:start_branch] ||= attrs[:branch_name] + attrs[:start_branch] ||= attrs[:branch_name] unless attrs[:start_sha] attrs[:start_project] = start_project if start_project result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -126,7 +130,7 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - Gitlab::UsageDataCounters::WebIdeCommitsCounter.increment if find_user_from_warden + Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count if find_user_from_warden present commit_detail, with: Entities::CommitDetail, stats: params[:stats] else diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index cc62ce22a1b..6c1acc3963f 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -4,6 +4,7 @@ module API class Discussions < Grape::API include PaginationParams helpers ::API::Helpers::NotesHelpers + helpers ::RendersNotes before { authenticate! } @@ -23,21 +24,15 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' use :pagination end - # rubocop: disable CodeReuse/ActiveRecord + get ":id/#{noteables_path}/:noteable_id/discussions" do noteable = find_noteable(parent_type, params[:id], noteable_type, params[:noteable_id]) - notes = noteable.notes - .inc_relations_for_view - .includes(:noteable) - .fresh - - notes = notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } + notes = readable_discussion_notes(noteable) discussions = Kaminari.paginate_array(Discussion.build_collection(notes, noteable)) present paginate(discussions), with: Entities::Discussion end - # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion @@ -226,13 +221,24 @@ module API helpers do # rubocop: disable CodeReuse/ActiveRecord - def readable_discussion_notes(noteable, discussion_id) + def readable_discussion_notes(noteable, discussion_id = nil) notes = noteable.notes - .where(discussion_id: discussion_id) + notes = notes.where(discussion_id: discussion_id) if discussion_id + notes = notes .inc_relations_for_view .includes(:noteable) .fresh + # Without RendersActions#prepare_notes_for_rendering, + # Note#cross_reference_not_visible_for? will attempt to render + # Markdown references mentioned in the note to see whether they + # should be redacted. For notes that reference a commit, this + # would also incur a Gitaly call to verify the commit exists. + # + # With prepare_notes_for_rendering, we can avoid Gitaly calls + # because notes are redacted if they point to projects that + # cannot be accessed by the user. + notes = prepare_notes_for_rendering(notes) notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 494da770279..5e66b4e76a5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -2,6 +2,19 @@ module API module Entities + class BlameRangeCommit < Grape::Entity + expose :id + expose :parent_ids + expose :message + expose :authored_date, :author_name, :author_email + expose :committed_date, :committer_name, :committer_email + end + + class BlameRange < Grape::Entity + expose :commit, using: BlameRangeCommit + expose :lines + end + class WikiPageBasic < Grape::Entity expose :format expose :slug @@ -64,6 +77,11 @@ module API expose :last_activity_on, as: :last_activity_at # Back-compat end + class UserStarsProject < Grape::Entity + expose :starred_since + expose :user, using: Entities::UserBasic + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -366,10 +384,7 @@ module API end expose :request_access_enabled expose :full_name, :full_path - - if ::Group.supports_nested_objects? - expose :parent_id - end + expose :parent_id expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes @@ -630,7 +645,10 @@ module API end end - expose :subscribed do |issue, options| + # Calculating the value of subscribed field triggers Markdown + # processing. We can't do that for multiple issues / merge + # requests in a single API request. + expose :subscribed, if: -> (_, options) { options.fetch(:include_subscribed, true) } do |issue, options| issue.subscribed?(options[:current_user], options[:project] || issue.project) end end @@ -1052,15 +1070,8 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) - - # MySQL doesn't support LIMIT inside an IN subquery - if Gitlab::Database.mysql? - project_ids = relation.pluck('projects.id') - namespace_ids = relation.pluck(:namespace_id) - else - project_ids = relation.select('projects.id') - namespace_ids = relation.select(:namespace_id) - end + project_ids = relation.select('projects.id') + namespace_ids = relation.select(:namespace_id) options[:project_members] = options[:current_user] .project_members @@ -1082,16 +1093,18 @@ module API end class Label < LabelBasic - expose :open_issues_count do |label, options| - label.open_issues_count(options[:current_user]) - end + with_options if: lambda { |_, options| options[:with_counts] } do + expose :open_issues_count do |label, options| + label.open_issues_count(options[:current_user]) + end - expose :closed_issues_count do |label, options| - label.closed_issues_count(options[:current_user]) - end + expose :closed_issues_count do |label, options| + label.closed_issues_count(options[:current_user]) + end - expose :open_merge_requests_count do |label, options| - label.open_merge_requests_count(options[:current_user]) + expose :open_merge_requests_count do |label, options| + label.open_merge_requests_count(options[:current_user]) + end end expose :subscribed do |label, options| @@ -1159,6 +1172,7 @@ module API attributes = ::ApplicationSettingsHelper.visible_attributes attributes.delete(:performance_bar_allowed_group_path) attributes.delete(:performance_bar_enabled) + attributes.delete(:allow_local_requests_from_hooks_and_services) attributes end @@ -1177,6 +1191,7 @@ module API # support legacy names, can be removed in v5 expose :password_authentication_enabled_for_web, as: :password_authentication_enabled expose :password_authentication_enabled_for_web, as: :signin_enabled + expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services end # deprecated old Release representation @@ -1336,6 +1351,7 @@ module API expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) } + expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) } end class Pipeline < PipelineBasic diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index 00833ca7480..6250f35c7cb 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -3,18 +3,20 @@ module API module Entities module ContainerRegistry - class Repository < Grape::Entity - expose :id + class Tag < Grape::Entity expose :name expose :path expose :location - expose :created_at end - class Tag < Grape::Entity + class Repository < Grape::Entity + expose :id expose :name expose :path + expose :project_id expose :location + expose :created_at + expose :tags, using: Tag, if: -> (_, options) { options[:tags] } end class TagDetails < Tag diff --git a/lib/api/files.rb b/lib/api/files.rb index ca59d330e1c..0b438fb5bbc 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -83,6 +83,31 @@ module API resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } + desc 'Get blame file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + head ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + + desc 'Get blame file from the repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + + blame_ranges = Gitlab::Blame.new(@blob, @commit).groups(highlight: false) + present blame_ranges, with: Entities::BlameRange + end + desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb new file mode 100644 index 00000000000..fd24662cc9a --- /dev/null +++ b/lib/api/group_container_repositories.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module API + class GroupContainerRepositories < Grape::API + include PaginationParams + + before { authorize_read_group_container_images! } + + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + tag_name: API::NO_SLASH_URL_PART_REGEX) + + params do + requires :id, type: String, desc: "Group's ID or path" + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a list of all repositories within a group' do + detail 'This feature was introduced in GitLab 12.2.' + success Entities::ContainerRegistry::Repository + end + params do + use :pagination + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' + end + get ':id/registry/repositories' do + repositories = ContainerRepositoriesFinder.new( + id: user_group.id, container_type: :group + ).execute + + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] + end + end + + helpers do + def authorize_read_group_container_images! + authorize! :read_container_image, user_group + end + end + end +end diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 0dbc5f45a68..79a44941c81 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -16,6 +16,8 @@ module API success Entities::GroupLabel end params do + optional :with_counts, type: Boolean, default: false, + desc: 'Include issue and merge request counts' use :pagination end get ':id/labels' do diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ec1020c7c78..f545f33c06b 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -114,10 +114,7 @@ module API params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - - if ::Group.supports_nested_objects? - optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' - end + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' use :optional_params end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8ae42c6dadd..1aa6dc44bf7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -544,5 +544,9 @@ module API params[:archived] end + + def ip_address + env["action_dispatch.remote_ip"].to_s || request.ip + end end end diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index c11e7d614ab..896b0aba52b 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -19,7 +19,11 @@ module API end def get_labels(parent, entity) - present paginate(available_labels_for(parent)), with: entity, current_user: current_user, parent: parent + present paginate(available_labels_for(parent)), + with: entity, + current_user: current_user, + parent: parent, + with_counts: params[:with_counts] end def create_label(parent, entity) diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index b03ac7deb71..6bf9057fad7 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -74,14 +74,14 @@ module API end def find_noteable(parent_type, parent_id, noteable_type, noteable_id) - params = params_by_noteable_type_and_id(noteable_type, noteable_id) + params = finder_params_by_noteable_type_and_id(noteable_type, noteable_id, parent_id) - noteable = NotesFinder.new(user_project, current_user, params).target + noteable = NotesFinder.new(current_user, params).target noteable = nil unless can?(current_user, noteable_read_ability_name(noteable), noteable) noteable || not_found!(noteable_type) end - def params_by_noteable_type_and_id(type, id) + def finder_params_by_noteable_type_and_id(type, id, parent_id) target_type = type.name.underscore { target_type: target_type }.tap do |h| if %w(issue merge_request).include?(target_type) @@ -89,9 +89,15 @@ module API else h[:target_id] = id end + + add_parent_to_finder_params(h, type, parent_id) end end + def add_parent_to_finder_params(finder_params, noteable_type, parent_id) + finder_params[:project] = user_project + end + def noteable_parent(noteable) public_send("user_#{noteable.class.parent_class.to_s.underscore}") # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 833e3b9ebaf..51b7cf05c8f 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -71,6 +71,7 @@ module API :build_timeout, :builds_access_level, :ci_config_path, + :ci_default_git_depth, :container_registry_enabled, :default_branch, :description, diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 100463fcb95..5b87eccf860 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -25,7 +25,7 @@ module API end def get_runner_ip - { ip_address: env["action_dispatch.remote_ip"].to_s || request.ip } + { ip_address: ip_address } end def current_runner diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index c4ecf55969c..422db5c7a50 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -489,32 +489,6 @@ module API desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' } ], - 'kubernetes' => [ - { - required: true, - name: :namespace, - type: String, - desc: 'The Kubernetes namespace to use' - }, - { - required: true, - name: :api_url, - type: String, - desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' - }, - { - required: true, - name: :token, - type: String, - desc: 'The service token to authenticate against the Kubernetes cluster with' - }, - { - required: false, - name: :ca_pem, - type: String, - desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - } - ], 'mattermost-slash-commands' => [ { required: true, @@ -739,7 +713,6 @@ module API ::HipchatService, ::IrkerService, ::JiraService, - ::KubernetesService, ::MattermostSlashCommandsService, ::SlackSlashCommandsService, ::PackagistService, diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb deleted file mode 100644 index 78a92d0f5a6..00000000000 --- a/lib/api/helpers/variables_helpers.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module VariablesHelpers - extend ActiveSupport::Concern - extend Grape::API::Helpers - - params :optional_params_ee do - end - end - end -end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index d687acf3423..7819c2de515 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -96,7 +96,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options @@ -122,7 +123,8 @@ module API with: Entities::Issue, with_labels_details: declared_params[:with_labels_details], current_user: current_user, - issuable_metadata: issuable_meta_data(issues, 'Issue', current_user) + issuable_metadata: issuable_meta_data(issues, 'Issue', current_user), + include_subscribed: false } present issues, options diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index e7fed55170e..b35aa952f81 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -27,7 +27,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end @@ -45,7 +45,7 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) diff --git a/lib/api/labels.rb b/lib/api/labels.rb index d729d3ee625..c183198d3c6 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,6 +15,8 @@ module API success Entities::ProjectLabel end params do + optional :with_counts, type: Boolean, default: false, + desc: 'Include issue and merge request counts' use :pagination end get ':id/labels' do diff --git a/lib/api/container_registry.rb b/lib/api/project_container_repositories.rb index 7dad20a822a..6d53abcc500 100644 --- a/lib/api/container_registry.rb +++ b/lib/api/project_container_repositories.rb @@ -1,10 +1,10 @@ # frozen_string_literal: true module API - class ContainerRegistry < Grape::API + class ProjectContainerRepositories < Grape::API include PaginationParams - REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) } @@ -20,11 +20,14 @@ module API end params do use :pagination + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' end get ':id/registry/repositories' do - repositories = user_project.container_repositories.ordered + repositories = ContainerRepositoriesFinder.new( + id: user_project.id, container_type: :project + ).execute - present paginate(repositories), with: Entities::ContainerRegistry::Repository + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags] end desc 'Delete repository' do @@ -33,7 +36,7 @@ module API params do requires :repository_id, type: Integer, desc: 'The ID of the repository' end - delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) @@ -49,7 +52,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' use :pagination end - get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + get ':id/registry/repositories/:repository_id/tags', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) @@ -65,7 +68,7 @@ module API optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name' optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month' end - delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id/tags', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_admin_container_image! message = 'This request has already been made. You can run this at most once an hour for a given container repository' @@ -85,7 +88,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' requires :tag_name, type: String, desc: 'The name of the tag' end - get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_read_container_image! validate_tag! @@ -99,7 +102,7 @@ module API requires :repository_id, type: Integer, desc: 'The ID of the repository' requires :tag_name, type: String, desc: 'The name of the tag' end - delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REPOSITORY_ENDPOINT_REQUIREMENTS do authorize_destroy_container_image! validate_tag! diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 71891e43dcc..bb1b037c08f 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -59,6 +59,7 @@ module API } override_params = import_params.delete(:override_params) + filter_attributes_using_license!(override_params) if override_params project = ::Projects::GitlabProjectsImportService.new( current_user, project_params, override_params diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 0923d31f5ff..996205d4b7b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -115,6 +115,22 @@ module API present_projects load_projects end + + desc 'Get projects starred by a user' do + success Entities::BasicProjectDetails + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :collection_params + use :statistics_params + end + get ":user_id/starred_projects" do + user = find_user(params[:user_id]) + not_found!('User') unless user + + starred_projects = StarredProjectsFinder.new(user, params: project_finder_params, current_user: current_user).execute + present_projects starred_projects + end end resource :projects do @@ -358,6 +374,19 @@ module API end end + desc 'Get the users who starred a project' do + success Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end + get ':id/starrers' do + starrers = UsersStarProjectsFinder.new(user_project, params, current_user: current_user).execute + + present paginate(starrers), with: Entities::UserStarsProject + end + desc 'Get languages in project repository' get ':id/languages' do ::Projects::RepositoryLanguagesService diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 4275d911708..c36ee5af63f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -59,7 +59,7 @@ module API optional :grafana_url, type: String, desc: 'Grafana URL' optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' - optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page' + optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page and help dropdown' optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' @@ -124,6 +124,13 @@ module API optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated" + optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 + optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking' + given snowplow_enabled: ->(val) { val } do + requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname' + optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain' + optional :snowplow_site_id, type: String, desc: 'The Snowplow site name / application ic' + end ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -158,6 +165,11 @@ module API attrs[:password_authentication_enabled_for_web] = attrs.delete(:password_authentication_enabled) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:allow_local_requests_from_hooks_and_services) + attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services) + end + attrs = filter_attributes_using_license(attrs) if ApplicationSettings::UpdateService.new(current_settings, current_user, attrs).execute diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 7260ecfb5ee..404675bfaec 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -13,6 +13,13 @@ module API 'issues' => ->(iid) { find_project_issue(iid) } }.freeze + helpers do + # EE::API::Todos would override this method + def find_todos + TodosFinder.new(current_user, params).execute + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -41,10 +48,6 @@ module API resource :todos do helpers do - def find_todos - TodosFinder.new(current_user, params).execute - end - def issuable_and_awardable?(type) obj_type = Object.const_get(type) @@ -107,3 +110,5 @@ module API end end end + +API::Todos.prepend_if_ee('EE::API::Todos') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 0e829c5699b..eeecc390256 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -112,27 +112,6 @@ module API end end - desc 'Take ownership of trigger' do - success Entities::Trigger - end - params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' - end - post ':id/triggers/:trigger_id/take_ownership' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find(params.delete(:trigger_id)) - break not_found!('Trigger') unless trigger - - if trigger.update(owner: current_user) - status :ok - present trigger, with: Entities::Trigger, current_user: current_user - else - render_validation_error!(trigger) - end - end - desc 'Delete a trigger' do success Entities::Trigger end diff --git a/lib/api/validations/types/labels_list.rb b/lib/api/validations/types/labels_list.rb index 47cd83c29cf..60277b99106 100644 --- a/lib/api/validations/types/labels_list.rb +++ b/lib/api/validations/types/labels_list.rb @@ -10,7 +10,7 @@ module API when String value.split(',').map(&:strip) when Array - value.map { |v| v.to_s.split(',').map(&:strip) }.flatten + value.flat_map { |v| v.to_s.split(',').map(&:strip) } when LabelsList value else diff --git a/lib/api/variables.rb b/lib/api/variables.rb index af1d7936556..f022b9e665a 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,8 +7,6 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } - helpers Helpers::VariablesHelpers - helpers do def filter_variable_parameters(params) # This method exists so that EE can more easily filter out certain @@ -59,8 +57,7 @@ module API optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - - use :optional_params_ee + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -84,8 +81,7 @@ module API optional :protected, type: Boolean, desc: 'Whether the variable is protected' optional :masked, type: Boolean, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - - use :optional_params_ee + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/backup/database.rb b/lib/backup/database.rb index cd8e29d14d3..7e457c4982d 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -23,11 +23,6 @@ module Backup dump_pid = case config["adapter"] - when /^mysql/ then - progress.print "Dumping MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -57,11 +52,6 @@ module Backup restore_pid = case config["adapter"] - when /^mysql/ then - progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env @@ -80,23 +70,6 @@ module Backup protected - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set', - # SSL - 'sslkey' => '--ssl-key', - 'sslcert' => '--ssl-cert', - 'sslca' => '--ssl-ca', - 'sslcapath' => '--ssl-capath', - 'sslcipher' => '--ssl-cipher' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - def pg_env args = { 'username' => 'PGUSER', diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 0224dd8fcd1..52af28ce8ec 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -337,6 +337,24 @@ module Banzai @current_project_namespace_path ||= project&.namespace&.full_path end + def records_per_parent + @_records_per_project ||= {} + + @_records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record_identifier(record)] = record + end + end + + hash + end + end + private def full_project_path(namespace, project_ref) diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 56214043d87..5f2cbc24c60 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -18,6 +18,7 @@ module Banzai # class AutolinkFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper + include Gitlab::Utils::SanitizeNodeLink # Pattern to match text that should be autolinked. # @@ -72,19 +73,11 @@ module Banzai private - # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme - def contains_unsafe?(scheme) - return false unless scheme - - scheme = scheme.strip.downcase - Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } - end - def autolink_match(match) # start by stripping out dangerous links begin uri = Addressable::URI.parse(match) - return match if contains_unsafe?(uri.scheme) + return match unless safe_protocol?(uri.scheme) rescue Addressable::URI::InvalidURIError return match end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 420e92cb1e8..2dabca3552d 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -11,6 +11,7 @@ module Banzai # Extends HTML::Pipeline::SanitizationFilter with common rules. class BaseSanitizationFilter < HTML::Pipeline::SanitizationFilter include Gitlab::Utils::StrongMemoize + extend Gitlab::Utils::SanitizeNodeLink UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze @@ -40,7 +41,7 @@ module Banzai # Allow any protocol in `a` elements # and then remove links with unsafe protocols whitelist[:protocols].delete('a') - whitelist[:transformers].push(self.class.remove_unsafe_links) + whitelist[:transformers].push(self.class.method(:remove_unsafe_links)) # Remove `rel` attribute from `a` elements whitelist[:transformers].push(self.class.remove_rel) @@ -54,35 +55,6 @@ module Banzai end class << self - def remove_unsafe_links - lambda do |env| - node = env[:node] - - return unless node.name == 'a' - return unless node.has_attribute?('href') - - begin - node['href'] = node['href'].strip - uri = Addressable::URI.parse(node['href']) - - return unless uri.scheme - - # Remove all invalid scheme characters before checking against the - # list of unsafe protocols. - # - # See https://tools.ietf.org/html/rfc3986#section-3.1 - scheme = uri.scheme - .strip - .downcase - .gsub(/[^A-Za-z0-9\+\.\-]+/, '') - - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') - end - end - end - def remove_rel lambda do |env| if env[:node_name] == 'a' diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index c3e5ac41cb8..e1d7b36b9a2 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -19,12 +19,11 @@ module Banzai end def find_object(project, id) - return unless project.is_a?(Project) + return unless project.is_a?(Project) && project.valid_repo? - if project && project.valid_repo? - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 - Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } - end + _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + + record end def referenced_merge_request_commit_shas @@ -66,6 +65,14 @@ module Banzai private + def record_identifier(record) + record.id + end + + def parent_records(parent, ids) + parent.commits_by(oids: ids.to_a) + end + def noteable context[:noteable] end diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb index 97394fd8f82..9f1ef0796f0 100644 --- a/lib/banzai/filter/inline_embeds_filter.rb +++ b/lib/banzai/filter/inline_embeds_filter.rb @@ -10,8 +10,6 @@ module Banzai # the link, and insert this node after any html content # surrounding the link. def call - return doc unless Feature.enabled?(:gfm_embedded_metrics, context[:project]) - doc.xpath(xpath_search).each do |node| next unless element = element_to_embed(node) diff --git a/lib/banzai/filter/inline_metrics_filter.rb b/lib/banzai/filter/inline_metrics_filter.rb index 0120cc37d6f..c5a328c21b2 100644 --- a/lib/banzai/filter/inline_metrics_filter.rb +++ b/lib/banzai/filter/inline_metrics_filter.rb @@ -15,17 +15,6 @@ module Banzai ) end - # Endpoint FE should hit to collect the appropriate - # chart information - def metrics_dashboard_url(params) - Gitlab::Metrics::Dashboard::Url.build_dashboard_url( - params['namespace'], - params['project'], - params['environment'], - embedded: true - ) - end - # Search params for selecting metrics links. A few # simple checks is enough to boost performance without # the cost of doing a full regex match. @@ -38,6 +27,28 @@ module Banzai def link_pattern Gitlab::Metrics::Dashboard::Url.regex end + + private + + # Endpoint FE should hit to collect the appropriate + # chart information + def metrics_dashboard_url(params) + Gitlab::Metrics::Dashboard::Url.build_dashboard_url( + params['namespace'], + params['project'], + params['environment'], + embedded: true, + **query_params(params['url']) + ) + end + + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def query_params(url) + Gitlab::Metrics::Dashboard::Url.parse_query(url) + end end end end diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb index ff91be2cbb7..4d8a5028898 100644 --- a/lib/banzai/filter/inline_metrics_redactor_filter.rb +++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb @@ -13,8 +13,6 @@ module Banzai # uses to identify the embedded content, removing # only unnecessary nodes. def call - return doc unless Feature.enabled?(:gfm_embedded_metrics, context[:project]) - nodes.each do |node| path = paths_by_node[node] user_has_access = user_access_by_path[path] diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb index 2963cba91e8..b91ba9f7256 100644 --- a/lib/banzai/filter/issuable_reference_filter.rb +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -3,22 +3,8 @@ module Banzai module Filter class IssuableReferenceFilter < AbstractReferenceFilter - def records_per_parent - @records_per_project ||= {} - - @records_per_project[object_class.to_s.underscore] ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - parent_per_reference.each do |path, parent| - record_ids = references_per_parent[path] - - parent_records(parent, record_ids).each do |record| - hash[parent][record.iid.to_i] = record - end - end - - hash - end + def record_identifier(record) + record.iid.to_i end def find_object(parent, iid) diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 1728a442533..18947679b69 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -8,15 +8,19 @@ module Banzai # Context options: # :project_wiki class WikiLinkFilter < HTML::Pipeline::Filter + include Gitlab::Utils::SanitizeNodeLink + def call return doc unless project_wiki? - doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) } - doc.search('video').each { |el| process_link_attr(el.attribute('src')) } + doc.search('a:not(.gfm)').each { |el| process_link(el.attribute('href'), el) } + + doc.search('video').each { |el| process_link(el.attribute('src'), el) } + doc.search('img').each do |el| attr = el.attribute('data-src') || el.attribute('src') - process_link_attr(attr) + process_link(attr, el) end doc @@ -24,6 +28,11 @@ module Banzai protected + def process_link(link_attr, node) + process_link_attr(link_attr) + remove_unsafe_links({ node: node }, remove_invalid_links: false) + end + def project_wiki? !context[:project_wiki].nil? end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 77b5053f38c..f4cc8beeb52 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -4,8 +4,6 @@ module Banzai module Filter class WikiLinkFilter < HTML::Pipeline::Filter class Rewriter - UNSAFE_SLUG_REGEXES = [/\Ajavascript:/i].freeze - def initialize(link_string, wiki:, slug:) @uri = Addressable::URI.parse(link_string) @wiki_base_path = wiki && wiki.wiki_base_path @@ -37,8 +35,6 @@ module Banzai # Of the form `./link`, `../link`, or similar def apply_hierarchical_link_rules! - return if slug_considered_unsafe? - @uri = Addressable::URI.join(@slug, @uri) if @uri.to_s[0] == '.' end @@ -58,10 +54,6 @@ module Banzai def repository_upload? @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) end - - def slug_considered_unsafe? - UNSAFE_SLUG_REGEXES.any? { |r| r.match?(@slug) } - end end end end diff --git a/lib/banzai/reference_redactor.rb b/lib/banzai/reference_redactor.rb index eb5c35da375..936436982e7 100644 --- a/lib/banzai/reference_redactor.rb +++ b/lib/banzai/reference_redactor.rb @@ -33,7 +33,7 @@ module Banzai # # data - An Array of a Hashes mapping an HTML document to nodes to redact. def redact_document_nodes(all_document_nodes) - all_nodes = all_document_nodes.map { |x| x[:nodes] }.flatten + all_nodes = all_document_nodes.flat_map { |x| x[:nodes] } visible = nodes_visible_to_user(all_nodes) metadata = [] diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index 1b8dc27793a..127598a76ab 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class Comment < Representation::Base def author - user['username'] + user['nickname'] end def note diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index a88797cdab9..3f6db9cb75b 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -14,7 +14,7 @@ module Bitbucket end def author - raw.dig('reporter', 'username') + raw.dig('reporter', 'nickname') end def description diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 6a0e8b354bf..a498c9bc213 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -4,7 +4,7 @@ module Bitbucket module Representation class PullRequest < Representation::Base def author - raw.fetch('author', {}).fetch('username', nil) + raw.fetch('author', {}).fetch('nickname', nil) end def description diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index 6a608058813..cf55c692271 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -23,8 +23,9 @@ module BitbucketServer BitbucketServer::Representation::Repo.new(parsed_response) end - def repos(page_offset: 0, limit: nil) + def repos(page_offset: 0, limit: nil, filter: nil) path = "/repos" + path += "?name=#{filter}" if filter get_collection(path, :repo, page_offset: page_offset, limit: limit) end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c80f49f5ae0..82810ea4076 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -7,7 +7,9 @@ module ContainerRegistry class Client attr_accessor :uri - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze + DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' + OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' + ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] @@ -60,12 +62,13 @@ module ContainerRegistry end def accept_manifest(conn) - conn.headers['Accept'] = MANIFEST_VERSION + conn.headers['Accept'] = ACCEPTED_TYPES conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' + conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE + conn.response :json, content_type: OCI_MANIFEST_V1_TYPE end def response_body(response, allow_redirect: false) @@ -79,7 +82,10 @@ module ContainerRegistry def redirect_response(location) return unless location - faraday_redirect.get(location) + uri = URI(@base_uri).merge(location) + raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme) + + faraday_redirect.get(uri) end def faraday diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index ef41dc560c9..ebea84fa1ca 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -6,6 +6,9 @@ module ContainerRegistry attr_reader :repository, :name + # https://github.com/docker/distribution/commit/3150937b9f2b1b5b096b2634d0e7c44d4a0f89fb + TAG_NAME_REGEX = /^[\w][\w.-]{0,127}$/.freeze + delegate :registry, :client, to: :repository delegate :revision, :short_revision, to: :config_blob, allow_nil: true @@ -13,6 +16,10 @@ module ContainerRegistry @repository, @name = repository, name end + def valid_name? + !name.match(TAG_NAME_REGEX).nil? + end + def valid? manifest.present? end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index c83cec9dc4a..45af30f46dc 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -3,6 +3,20 @@ module ExpandVariables class << self def expand(value, variables) + variables_hash = nil + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables_hash ||= transform_variables(variables) + variables_hash[$1 || $2] + end + end + + private + + def transform_variables(variables) + # Lazily initialise variables + variables = variables.call if variables.is_a?(Proc) + # Convert hash array to variables if variables.is_a?(Array) variables = variables.reduce({}) do |hash, variable| @@ -11,9 +25,7 @@ module ExpandVariables end end - value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do - variables[$1 || $2] - end + variables end end end diff --git a/lib/feature.rb b/lib/feature.rb index e28333aa58e..c70a6980f19 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -80,6 +80,13 @@ class Feature get(key).disable_group(group) end + def remove(key) + feature = get(key) + return unless persisted?(feature) + + feature.remove + end + def flipper if Gitlab::SafeRequestStore.active? Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index edfd2fb17f3..9ded1aed4e3 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -7,7 +7,7 @@ class Feature # Server feature flags should use '_' to separate words. SERVER_FEATURE_FLAGS = [ - 'get_commit_signatures'.freeze + # 'get_commit_signatures'.freeze ].freeze DEFAULT_ON_FLAGS = Set.new([]).freeze diff --git a/lib/forever.rb b/lib/forever.rb index 0a37118fe68..3f923557441 100644 --- a/lib/forever.rb +++ b/lib/forever.rb @@ -1,15 +1,9 @@ # frozen_string_literal: true class Forever - POSTGRESQL_DATE = DateTime.new(3000, 1, 1) - MYSQL_DATE = DateTime.new(2038, 01, 19) + DATE = DateTime.new(3000, 1, 1) - # MySQL timestamp has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC def self.date - if Gitlab::Database.postgresql? - POSTGRESQL_DATE - else - MYSQL_DATE - end + DATE end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index c62d1071dba..d9d8dcf7900 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency File.expand_path('gitlab/popen', __dir__) +require 'pathname' module Gitlab def self.root @@ -61,7 +61,7 @@ module Gitlab def self.ee? @is_ee ||= - if ENV['IS_GITLAB_EE'].present? + if ENV['IS_GITLAB_EE'] && !ENV['IS_GITLAB_EE'].empty? Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) else # We may use this method when the Rails environment is not loaded. This diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6eb08f674c2..7ef9f7ef630 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -29,6 +29,10 @@ module Gitlab MAINTAINER_PROJECT_ACCESS = 1 DEVELOPER_MAINTAINER_PROJECT_ACCESS = 2 + # Default subgroup creation level + OWNER_SUBGROUP_ACCESS = 0 + MAINTAINER_SUBGROUP_ACCESS = 1 + class << self delegate :values, to: :options @@ -106,6 +110,13 @@ module Gitlab def project_creation_level_name(name) project_creation_options.key(name) end + + def subgroup_creation_options + { + s_('SubgroupCreationlevel|Owners') => OWNER_SUBGROUP_ACCESS, + s_('SubgroupCreationlevel|Maintainers') => MAINTAINER_SUBGROUP_ACCESS + } + end end def human_access diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index c442211e073..0e8707af631 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -33,16 +33,48 @@ module Gitlab # Increments the given key and returns true if the action should # be throttled. # - # key - An array of ActiveRecord instances - # threshold_value - The maximum number of times this action should occur in the given time interval + # key - An array of ActiveRecord instances or strings + # threshold_value - The maximum number of times this action should occur in the given time interval. If number is zero is considered disabled. def throttled?(key, threshold_value) - self.increment(key) > threshold_value + threshold_value > 0 && + self.increment(key) > threshold_value + end + + # Logs request into auth.log + # + # request - Web request to be logged + # type - A symbol key that represents the request. + # current_user - Current user of the request, it can be nil. + def log_request(request, type, current_user) + request_information = { + message: 'Action_Rate_Limiter_Request', + env: type, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath + } + + if current_user + request_information.merge!({ + user_id: current_user.id, + username: current_user.username + }) + end + + Gitlab::AuthLogger.error(request_information) end private def action_key(key) - serialized = key.map { |obj| "#{obj.class.model_name.to_s.underscore}:#{obj.id}" }.join(":") + serialized = key.map do |obj| + if obj.is_a?(String) + "#{obj}" + else + "#{obj.class.model_name.to_s.underscore}:#{obj.id}" + end + end.join(":") + "action_rate_limiter:#{action}:#{serialized}" end end diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb index 558628b5422..988ff196193 100644 --- a/lib/gitlab/auth/activity.rb +++ b/lib/gitlab/auth/activity.rb @@ -37,14 +37,17 @@ module Gitlab def user_authenticated! self.class.user_authenticated_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + end end def user_session_override! self.class.user_session_override_counter_increment! case @opts[:message] - when :two_factor_authenticated - self.class.user_two_factor_authenticated_counter_increment! when :sessionless_sign_in self.class.user_sessionless_authentication_counter_increment! end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 72a187377d0..91b9ddc0d00 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -60,8 +60,7 @@ module Gitlab def get_info(key) value = info[key] - Gitlab::Utils.force_utf8(value) if value - value + value.is_a?(String) ? Gitlab::Utils.force_utf8(value) : value end def username_and_email diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index a5efe33bdc6..bba7e2cbb3c 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -90,8 +90,8 @@ module Gitlab def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || - current_request.env[PRIVATE_TOKEN_HEADER].presence - + current_request.env[PRIVATE_TOKEN_HEADER].presence || + parsed_oauth_token return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` @@ -99,9 +99,12 @@ module Gitlab end def find_oauth_access_token - token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + token = parsed_oauth_token return unless token + # PATs with OAuth headers are not handled by OauthAccessToken + return if matches_personal_access_token_length?(token) + # Expiration, revocation and scopes are verified in `validate_access_token!` oauth_token = OauthAccessToken.by_token(token) raise UnauthorizedError unless oauth_token @@ -110,6 +113,14 @@ module Gitlab oauth_token end + def parsed_oauth_token + Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) + end + + def matches_personal_access_token_length?(token) + token.length == PersonalAccessToken::TOKEN_LENGTH + end + # Check if the request is GET/HEAD, or if CSRF token is valid. def verified_request? Gitlab::RequestForgeryProtection.verified?(current_request.env) diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index c8d83cc1803..1d9aa050041 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -40,7 +40,7 @@ module Gitlab end def reload! - @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + @shards = Hash[*Shard.all.flat_map { |shard| [shard.name, shard.id] }] end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 6046d33aeac..4016b807f21 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -19,18 +19,11 @@ module Gitlab def perform(start_id, stop_id) PagesDomain.where(id: start_id..stop_id).find_each do |domain| - if Gitlab::Database.mysql? - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before, - certificate_valid_not_after: domain.x509&.not_after - ) - else - # for some reason activerecord doesn't append timezone, iso8601 forces this - domain.update_columns( - certificate_valid_not_before: domain.x509&.not_before&.iso8601, - certificate_valid_not_after: domain.x509&.not_after&.iso8601 - ) - end + # for some reason activerecord doesn't append timezone, iso8601 forces this + domain.update_columns( + certificate_valid_not_before: domain.x509&.not_before&.iso8601, + certificate_valid_not_after: domain.x509&.not_after&.iso8601 + ) rescue => e Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger end diff --git a/lib/gitlab/background_migration/legacy_upload_mover.rb b/lib/gitlab/background_migration/legacy_upload_mover.rb new file mode 100644 index 00000000000..051c1176edb --- /dev/null +++ b/lib/gitlab/background_migration/legacy_upload_mover.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class takes a legacy upload and migrates it to the correct location + class LegacyUploadMover + include Gitlab::Utils::StrongMemoize + + attr_reader :upload, :project, :note + attr_accessor :logger + + def initialize(upload) + @upload = upload + @note = Note.find_by(id: upload.model_id) + @project = note&.project + @logger = Gitlab::BackgroundMigration::Logger.build + end + + def execute + return unless upload + + if !project + # if we don't have models associated with the upload we can not move it + warn('Deleting upload due to model not found.') + + destroy_legacy_upload + elsif note.is_a?(LegacyDiffNote) + return unless move_legacy_diff_file + + migrate_upload + elsif !legacy_file_exists? + warn('Deleting upload due to file not found.') + destroy_legacy_upload + else + migrate_upload + end + end + + private + + def migrate_upload + return unless copy_upload_to_project + + add_upload_link_to_note_text + destroy_legacy_file + destroy_legacy_upload + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def copy_upload_to_project + @uploader = FileUploader.copy_to(legacy_file_uploader, project) + + logger.info( + message: 'MigrateLegacyUploads: File copied successfully', + old_path: legacy_file_uploader.file.path, new_path: @uploader.file.path + ) + true + rescue Exception => e + warn( + 'File could not be copied to project uploads', + file_path: legacy_file_uploader.file.path, error: e.message + ) + false + end + # rubocop: enable Lint/RescueException + + def destroy_legacy_upload + if note + note.remove_attachment = true + note.save + end + + if upload.destroy + logger.info(message: 'MigrateLegacyUploads: Upload was destroyed.', upload: upload.inspect) + else + warn('MigrateLegacyUploads: Upload destroy failed.') + end + end + + def destroy_legacy_file + legacy_file_uploader.file.delete + end + + def add_upload_link_to_note_text + new_text = "#{note.note} \n #{@uploader.markdown_link}" + # Bypass validations because old data may have invalid + # noteable values. If we fail hard here, we may kill the + # entire background migration, which affects a range of notes. + note.update_attribute(:note, new_text) + end + + def legacy_file_uploader + strong_memoize(:legacy_file_uploader) do + uploader = upload.build_uploader + uploader.retrieve_from_store!(File.basename(upload.path)) + uploader + end + end + + def legacy_file_exists? + legacy_file_uploader.file.exists? + end + + # we should proceed and log whenever one upload copy fails, no matter the reasons + # rubocop: disable Lint/RescueException + def move_legacy_diff_file + old_path = upload.absolute_path + old_path_sub = '-/system/note/attachment' + + if !File.exist?(old_path) || !old_path.include?(old_path_sub) + log_legacy_diff_note_problem(old_path) + return false + end + + new_path = upload.absolute_path.sub(old_path_sub, '-/system/legacy_diff_note/attachment') + new_dir = File.dirname(new_path) + FileUtils.mkdir_p(new_dir) + + FileUtils.mv(old_path, new_path) + rescue Exception => e + log_legacy_diff_note_problem(old_path, new_path, e) + false + end + + def warn(message, params = {}) + logger.warn( + params.merge(message: "MigrateLegacyUploads: #{message}", upload: upload.inspect) + ) + end + + def log_legacy_diff_note_problem(old_path, new_path = nil, error = nil) + warn('LegacyDiffNote upload could not be moved to a new path', + old_path: old_path, new_path: new_path, error: error&.message + ) + end + # rubocop: enable Lint/RescueException + end + end +end diff --git a/lib/gitlab/background_migration/legacy_uploads_migrator.rb b/lib/gitlab/background_migration/legacy_uploads_migrator.rb new file mode 100644 index 00000000000..a9d38a27e0c --- /dev/null +++ b/lib/gitlab/background_migration/legacy_uploads_migrator.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration takes all legacy uploads (that were uploaded using AttachmentUploader) + # and migrate them to the new (FileUploader) location (=under projects). + # + # We have dependencies (uploaders) in this migration because extracting code would add a lot of complexity + # and possible errors could appear as the logic in the uploaders is not trivial. + # + # This migration will be removed in 13.0 in order to get rid of a migration that depends on + # the application code. + class LegacyUploadsMigrator + include Database::MigrationHelpers + + def perform(start_id, end_id) + Upload.where(id: start_id..end_id, uploader: 'AttachmentUploader').find_each do |upload| + LegacyUploadMover.new(upload).execute + end + end + end + end +end diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb new file mode 100644 index 00000000000..4ea89771eff --- /dev/null +++ b/lib/gitlab/background_migration/logger.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Logger that can be used for migrations logging + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'migrations' + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb index 5cd638083b0..4377ec2987c 100644 --- a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -39,10 +39,10 @@ module Gitlab SELECT project_id, id, - artifacts_expire_at, + artifacts_expire_at #{add_missing_db_timezone}, #{LEGACY_PATH_FILE_LOCATION}, - created_at, - created_at, + created_at #{add_missing_db_timezone}, + created_at #{add_missing_db_timezone}, artifacts_file, artifacts_size, COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), @@ -81,10 +81,10 @@ module Gitlab SELECT project_id, id, - artifacts_expire_at, + artifacts_expire_at #{add_missing_db_timezone}, #{LEGACY_PATH_FILE_LOCATION}, - created_at, - created_at, + created_at #{add_missing_db_timezone}, + created_at #{add_missing_db_timezone}, artifacts_metadata, NULL, COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), @@ -121,6 +121,12 @@ module Gitlab AND artifacts_file <> '' SQL end + + def add_missing_db_timezone + return '' unless Gitlab::Database.postgresql? + + 'at time zone \'UTC\'' + end end end end diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb index f921233460d..55608529cee 100644 --- a/lib/gitlab/background_migration/migrate_stage_index.rb +++ b/lib/gitlab/background_migration/migrate_stage_index.rb @@ -13,34 +13,22 @@ module Gitlab private def migrate_stage_index_sql(start_id, stop_id) - if Gitlab::Database.postgresql? - <<~SQL - WITH freqs AS ( - SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds - WHERE stage_id BETWEEN #{start_id} AND #{stop_id} - AND stage_idx IS NOT NULL - GROUP BY stage_id, stage_idx - ), indexes AS ( - SELECT DISTINCT stage_id, first_value(stage_idx) - OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index - FROM freqs - ) + <<~SQL + WITH freqs AS ( + SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds + WHERE stage_id BETWEEN #{start_id} AND #{stop_id} + AND stage_idx IS NOT NULL + GROUP BY stage_id, stage_idx + ), indexes AS ( + SELECT DISTINCT stage_id, first_value(stage_idx) + OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index + FROM freqs + ) - UPDATE ci_stages SET position = indexes.index - FROM indexes WHERE indexes.stage_id = ci_stages.id - AND ci_stages.position IS NULL; - SQL - else - <<~SQL - UPDATE ci_stages - SET position = - (SELECT stage_idx FROM ci_builds - WHERE ci_builds.stage_id = ci_stages.id - GROUP BY ci_builds.stage_idx ORDER BY COUNT(*) DESC LIMIT 1) - WHERE ci_stages.id BETWEEN #{start_id} AND #{stop_id} - AND ci_stages.position IS NULL - SQL - end + UPDATE ci_stages SET position = indexes.index + FROM indexes WHERE indexes.stage_id = ci_stages.id + AND ci_stages.position IS NULL; + SQL end end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index 1924f2ffee2..f5fb33f1660 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -176,23 +176,12 @@ module Gitlab self.table_name = 'projects' def self.find_by_full_path(path) - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" + order_sql = "(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)" where_full_path_in(path).reorder(order_sql).take end def self.where_full_path_in(path) - cast_lower = Gitlab::Database.postgresql? - - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - + where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index cce2a82c098..2ac51dd7b55 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -55,7 +55,7 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless ActiveRecord::Base.connection.data_source_exists?(table_name) + unless ActiveRecord::Base.connection.table_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true @@ -133,12 +133,9 @@ module Gitlab def insert_sql(file_paths) if postgresql_pre_9_5? "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? + else "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ " ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO"\ - " #{table_columns_and_values_for_insert(file_paths)};" end end @@ -150,19 +147,13 @@ module Gitlab "#{UntrackedFile.table_name} (path) VALUES #{values}" end - def postgresql? - strong_memoize(:postgresql) do - Gitlab::Database.postgresql? - end - end - def can_bulk_insert_and_ignore_duplicates? !postgresql_pre_9_5? end def postgresql_pre_9_5? strong_memoize(:postgresql_pre_9_5) do - postgresql? && Gitlab::Database.version.to_f < 9.5 + Gitlab::Database.version.to_f < 9.5 end end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 47579d46c1b..9ef6d8654ae 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -50,14 +50,7 @@ module Gitlab private def remove_non_members_todos(project_id) - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id) - else - unauthorized_project_todos(project_id) - .each_batch(of: 5000) do |batch| - batch.delete_all - end - end + batch_remove_todos_cte(project_id) end def remove_confidential_issue_todos(project_id) @@ -90,13 +83,7 @@ module Gitlab next if target_types.empty? - if Gitlab::Database.postgresql? - batch_remove_todos_cte(project_id, target_types) - else - unauthorized_project_todos(project_id) - .where(target_type: target_types) - .delete_all - end + batch_remove_todos_cte(project_id, target_types) end end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 7f7cc62c8ef..15cccc6f287 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -14,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.ci_pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 8047ef4fa15..24bc73e0de5 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -262,13 +262,19 @@ module Gitlab def pull_request_comment_attributes(comment) { project: project, - note: comment.note, author_id: gitlab_user_id(project, comment.author), + note: comment_note(comment), created_at: comment.created_at, updated_at: comment.updated_at } end + def comment_note(comment) + author = @formatter.author_line(comment.author) unless find_user_id(comment.author) + + author.to_s + comment.note.to_s + end + def log_error(details) logger.error(log_base_data.merge(details)) end diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index d3e15a79a8b..fc579ad8d2a 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -45,7 +45,7 @@ module Gitlab end def image? - ['.png', '.jpg', '.jpeg', '.gif'].include?(extname.downcase) + ['.png', '.jpg', '.jpeg', '.gif', '.svg'].include?(extname.downcase) end # Internal: Lookup mime type for extension. diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb new file mode 100644 index 00000000000..4f47cdef971 --- /dev/null +++ b/lib/gitlab/chaos.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + # Chaos methods for GitLab. + # See https://docs.gitlab.com/ee/development/chaos_endpoints.html for more details. + class Chaos + # leak_mem will retain the specified amount of memory and sleep. + # On return, the memory will be released. + def self.leak_mem(memory_mb, duration_s) + start_time = Time.now + + retainer = [] + # Add `n` 1mb chunks of memory to the retainer array + memory_mb.times { retainer << "x" * 1.megabyte } + + duration_left = [start_time + duration_s - Time.now, 0].max + Kernel.sleep(duration_left) + end + + # cpu_spin will consume all CPU on a single core for the specified duration + def self.cpu_spin(duration_s) + expected_end_time = Time.now + duration_s + + rand while Time.now < expected_end_time + end + + # db_spin will query the database in a tight loop for the specified duration + def self.db_spin(duration_s, interval_s) + expected_end_time = Time.now + duration_s + + while Time.now < expected_end_time + ActiveRecord::Base.connection.execute("SELECT 1") + + end_interval_time = Time.now + [duration_s, interval_s].min + rand while Time.now < end_interval_time + end + end + + # sleep will sleep for the specified duration + def self.sleep(duration_s) + Kernel.sleep(duration_s) + end + + # Kill will send a SIGKILL signal to the current process + def self.kill + Process.kill("KILL", Process.pid) + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index fc3223e7442..b7886114e9c 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -194,16 +194,10 @@ module Gitlab end def handle_new_line - css_classes = [] - - if @sections.any? - css_classes = %w[section line] + sections.map { |section| "s_#{section}" } - end - write_in_tag %{<br/>} - write_raw %{<span class="#{css_classes.join(' ')}"></span>} if css_classes.any? + + close_open_tags if @sections.any? && @lineno_in_section == 0 @lineno_in_section += 1 - open_new_tag end def handle_section(scanner) @@ -224,7 +218,7 @@ module Gitlab return if @sections.include?(section) @sections << section - write_raw %{<div class="js-section-start fa fa-caret-down append-right-8 cursor-pointer" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} + write_raw %{<div class="js-section-start section-start fa fa-caret-down pr-2 cursor-pointer" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} @lineno_in_section = 0 end @@ -310,11 +304,24 @@ module Gitlab if @sections.any? css_classes << "section" - css_classes << "js-section-header section-header" if @lineno_in_section == 0 + + css_classes << if @lineno_in_section == 0 + "js-section-header section-header cursor-pointer" + else + "line" + end + css_classes += sections.map { |section| "js-s-#{section}" } end - @out << %{<span class="#{css_classes.join(' ')}">} + close_open_tags + + @out << if css_classes.any? + %{<span class="#{css_classes.join(' ')}">} + else + %{<span>} + end + @n_open_tags += 1 end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 0698136166a..e9c8864123f 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - variables = seed.to_resource.scoped_variables_hash + variables = seed.scoped_variables_hash statements = @expressions.map do |statement| ::Gitlab::Ci::Pipeline::Expression::Statement diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index e6e0aaab60b..6ab4fca3854 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -8,31 +8,51 @@ module Gitlab def unmet? deployment_cluster.present? && deployment_cluster.managed? && - (kubernetes_namespace.new_record? || kubernetes_namespace.service_account_token.blank?) + missing_namespace? end def complete! return unless unmet? - create_or_update_namespace + create_namespace end private + def missing_namespace? + kubernetes_namespace.nil? || kubernetes_namespace.service_account_token.blank? + end + def deployment_cluster build.deployment&.cluster end + def environment + build.deployment.environment + end + def kubernetes_namespace strong_memoize(:kubernetes_namespace) do - deployment_cluster.find_or_initialize_kubernetes_namespace_for_project(build.project) + Clusters::KubernetesNamespaceFinder.new( + deployment_cluster, + project: environment.project, + environment_slug: environment.slug, + allow_blank_token: true + ).execute end end - def create_or_update_namespace + def create_namespace Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, - kubernetes_namespace: kubernetes_namespace + kubernetes_namespace: kubernetes_namespace || build_namespace_record + ).execute + end + + def build_namespace_record + Clusters::BuildKubernetesNamespaceService.new( + deployment_cluster, + environment: environment ).execute end end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb new file mode 100644 index 00000000000..89623a809c9 --- /dev/null +++ b/lib/gitlab/ci/build/rules.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules + include ::Gitlab::Utils::StrongMemoize + + Result = Struct.new(:when, :start_in) + + def initialize(rule_hashes, default_when = 'on_success') + @rule_list = Rule.fabricate_list(rule_hashes) + @default_when = default_when + end + + def evaluate(pipeline, build) + if @rule_list.nil? + Result.new(@default_when) + elsif matched_rule = match_rule(pipeline, build) + Result.new( + matched_rule.attributes[:when] || @default_when, + matched_rule.attributes[:start_in] + ) + else + Result.new('never') + end + end + + private + + def match_rule(pipeline, build) + @rule_list.find { |rule| rule.matches?(pipeline, build) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule.rb b/lib/gitlab/ci/build/rules/rule.rb new file mode 100644 index 00000000000..8d52158c8d2 --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule + attr_accessor :attributes + + def self.fabricate_list(list) + list.map(&method(:new)) if list + end + + def initialize(spec) + @clauses = [] + @attributes = {} + + spec.each do |type, value| + if clause = Clause.fabricate(type, value) + @clauses << clause + else + @attributes.merge!(type => value) + end + end + end + + def matches?(pipeline, build) + @clauses.all? { |clause| clause.satisfied_by?(pipeline, build) } + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause.rb b/lib/gitlab/ci/build/rules/rule/clause.rb new file mode 100644 index 00000000000..ff0baf3348c --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause + ## + # Abstract class that defines an interface of a single + # job rule specification. + # + # Used for job's inclusion rules configuration. + # + UnknownClauseError = Class.new(StandardError) + + def self.fabricate(type, value) + type = type.to_s.camelize + + self.const_get(type).new(value) if self.const_defined?(type) + end + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline, seed = nil) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb new file mode 100644 index 00000000000..81d2ee6c24c --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::Changes < Rules::Rule::Clause + def initialize(globs) + @globs = Array(globs) + end + + def satisfied_by?(pipeline, seed) + return true if pipeline.modified_paths.nil? + + pipeline.modified_paths.any? do |path| + @globs.any? do |glob| + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb new file mode 100644 index 00000000000..18c3b450f95 --- /dev/null +++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Rules::Rule::Clause::If < Rules::Rule::Clause + def initialize(expression) + @expression = expression + end + + def satisfied_by?(pipeline, seed) + variables = seed.scoped_variables_hash + + ::Gitlab::Ci::Pipeline::Expression::Statement.new(@expression, variables).truthful? + end + end + end + end +end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 7cabaadb122..3fbfdffe277 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -21,16 +21,10 @@ module Gitlab module MonthlyInterval # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) - if Gitlab::Database.postgresql? - query - .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - else - query - .group("DATE_FORMAT(#{::Ci::Pipeline.table_name}.created_at, '01 %M %Y')") - .count(:created_at) - end + query + .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") + .count(:created_at) + .transform_keys(&:squish) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 5ab795359b8..6e11c582750 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,17 +11,28 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[tags script only except type image services + ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze + ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache - dependencies before_script after_script variables + dependencies needs before_script after_script variables environment coverage retry parallel extends].freeze + REQUIRED_BY_NEEDS = %i[stage].freeze + validations do + validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :config, presence: true validates :script, presence: true validates :name, presence: true validates :name, type: Symbol + validates :config, + disallowed_keys: { + in: %i[only except when start_in], + message: 'key may not be used with `rules`' + }, + if: :has_rules? with_options allow_nil: true do validates :tags, array_of_strings: true @@ -29,16 +40,29 @@ module Gitlab validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, less_than_or_equal_to: 50 } - validates :when, - inclusion: { in: %w[on_success on_failure always manual delayed], - message: 'should be on_success, on_failure, ' \ - 'always, manual or delayed' } + validates :when, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } + validates :dependencies, array_of_strings: true + validates :needs, array_of_strings: true validates :extends, array_of_strings_or_string: true + validates :rules, array_of_hashes: true end validates :start_in, duration: { limit: '1 day' }, if: :delayed? - validates :start_in, absence: true, unless: :delayed? + validates :start_in, absence: true, if: -> { has_rules? || !delayed? } + + validate do + next unless dependencies.present? + next unless needs.present? + + missing_needs = dependencies - needs + if missing_needs.any? + errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") + end + end end entry :before_script, Entry::Script, @@ -77,6 +101,9 @@ module Gitlab entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' + entry :rules, Entry::Rules, + description: 'List of evaluable Rules to determine job inclusion.' + entry :variables, Entry::Variables, description: 'Environment variables available for this job.' @@ -95,10 +122,10 @@ module Gitlab helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, - :parallel + :parallel, :needs attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :parallel, :extends, :start_in + :needs, :retry, :parallel, :extends, :start_in, :rules def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -137,6 +164,10 @@ module Gitlab self.when == 'delayed' end + def has_rules? + @config.try(:key?, :rules) + end + def ignored? allow_failure.nil? ? manual_action? : allow_failure end @@ -178,7 +209,8 @@ module Gitlab parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, - ignore: ignored? } + ignore: ignored?, + needs: needs_defined? ? needs_value : nil } end end end diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb new file mode 100644 index 00000000000..65cad0880f5 --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: Array + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |rule, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule) + .value(rule) + .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb new file mode 100644 index 00000000000..1f2a34ec90e --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules::Rule < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + CLAUSES = %i[if changes].freeze + ALLOWED_KEYS = %i[if changes when start_in].freeze + ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze + + attributes :if, :changes, :when, :start_in + + validations do + validates :config, presence: true + validates :config, type: { with: Hash } + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? + validates :start_in, presence: true, if: :specifies_delay? + validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay? + + with_options allow_nil: true do + validates :if, expression: true + validates :changes, array_of_strings: true + validates :when, allowed_values: { in: ALLOWED_WHEN } + end + end + + def specifies_delay? + self.when == 'delayed' + end + + def default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 191f5d09645..09f9bf5f69f 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -4,61 +4,63 @@ module Gitlab module Ci class Config class Normalizer + include Gitlab::Utils::StrongMemoize + def initialize(jobs_config) @jobs_config = jobs_config end def normalize_jobs - extract_parallelized_jobs! - return @jobs_config if @parallelized_jobs.empty? + return @jobs_config if parallelized_jobs.empty? + + expand_parallelize_jobs do |job_name, config| + if config[:dependencies] + config[:dependencies] = expand_names(config[:dependencies]) + end - parallelized_config = parallelize_jobs - parallelize_dependencies(parallelized_config) + if config[:needs] + config[:needs] = expand_names(config[:needs]) + end + + config + end end private - def extract_parallelized_jobs! - @parallelized_jobs = {} + def expand_names(job_names) + return unless job_names - @jobs_config.each do |job_name, config| - if config[:parallel] - @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) - end + job_names.flat_map do |job_name| + parallelized_jobs[job_name.to_sym] || job_name end - - @parallelized_jobs end - def parallelize_jobs - @jobs_config.each_with_object({}) do |(job_name, config), hash| - if @parallelized_jobs.key?(job_name) - @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } - else - hash[job_name] = config - end + def parallelized_jobs + strong_memoize(:parallelized_jobs) do + @jobs_config.each_with_object({}) do |(job_name, config), hash| + next unless config[:parallel] - hash + hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end end end - def parallelize_dependencies(parallelized_config) - parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) - parallelized_config.each_with_object({}) do |(job_name, config), hash| - if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? - parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten - deps = config[:dependencies] - intersection + parallelized_deps - hash[job_name] = config.merge(dependencies: deps) + def expand_parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if parallelized_jobs.key?(job_name) + parallelized_jobs[job_name].each_with_index do |name, index| + hash[name.to_sym] = + yield(name, config.merge(name: name, instance: index + 1)) + end else - hash[job_name] = config + hash[job_name] = yield(job_name, config) end - - hash end end def self.parallelize_job_names(name, total) - Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 0405292a25b..65029f5ce7f 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -23,12 +23,17 @@ module Gitlab @command.seeds_block&.call(pipeline) ## - # Populate pipeline with all stages, and stages with builds. + # Gather all runtime build/stage errors # - pipeline.stage_seeds.each do |stage| - pipeline.stages << stage.to_resource + if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence + return error(seeds_errors.join("\n")) end + ## + # Populate pipeline with all stages, and stages with builds. + # + pipeline.stages = pipeline.stage_seeds.map(&:to_resource) + if pipeline.stages.none? return error('No stages / jobs for this pipeline.') end diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb index 1fd3a61017f..e9e22569ae0 100644 --- a/lib/gitlab/ci/pipeline/seed/base.rb +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -13,6 +13,10 @@ module Gitlab raise NotImplementedError end + def errors + raise NotImplementedError + end + def to_resource raise NotImplementedError end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index d8296940a04..1066331062b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -7,39 +7,65 @@ module Gitlab class Build < Seed::Base include Gitlab::Utils::StrongMemoize - delegate :dig, to: :@attributes + delegate :dig, to: :@seed_attributes - def initialize(pipeline, attributes) + # When the `ci_dag_limit_needs` is enabled it uses the lower limit + LOW_NEEDS_LIMIT = 5 + HARD_NEEDS_LIMIT = 50 + + def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline - @attributes = attributes + @seed_attributes = attributes + @previous_stages = previous_stages + @needs_attributes = dig(:needs_attributes) + + @using_rules = attributes.key?(:rules) + @using_only = attributes.key?(:only) + @using_except = attributes.key?(:except) @only = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:only)) @except = Gitlab::Ci::Build::Policy .fabricate(attributes.delete(:except)) + @rules = Gitlab::Ci::Build::Rules + .new(attributes.delete(:rules)) + end + + def name + dig(:name) end def included? strong_memoize(:inclusion) do - @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && - @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + if @using_rules + included_by_rules? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true + end + end + end + + def errors + return unless included? + + strong_memoize(:errors) do + needs_errors end end def attributes - @attributes.merge( - pipeline: @pipeline, - project: @pipeline.project, - user: @pipeline.user, - ref: @pipeline.ref, - tag: @pipeline.tag, - trigger_request: @pipeline.legacy_trigger, - protected: @pipeline.protected_ref? - ) + @seed_attributes + .deep_merge(pipeline_attributes) + .deep_merge(rules_attributes) end def bridge? - @attributes.to_h.dig(:options, :trigger).present? + attributes_hash = @seed_attributes.to_h + attributes_hash.dig(:options, :trigger).present? || + (attributes_hash.dig(:options, :bridge_needs).instance_of?(Hash) && + attributes_hash.dig(:options, :bridge_needs, :pipeline).present?) end def to_resource @@ -51,6 +77,77 @@ module Gitlab end end end + + def scoped_variables_hash + strong_memoize(:scoped_variables_hash) do + # This is a temporary piece of technical debt to allow us access + # to the CI variables to evaluate rules before we persist a Build + # with the result. We should refactor away the extra Build.new, + # but be able to get CI Variables directly from the Seed::Build. + ::Ci::Build.new( + @seed_attributes.merge(pipeline_attributes) + ).scoped_variables_hash + end + end + + private + + def all_of_only? + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } + end + + def none_of_except? + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } + end + + def needs_errors + return if @needs_attributes.nil? + + if @needs_attributes.size > max_needs_allowed + return [ + "#{name}: one job can only need #{max_needs_allowed} others, but you have listed #{@needs_attributes.size}. " \ + "See needs keyword documentation for more details" + ] + end + + @needs_attributes.flat_map do |need| + result = @previous_stages.any? do |stage| + stage.seeds_names.include?(need[:name]) + end + + "#{name}: needs '#{need[:name]}'" unless result + end.compact + end + + def max_needs_allowed + if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true) + LOW_NEEDS_LIMIT + else + HARD_NEEDS_LIMIT + end + end + + def pipeline_attributes + { + pipeline: @pipeline, + project: @pipeline.project, + user: @pipeline.user, + ref: @pipeline.ref, + tag: @pipeline.tag, + trigger_request: @pipeline.legacy_trigger, + protected: @pipeline.protected_ref? + } + end + + def included_by_rules? + rules_attributes[:when] != 'never' + end + + def rules_attributes + strong_memoize(:rules_attributes) do + @using_rules ? @rules.evaluate(@pipeline, self).to_h.compact : {} + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 9c15064756a..b600df2f656 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,12 +10,13 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(pipeline, attributes) + def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @attributes = attributes + @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(@pipeline, attributes) + Seed::Build.new(@pipeline, attributes, previous_stages) end end @@ -32,6 +33,18 @@ module Gitlab end end + def errors + strong_memoize(:errors) do + seeds.flat_map(&:errors).compact + end + end + + def seeds_names + strong_memoize(:seeds_names) do + seeds.map(&:name).to_set + end + end + def included? seeds.any? end @@ -39,13 +52,7 @@ module Gitlab def to_resource strong_memoize(:stage) do ::Ci::Stage.new(attributes).tap do |stage| - seeds.each do |seed| - if seed.bridge? - stage.bridges << seed.to_resource - else - stage.builds << seed.to_resource - end - end + stage.statuses = seeds.map(&:to_resource) end end end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index d01b09f1398..df572188194 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -10,7 +10,7 @@ module Gitlab image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') } end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 3446644eff8..2a0bf060c9b 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -34,11 +34,9 @@ module Gitlab def extended_statuses return @extended_statuses if defined?(@extended_statuses) - groups = self.class.extended_statuses.map do |group| + @extended_statuses = self.class.extended_statuses.flat_map do |group| Array(group).find { |status| status.matches?(@subject, @user) } - end - - @extended_statuses = groups.flatten.compact + end.compact end def self.extended_statuses diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cf3d261c1cb..5c1c0c142e5 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -50,13 +50,12 @@ variables: POSTGRES_DB: $CI_ENVIRONMENT_SLUG POSTGRES_VERSION: 9.6.2 - KUBERNETES_VERSION: 1.11.10 - HELM_VERSION: 2.14.0 - DOCKER_DRIVER: overlay2 ROLLOUT_RESOURCE_TYPE: deployment + DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + stages: - build - test diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index a09217e8cf0..b0a79950667 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,6 +2,8 @@ performance: stage: performance image: docker:stable allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 18f7290e1d9..8061da968ed 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,8 @@ build: stage: build image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + variables: + DOCKER_TLS_CERTDIR: "" services: - docker:stable-dind script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 005ea4b7a46..3adc6a72874 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -6,6 +6,7 @@ code_quality: - docker:stable-dind variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 108f0119ae1..a8ec2d4781d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,14 +1,17 @@ +.auto-deploy: + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.1.0" + review: + extends: .auto-deploy stage: review script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy persist_environment_url environment: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -27,13 +30,13 @@ review: - $REVIEW_DISABLED stop_review: + extends: .auto-deploy stage: cleanup variables: GIT_STRATEGY: none script: - - install_dependencies - - initialize_tiller - - delete + - auto-deploy initialize_tiller + - auto-deploy delete environment: name: review/$CI_COMMIT_REF_NAME action: stop @@ -57,15 +60,15 @@ stop_review: # STAGING_ENABLED. staging: + extends: .auto-deploy stage: staging script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy environment: name: staging url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN @@ -81,15 +84,15 @@ staging: # CANARY_ENABLED. canary: + extends: .auto-deploy stage: canary script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy canary + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy canary environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -102,18 +105,18 @@ canary: - $CANARY_ENABLED .production: &production_template + extends: .auto-deploy stage: production script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - delete canary - - delete rollout - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy + - auto-deploy delete canary + - auto-deploy delete rollout + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -152,17 +155,17 @@ production_manual: # This job implements incremental rollout on for every push to `master`. .rollout: &rollout_template + extends: .auto-deploy script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy rollout $ROLLOUT_PERCENTAGE - - scale stable $((100-ROLLOUT_PERCENTAGE)) - - delete canary - - persist_environment_url + - auto-deploy check_kube_domain + - auto-deploy download_chart + - auto-deploy ensure_namespace + - auto-deploy initialize_tiller + - auto-deploy create_secret + - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE + - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) + - auto-deploy delete canary + - auto-deploy persist_environment_url environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -240,330 +243,3 @@ rollout 100%: <<: *manual_rollout_template <<: *production_template allow_failure: false - -.deploy_helpers: &deploy_helpers | - [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - export TILLER_NAMESPACE=$KUBE_NAMESPACE - - function get_replicas() { - track="${1:-stable}" - percentage="${2:-100}" - - env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) - env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) - - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\${env_track}_REPLICAS - fi - fi - - replicas="${new_replicas:-1}" - replicas="$(($replicas * $percentage / 100))" - - # always return at least one replicas - if [[ $replicas -gt 0 ]]; then - echo "$replicas" - else - echo 1 - fi - } - - # Extracts variables prefixed with K8S_SECRET_ - # and creates a Kubernetes secret. - # - # e.g. If we have the following environment variables: - # K8S_SECRET_A=value1 - # K8S_SECRET_B=multi\ word\ value - # - # Then we will create a secret with the following key-value pairs: - # data: - # A: dmFsdWUxCg== - # B: bXVsdGkgd29yZCB2YWx1ZQo= - function create_application_secret() { - track="${1-stable}" - export APPLICATION_SECRET_NAME=$(application_secret_name "$track") - - env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables - - kubectl create secret \ - -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ - --from-env-file k8s_prefixed_variables -o yaml --dry-run | - kubectl replace -n "$KUBE_NAMESPACE" --force -f - - - export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) - - rm k8s_prefixed_variables - } - - function deploy_name() { - name="$CI_ENVIRONMENT_SLUG" - track="${1-stable}" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - echo $name - } - - function application_secret_name() { - track="${1-stable}" - name=$(deploy_name "$track") - - echo "${name}-secret" - } - - function deploy() { - track="${1-stable}" - percentage="${2:-100}" - name=$(deploy_name "$track") - - if [[ -z "$CI_COMMIT_TAG" ]]; then - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} - else - image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} - image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} - fi - - service_enabled="true" - postgres_enabled="$POSTGRES_ENABLED" - - # if track is different than stable, - # re-use all attached resources - if [[ "$track" != "stable" ]]; then - service_enabled="false" - postgres_enabled="false" - fi - - replicas=$(get_replicas "$track" "$percentage") - - if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then - secret_name='gitlab-registry' - else - secret_name='' - fi - - create_application_secret "$track" - - env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') - eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS - if [ -n "$env_ADDITIONAL_HOSTS" ]; then - additional_hosts="{$env_ADDITIONAL_HOSTS}" - elif [ -n "$ADDITIONAL_HOSTS" ]; then - additional_hosts="{$ADDITIONAL_HOSTS}" - fi - - if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then - echo "Deploying first release with database initialization..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.initializeCommand="$DB_INITIALIZE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - - echo "Deploying second release..." - helm upgrade --reuse-values \ - --wait \ - --set application.initializeCommand="" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - else - echo "Deploying new release..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$image_repository" \ - --set image.tag="$image_tag" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le-$CI_PROJECT_ID.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.migrateCommand="$DB_MIGRATE" \ - $HELM_UPGRADE_EXTRA_ARGS \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - - if [[ -z "$ROLLOUT_STATUS_DISABLED" ]]; then - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" - fi - } - - function scale() { - track="${1-stable}" - percentage="${2-100}" - name=$(deploy_name "$track") - - replicas=$(get_replicas "$track" "$percentage") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm upgrade --reuse-values \ - --wait \ - --set replicaCount="$replicas" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - } - - function install_dependencies() { - apk add -U openssl curl tar gzip bash ca-certificates git - curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add glibc-2.28-r0.apk - rm glibc-2.28-r0.apk - - curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx - mv linux-amd64/helm /usr/bin/ - mv linux-amd64/tiller /usr/bin/ - helm version --client - tiller -version - - curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" - chmod +x /usr/bin/kubectl - kubectl version --client - } - - function download_chart() { - if [[ ! -d chart ]]; then - auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} - auto_chart_name=$(basename $auto_chart) - auto_chart_name=${auto_chart_name%.tgz} - auto_chart_name=${auto_chart_name%.tar.gz} - else - auto_chart="chart" - auto_chart_name="chart" - fi - - helm init --client-only - helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} - if [[ ! -d "$auto_chart" ]]; then - helm fetch ${auto_chart} --untar - fi - if [ "$auto_chart_name" != "chart" ]; then - mv ${auto_chart_name} chart - fi - - helm dependency update chart/ - helm dependency build chart/ - } - - function ensure_namespace() { - kubectl get namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" - } - - function check_kube_domain() { - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then - echo "In order to deploy or use Review Apps," - echo "KUBE_INGRESS_BASE_DOMAIN variables must be set" - echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" - echo "or by defining a variable at group or project level." - echo "You can also manually add it in .gitlab-ci.yml" - false - else - true - fi - } - - function initialize_tiller() { - echo "Checking Tiller..." - - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & - echo "Tiller is listening on ${HELM_HOST}" - - if ! helm version --debug; then - echo "Failed to init Tiller." - return 1 - fi - echo "" - } - - function create_secret() { - echo "Create secret..." - if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then - return - fi - - kubectl create secret -n "$KUBE_NAMESPACE" \ - docker-registry gitlab-registry \ - --docker-server="$CI_REGISTRY" \ - --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ - --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ - --docker-email="$GITLAB_USER_EMAIL" \ - -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - - } - - function persist_environment_url() { - echo $CI_ENVIRONMENT_URL > environment_url.txt - } - - function delete() { - track="${1-stable}" - name=$(deploy_name "$track") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm delete --purge "$name" - fi - - secret_name=$(application_secret_name "$track") - kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" - } - -before_script: - - *deploy_helpers diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 13ab98d3a16..84bb0ff3b33 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,5 +1,3 @@ -# This file is a template, and might need editing before it works on your project. - # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 83e179f37c3..0a3cf3dcf77 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -1,5 +1,5 @@ image: - name: hashicorp/packer:1.0.4 + name: hashicorp/packer:latest entrypoint: - '/usr/bin/env' - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 5ad624bb15f..2afc99d0bf8 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -5,6 +5,7 @@ container_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" # Defining two new variables based on GitLab's CI/CD predefined variables # https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG @@ -22,8 +23,9 @@ container_scanning: DOCKER_SERVICE: docker DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags - CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 - CLAIR_EXECUTABLE_VERSION: v11 + CLAIR_LOCAL_SCAN_VERSION: v2.0.8_0ed98e9ead65a51ba53f7cc53fa5e80c92169207 + CLAIR_EXECUTABLE_VERSION: v12 + CLAIR_EXECUTABLE_SHA: 44f2a3fdd7b0d102c98510e7586f6956edc89ab72c6943980f92f4979f7f4081 ## Disable the proxy for clair-local-scan, otherwise Container Scanning will ## fail when a proxy is used. NO_PROXY: ${DOCKER_SERVICE},localhost @@ -43,6 +45,7 @@ container_scanning: - apk add -U wget ca-certificates - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - wget https://github.com/arminc/clair-scanner/releases/download/${CLAIR_EXECUTABLE_VERSION}/clair-scanner_linux_amd64 + - echo "${CLAIR_EXECUTABLE_SHA} clair-scanner_linux_amd64" | sha256sum -c - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 89eccce69f6..15b84f1540d 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -9,6 +9,7 @@ dependency_scanning: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -41,6 +42,7 @@ dependency_scanning: DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ ) \ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 0a97a16b83c..90278122361 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,6 +9,7 @@ sast: image: docker:stable variables: DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" allow_failure: true services: - docker:stable-dind @@ -45,11 +46,14 @@ sast: SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ SAST_RUN_ANALYZER_TIMEOUT \ + SAST_JAVA_VERSION \ ANT_HOME \ ANT_PATH \ GRADLE_PATH \ JAVA_OPTS \ JAVA_PATH \ + JAVA_8_VERSION \ + JAVA_11_VERSION \ MAVEN_CLI_OPTS \ MAVEN_PATH \ MAVEN_REPO_PATH \ diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index a3db2705bf6..280e75d46f5 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -8,26 +8,23 @@ stages: - deploy .serverless:build:image: - stage: build image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl app build .serverless:deploy:image: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy - image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11 environment: development - script: - - echo "$CI_REGISTRY_IMAGE" - - tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait + script: /usr/bin/gitlabktl app deploy .serverless:build:functions: - stage: build - environment: development image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl serverless build .serverless:deploy:functions: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy environment: development - image: registry.gitlab.com/gitlab-org/gitlabktl:latest script: /usr/bin/gitlabktl serverless deploy diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index ce5857965bf..9550bc6d39c 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -63,7 +63,15 @@ module Gitlab end def exist? - trace_artifact&.exists? || job.trace_chunks.any? || current_path.present? || old_trace.present? + archived_trace_exist? || live_trace_exist? + end + + def archived_trace_exist? + trace_artifact&.exists? + end + + def live_trace_exist? + job.trace_chunks.any? || current_path.present? || old_trace.present? end def read @@ -118,7 +126,7 @@ module Gitlab raise AlreadyArchivedError, 'Could not write to the archived trace' elsif current_path File.open(current_path, mode) - elsif Feature.enabled?('ci_enable_live_trace') + elsif Feature.enabled?('ci_enable_live_trace', job.project) Gitlab::Ci::Trace::ChunkedIO.new(job) else File.open(ensure_path, mode) @@ -167,7 +175,7 @@ module Gitlab def clone_file!(src_stream, temp_dir) FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| temp_path = File.join(dir_path, "job.log") FileUtils.touch(temp_path) size = IO.copy_stream(src_stream, temp_path) diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 8c6fd56493f..e99889f4a25 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,6 +166,13 @@ module Gitlab end def destroy! + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless build.has_archived_trace? + Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', + job_id: build.id) + end + trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a5693dc4f81..2e1eab270ff 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -40,6 +40,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), + needs_attributes: job[:needs]&.map { |need| { name: need } }, options: { image: job[:image], services: job[:services], @@ -54,7 +55,8 @@ module Gitlab parallel: job[:parallel], instance: job[:instance], start_in: job[:start_in], - trigger: job[:trigger] + trigger: job[:trigger], + bridge_needs: job[:needs] }.compact }.compact end @@ -108,6 +110,7 @@ module Gitlab validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_needs!(name, job) validate_job_environment!(name, job) end end @@ -144,12 +147,30 @@ module Gitlab job[:dependencies].each do |dependency| raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym] - unless @stages.index(@jobs[dependency.to_sym][:stage]) < stage_index + dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage]) + + unless dependency_stage_index.present? && dependency_stage_index < stage_index raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages" end end end + def validate_job_needs!(name, job) + return unless job[:needs] + + stage_index = @stages.index(job[:stage]) + + job[:needs].each do |need| + raise ValidationError, "#{name} job: undefined need: #{need}" unless @jobs[need.to_sym] + + needs_stage_index = @stages.index(@jobs[need.to_sym][:stage]) + + unless needs_stage_index.present? && needs_stage_index < stage_index + raise ValidationError, "#{name} job: need #{need} is not defined in prior stages" + end + end + end + def validate_job_environment!(name, job) return unless job[:environment] return unless job[:environment].is_a?(Hash) diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb index 3b4ebc3fbae..f53051c32ff 100644 --- a/lib/gitlab/cluster/puma_worker_killer_observer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_observer.rb @@ -15,9 +15,7 @@ module Gitlab private def log_termination(worker) - labels = { worker: "worker_#{worker.index}" } - - @counter.increment(labels) + @counter.increment end end end diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index 560fe63df0e..87bd257f69a 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -18,6 +18,10 @@ module Gitlab config[attribute] end + + define_method("has_#{attribute}?") do + config.is_a?(Hash) && config.key?(attribute) + end end end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 6796fcce75f..374f929878e 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -20,7 +20,20 @@ module Gitlab present_keys = value.try(:keys).to_a & options[:in] if present_keys.any? - record.errors.add(attribute, "contains disallowed keys: " + + message = options[:message] || "contains disallowed keys" + message += ": #{present_keys.join(', ')}" + + record.errors.add(attribute, message) + end + end + end + + class RequiredKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + present_keys = options[:in] - value.try(:keys).to_a + + if present_keys.any? + record.errors.add(attribute, "missing required keys: " + present_keys.join(', ')) end end @@ -54,6 +67,16 @@ module Gitlab end end + class ArrayOfHashesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all? + record.errors.add(attribute, 'should be an array of hashes') + end + end + end + class ArrayOrStringValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless value.is_a?(Array) || value.is_a?(String) @@ -220,6 +243,14 @@ module Gitlab end end + class ExpressionValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid? + record.errors.add(attribute, 'Invalid expression syntax') + end + end + end + class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) return unless value.is_a?(Array) diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb new file mode 100644 index 00000000000..ff844645b11 --- /dev/null +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ContentSecurityPolicy + class ConfigLoader + DIRECTIVES = %w(base_uri child_src connect_src default_src font_src + form_action frame_ancestors frame_src img_src manifest_src + media_src object_src report_uri script_src style_src worker_src).freeze + + def self.default_settings_hash + { + 'enabled' => false, + 'report_only' => false, + 'directives' => DIRECTIVES.each_with_object({}) { |directive, hash| hash[directive] = nil } + } + end + + def initialize(csp_directives) + @csp_directives = HashWithIndifferentAccess.new(csp_directives) + end + + def load(policy) + DIRECTIVES.each do |directive| + arguments = arguments_for(directive) + + next unless arguments.present? + + policy.public_send(directive, *arguments) # rubocop:disable GitlabSecurity/PublicSend + end + end + + private + + def arguments_for(directive) + arguments = @csp_directives[directive.to_s] + + return unless arguments.present? && arguments.is_a?(String) + + arguments.strip.split(' ').map(&:strip) + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index f7d046600e8..5b0b91de5da 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -84,11 +84,7 @@ module Gitlab .and(t[:created_at].lteq(Date.current.end_of_day)) .and(t[:author_id].eq(contributor.id)) - date_interval = if Gitlab::Database.postgresql? - "INTERVAL '#{Time.zone.now.utc_offset} seconds'" - else - "INTERVAL #{Time.zone.now.utc_offset} SECOND" - end + date_interval = "INTERVAL '#{Time.zone.now.utc_offset} seconds'" Event.reorder(nil) .select(t[:project_id], t[:target_type], t[:action], "date(created_at + #{date_interval}) AS date", 'count(id) as total_amount') diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 32d5e4b9ea3..6ce47650562 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,11 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end + def expire_current_application_settings + ::ApplicationSetting.expire + Gitlab::SafeRequestStore.delete(:current_application_settings) + end + def clear_in_memory_application_settings! @in_memory_application_settings = nil end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 0cacef5b278..07ae430c45e 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,13 +4,13 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery + include GroupProjectsProvider - attr_reader :projections, :query, :stage, :order, :project, :options + attr_reader :projections, :query, :stage, :order, :options MAX_EVENTS = 50 - def initialize(project: nil, stage:, options:) - @project = project + def initialize(stage:, options:) @stage = stage @options = options end @@ -68,11 +68,11 @@ module Gitlab end def allowed_ids_source - { project_id: project.id } + group ? { group_id: group.id, include_subgroups: true } : { project_id: project.id } end - def projects - [project] + def serialization_context + {} end end end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 39fc1759cfc..459bb5177b5 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -16,17 +16,32 @@ module Gitlab def stage_query(project_ids) query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) - .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(options[:from])) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + + query = limit_query(query, project_ids) # Load merge_requests - query = query.join(mr_table, Arel::Nodes::OuterJoin) + + query = load_merge_requests(query) + + query + end + + def limit_query(query, project_ids) + query.where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + .where(issue_table[:created_at].gteq(options[:from])) + end + + def load_merge_requests(query) + query.join(mr_table, Arel::Nodes::OuterJoin) .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) .join(mr_metrics_table) .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - - query end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 98b86e54340..1cd54238bb4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,11 +4,11 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery + include GroupProjectsProvider - attr_reader :project, :options + attr_reader :options - def initialize(project: nil, options:) - @project = project + def initialize(options:) @options = options end @@ -24,7 +24,7 @@ module Gitlab raise NotImplementedError.new("Expected #{self.name} to implement title") end - def median + def project_median return if project.nil? BatchLoader.for(project.id).batch(key: name) do |project_ids, loader| @@ -42,6 +42,10 @@ module Gitlab end end + def group_median + median_query(projects.map(&:id)) + end + def median_query(project_ids) # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). @@ -67,18 +71,13 @@ module Gitlab private def event_fetcher - @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: project, - stage: name, + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(stage: name, options: event_options) end def event_options options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) end - - def projects - [project] - end end end end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 9e7ca529579..fcc282bf7a6 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -20,7 +20,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: project).represent(event) + AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/group_projects_provider.rb b/lib/gitlab/cycle_analytics/group_projects_provider.rb new file mode 100644 index 00000000000..1287a48daaa --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_projects_provider.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module GroupProjectsProvider + def projects + group ? projects_for_group : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + private + + def projects_for_group + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb new file mode 100644 index 00000000000..a1fc941495d --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + class GroupStageSummary + attr_reader :group, :from, :current_user, :options + + def initialize(group, options:) + @group = group + @from = options[:from] + @current_user = options[:current_user] + @options = options + end + + def data + [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index bb3520ae920..6914cf24c19 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -18,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb index ac836b8bf0f..295eca5edca 100644 --- a/lib/gitlab/cycle_analytics/issue_helper.rb +++ b/lib/gitlab/cycle_analytics/issue_helper.rb @@ -5,13 +5,23 @@ module Gitlab module IssueHelper def stage_query(project_ids) query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) - .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(options[:from])) - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) + + query = limit_query(query, project_ids) query end + + def limit_query(query, project_ids) + query.where(issue_table[:project_id].in(project_ids)) + .where(routes_table[:source_type].eq('Namespace')) + .where(issue_table[:created_at].gteq(options[:from])) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + end end end end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index 3e0302d308d..015f7bfde24 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -35,6 +35,14 @@ module Gitlab User.arel_table end + def projects_table + Project.arel_table + end + + def routes_table + Route.arel_table + end + def build_table ::CommitStatus.arel_table end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index 03ba98b4dfb..55214e6b896 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -23,7 +23,7 @@ module Gitlab end def get - ::CycleAnalytics::Base::STAGES.each do |stage| + ::CycleAnalytics::LevelBase::STAGES.each do |stage| @stage_permission_hash[stage] = authorized_stage?(stage) end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 49a6b099f34..bad02e00a13 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -18,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb index ae578d45ad5..a63ae58ad21 100644 --- a/lib/gitlab/cycle_analytics/plan_helper.rb +++ b/lib/gitlab/cycle_analytics/plan_helper.rb @@ -5,14 +5,23 @@ module Gitlab module PlanHelper def stage_query(project_ids) query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) + .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) + .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) .project(issue_table[:project_id].as("project_id")) + .project(projects_table[:path].as("project_path")) + .project(routes_table[:path].as("namespace_path")) .where(issue_table[:project_id].in(project_ids)) - .where(issue_table[:created_at].gteq(options[:from])) - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + .where(routes_table[:source_type].eq('Namespace')) + query = limit_query(query) query end + + def limit_query(query) + query.where(issue_table[:created_at].gteq(options[:from])) + .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) + .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) + end end end end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb index 949119d69a0..8843ab2bcb9 100644 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -10,7 +10,8 @@ module Gitlab issue_table[:iid], issue_table[:id], issue_table[:created_at], - issue_table[:author_id]] + issue_table[:author_id], + routes_table[:path]] super(*args) end @@ -18,7 +19,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: project).represent(event) + AnalyticsIssueSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index d31736e755d..4b5d79097b7 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -19,7 +19,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: project).represent(event) + AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) end def allowed_ids_finder_class diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb new file mode 100644 index 00000000000..48d8164bde1 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Base + attr_reader :group, :from, :options + + def initialize(group:, from:, options:) + @group = group + @from = from + @options = options + end + + def title + raise NotImplementedError.new("Expected #{self.name} to implement title") + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb new file mode 100644 index 00000000000..78d677cf558 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Deploy < Group::Base + include GroupProjectsProvider + + def title + n_('Deploy', 'Deploys', value) + end + + def value + @value ||= find_deployments + end + + private + + def find_deployments + deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) + deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] + deployments = deployments.where("deployments.created_at > ?", from) + deployments.success.count + end + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb new file mode 100644 index 00000000000..9daae8531d8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Group + class Issue < Group::Base + attr_reader :group, :from, :current_user, :options + + def initialize(group:, from:, current_user:, options:) + @group = group + @from = from + @current_user = current_user + @options = options + end + + def title + n_('New Issue', 'New Issues', value) + end + + def value + @value ||= find_issues + end + + private + + def find_issues + issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = issues.where(projects: { id: options[:projects] }) if options[:projects] + issues.count + end + end + end + end + end +end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 0fc145534bf..332ca8bf9b8 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -46,6 +46,16 @@ module Gitlab ee? ? 'gitlab-ee' : 'gitlab-ce' end + def markdown_list(items) + list = items.map { |item| "* `#{item}`" }.join("\n") + + if items.size > 10 + "\n<details>\n\n#{list}\n\n</details>\n" + else + list + end + end + # @return [Hash<String,Array<String>>] def changes_by_category all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| @@ -103,7 +113,7 @@ module Gitlab yarn\.lock )\z}x => :frontend, - %r{\A(ee/)?db/} => :database, + %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, %r{\Arubocop/cop/migration(/|\.rb)} => :database, @@ -132,6 +142,22 @@ module Gitlab def new_teammates(usernames) usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end + + def missing_database_labels(current_mr_labels) + labels = if has_database_scoped_labels?(current_mr_labels) + ['database'] + else + ['database', 'database::review pending'] + end + + labels - current_mr_labels + end + + private + + def has_database_scoped_labels?(current_mr_labels) + current_mr_labels.any? { |label| label.start_with?('database::') } + end end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index b44f134f2c1..74cbcc11255 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -39,7 +39,7 @@ module Gitlab def has_capability?(project, category, kind, labels) case category when :test - area = role[/Test Automation Engineer, (\w+)/, 1] + area = role[/Test Automation Engineer(?:.*?, (\w+))/, 1] area && labels.any?(area) if kind == :reviewer else diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 40bda3410e1..75d9a2d55b9 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -60,7 +60,8 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def build( project:, user:, ref:, oldrev: nil, newrev: nil, - commits: [], commits_count: nil, message: nil, push_options: {}) + commits: [], commits_count: nil, message: nil, push_options: {}, + with_changed_files: true) commits = Array(commits) @@ -75,7 +76,7 @@ module Gitlab # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259 commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do commits_limited.map do |commit| - commit.hook_attrs(with_changed_files: true) + commit.hook_attrs(with_changed_files: with_changed_files) end end @@ -128,8 +129,6 @@ module Gitlab SAMPLE_DATA end - private - def checkout_sha(repository, newrev, ref) # Checkout sha is nil when we remove branch or tag return if Gitlab::Git.blank_ref?(newrev) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 3e4c720b49a..cbdff0ab060 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -4,13 +4,13 @@ module Gitlab module Database include Gitlab::Metrics::Methods - # The max value of INTEGER type is the same between MySQL and PostgreSQL: # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html - # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz: # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html # https://dev.mysql.com/doc/refman/5.7/en/datetime.html + # FIXME: this should just be the max value of timestampz MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze # Minimum schema version from which migrations are supported @@ -39,13 +39,14 @@ module Gitlab end def self.human_adapter_name - postgresql? ? 'PostgreSQL' : 'MySQL' - end - - def self.mysql? - adapter_name.casecmp('mysql2').zero? + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end end + # @deprecated def self.postgresql? adapter_name.casecmp('postgresql').zero? end @@ -60,15 +61,14 @@ module Gitlab # Check whether the underlying database is in read-only mode def self.db_read_only? - if postgresql? - pg_is_in_recovery = - ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') - .first.fetch('pg_is_in_recovery') + pg_is_in_recovery = + ActiveRecord::Base + .connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') - Gitlab::Utils.to_boolean(pg_is_in_recovery) - else - false - end + Gitlab::Utils.to_boolean(pg_is_in_recovery) end def self.db_read_write? @@ -80,19 +80,19 @@ module Gitlab end def self.postgresql_9_or_less? - postgresql? && version.to_f < 10 + version.to_f < 10 end def self.join_lateral_supported? - postgresql? && version.to_f >= 9.3 + version.to_f >= 9.3 end def self.replication_slots_supported? - postgresql? && version.to_f >= 9.4 + version.to_f >= 9.4 end def self.postgresql_minimum_supported_version? - postgresql? && version.to_f >= 9.6 + version.to_f >= 9.6 end # map some of the function names that changed between PostgreSQL 9 and 10 @@ -118,51 +118,23 @@ module Gitlab end def self.nulls_last_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS LAST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'ASC' - end - - Arel.sql(order) + Arel.sql("#{field} #{direction} NULLS LAST") end def self.nulls_first_order(field, direction = 'ASC') - order = "#{field} #{direction}" - - if postgresql? - order = "#{order} NULLS FIRST" - else - # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL - # columns. In the (default) ascending order, `0` comes first. - order = "#{field} IS NULL, #{order}" if direction == 'DESC' - end - - Arel.sql(order) + Arel.sql("#{field} #{direction} NULLS FIRST") end def self.random - postgresql? ? "RANDOM()" : "RAND()" + "RANDOM()" end def self.true_value - if postgresql? - "'t'" - else - 1 - end + "'t'" end def self.false_value - if postgresql? - "'f'" - else - 0 - end + "'f'" end def self.with_connection_pool(pool_size) @@ -182,7 +154,7 @@ module Gitlab # rows - An Array of Hash instances, each mapping the columns to their # values. # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows, this only works on PostgreSQL. + # the inserted rows # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) @@ -191,7 +163,6 @@ module Gitlab keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } - return_ids = false if mysql? disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| @@ -258,11 +229,7 @@ module Gitlab def self.database_version row = connection.execute("SELECT VERSION()").first - if postgresql? - row['version'] - else - row.first - end + row['version'] end private_class_method :database_version diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index f3d37ccd72a..eac61254bdf 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -37,16 +37,14 @@ module Gitlab # @return [Hash] of Model -> count mapping def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) strategies.each_with_object({}) do |strategy, counts_by_model| - if strategy.enabled? - models_with_missing_counts = models - counts_by_model.keys + models_with_missing_counts = models - counts_by_model.keys - break counts_by_model if models_with_missing_counts.empty? + break counts_by_model if models_with_missing_counts.empty? - counts = strategy.new(models_with_missing_counts).count + counts = strategy.new(models_with_missing_counts).count - counts.each do |model, count| - counts_by_model[model] = count - end + counts.each do |model, count| + counts_by_model[model] = count end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index fa6951eda22..0b8fe640bf8 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -23,10 +23,6 @@ module Gitlab rescue *CONNECTION_ERRORS {} end - - def self.enabled? - true - end end end end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 695f6fa766e..6cd90c01ab2 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -31,10 +31,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? - end - private # Models using single-type inheritance (STI) don't work with diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 7777f31f702..e9387a91a14 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -28,10 +28,6 @@ module Gitlab {} end - def self.enabled? - Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) - end - private def perform_count(model, estimate) diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 79d2caff151..1392b397012 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,8 +7,7 @@ module Gitlab # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval # along with an alias specified by the `as` parameter. # - # Note: For MySQL, the interval is returned in seconds. - # For PostgreSQL, the interval is returned as an INTERVAL type. + # Note: the interval is returned as an INTERVAL type. def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) @@ -16,17 +15,10 @@ module Gitlab end def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) - if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)) + ) end end end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 26adf4e221b..1f47f320a29 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -6,47 +6,25 @@ module Gitlab class Grant < ActiveRecord::Base include FromUnion - self.table_name = - if Database.postgresql? - 'information_schema.role_table_grants' - else - 'information_schema.schema_privileges' - end + self.table_name = 'information_schema.role_table_grants' # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - if Database.postgresql? - # We _must not_ use quote_table_name as this will produce double - # quotes on PostgreSQL and for "has_table_privilege" we need single - # quotes. - quoted_table = connection.quote(table) - - begin - from(nil) - .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) - .first - rescue ActiveRecord::StatementInvalid - # This error is raised when using a non-existing table name. In this - # case we just want to return false as a user technically can't - # create triggers for such a table. - false - end - else - queries = [ - Grant.select(1) - .from('information_schema.user_privileges') - .where("PRIVILEGE_TYPE = 'SUPER'") - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"), - - Grant.select(1) - .from('information_schema.schema_privileges') - .where("PRIVILEGE_TYPE = 'TRIGGER'") - .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name) - .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") - ] + # We _must not_ use quote_table_name as this will produce double + # quotes on PostgreSQL and for "has_table_privilege" we need single + # quotes. + quoted_table = connection.quote(table) - Grant.from_union(queries, alias_as: 'privs').any? + begin + from(nil) + .pluck(Arel.sql("has_table_privilege(#{quoted_table}, 'TRIGGER')")) + .first + rescue ActiveRecord::StatementInvalid + # This error is raised when using a non-existing table name. In this + # case we just want to return false as a user technically can't + # create triggers for such a table. + false end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index b8d895dee7d..603b125d8b4 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -17,13 +17,9 @@ module Gitlab def extract_median(results) result = results.compact.first - if Gitlab::Database.postgresql? - result = result.first.presence + result = result.first.presence - result['median']&.to_f if result - elsif Gitlab::Database.mysql? - result.to_a.flatten.first - end + result['median']&.to_f if result end def extract_medians(results) @@ -34,31 +30,6 @@ module Gitlab end end - def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table.from - .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) - .project(average([arel_table[column_sym]], 'median')) - .where( - Arel::Nodes::Between.new( - Arel.sql("(select @row_id := @row_id + 1)"), - Arel::Nodes::And.new( - [Arel.sql('@ct/2.0'), - Arel.sql('@ct/2.0 + 1')] - ) - ) - ). - # Disallow negative values - where(arel_table[column_sym].gteq(0)) - - [ - Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), - Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), - Arel.sql("set @row_id := 0;"), - query.to_sql, - Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") - ] - end - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) # Create a CTE with the column we're operating on, row number (after sorting by the column # we're operating on), and count of the table we're operating on (duplicated across) all rows @@ -113,18 +84,8 @@ module Gitlab private - def median_queries(arel_table, query_so_far, column_sym, partition_column = nil) - if Gitlab::Database.postgresql? - pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - elsif Gitlab::Database.mysql? - raise NotSupportedError, "partition_column is not supported for MySQL" if partition_column - - mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - end - end - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = median_queries(arel_table, query_so_far, column_sym, partition_column) + queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } end @@ -176,8 +137,6 @@ module Gitlab end def extract_diff_epoch(diff) - return diff unless Gitlab::Database.postgresql? - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0c5f33e1b2a..9bba4f6ce1e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,31 +6,45 @@ module Gitlab BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time + PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze + DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze + # Adds `created_at` and `updated_at` columns with timezone information. # # This method is an improved version of Rails' built-in method `add_timestamps`. # + # By default, adds `created_at` and `updated_at` columns, but these can be specified as: + # + # add_timestamps_with_timezone(:my_table, columns: [:created_at, :deleted_at]) + # + # This allows you to create just the timestamps you need, saving space. + # # Available options are: - # default - The default value for the column. - # null - When set to `true` the column will allow NULL values. + # :default - The default value for the column. + # :null - When set to `true` the column will allow NULL values. # The default is to not allow NULL values. + # :columns - the column names to create. Must be one + # of `Gitlab::Database::MigrationHelpers::PERMITTED_TIMESTAMP_COLUMNS`. + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. def add_timestamps_with_timezone(table_name, options = {}) options[:null] = false if options[:null].nil? + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + default_value = options[:default] - [:created_at, :updated_at].each do |column_name| - if options[:default] && transaction_open? - raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` ' \ - 'in the body of your migration class' - end + validate_not_in_transaction!(:add_timestamps_with_timezone, 'with default value') if default_value + + columns.each do |column_name| + validate_timestamp_column_name!(column_name) # If default value is presented, use `add_column_with_default` method instead. - if options[:default] + if default_value add_column_with_default( table_name, column_name, :datetime_with_timezone, - default: options[:default], + default: default_value, allow_null: options[:null] ) else @@ -39,10 +53,22 @@ module Gitlab end end - # Creates a new index, concurrently when supported + # To be used in the `#down` method of migrations that + # use `#add_timestamps_with_timezone`. # - # On PostgreSQL this method creates an index concurrently, on MySQL this - # creates a regular index. + # Available options are: + # :columns - the column names to remove. Must be one + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. + def remove_timestamps(table_name, options = {}) + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + columns.each do |column_name| + remove_column(table_name, column_name) + end + end + + # Creates a new index, concurrently # # Example: # @@ -56,9 +82,7 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? - options = options.merge({ algorithm: :concurrently }) - end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, options) Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" # rubocop:disable Gitlab/RailsLogger @@ -70,9 +94,7 @@ module Gitlab end end - # Removes an existed index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existed index, concurrently # # Example: # @@ -100,9 +122,7 @@ module Gitlab end end - # Removes an existing index, concurrently when supported - # - # On PostgreSQL this method removes an index concurrently. + # Removes an existing index, concurrently # # Example: # @@ -132,8 +152,6 @@ module Gitlab # Only available on Postgresql >= 9.2 def supports_drop_index_concurrently? - return false unless Database.postgresql? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i version >= 90200 @@ -141,8 +159,7 @@ module Gitlab # Adds a foreign key with only minimal locking on the tables involved. # - # This method only requires minimal locking when using PostgreSQL. When - # using MySQL this method will use Rails' default `add_foreign_key`. + # This method only requires minimal locking # # source - The source table containing the foreign key. # target - The target table the key points to. @@ -158,27 +175,7 @@ module Gitlab raise 'add_concurrent_foreign_key can not be run inside a transaction' end - # While MySQL does allow disabling of foreign keys it has no equivalent - # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall - # back to the normal foreign key procedure. - if Database.mysql? - if foreign_key_exists?(source, target, column: column) - Rails.logger.warn "Foreign key not created because it exists already " \ - "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{column}" - return - end - - key_options = { column: column, on_delete: on_delete } - - # The MySQL adapter tries to create a foreign key without a name when - # `:name` is nil, instead of generating a name for us. - key_options[:name] = name if name - - return add_foreign_key(source, target, key_options) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify key_name = name || concurrent_foreign_key_name(source, column) @@ -236,7 +233,7 @@ module Gitlab # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. (PostgreSQL only) + # migrations don't get killed prematurely. # # There are two possible ways to disable the statement timeout: # @@ -248,15 +245,6 @@ module Gitlab # otherwise the statement will still be disabled until connection is dropped # or `RESET ALL` is executed def disable_statement_timeout - # bypass disabled_statement logic when not using postgres, but still execute block when one is given - unless Database.postgresql? - if block_given? - yield - end - - return - end - if block_given? begin execute('SET statement_timeout TO 0') @@ -506,13 +494,12 @@ module Gitlab quoted_old = quote_column_name(old_column) quoted_new = quote_column_name(new_column) - if Database.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql( + trigger_name, + quoted_table, + quoted_old, + quoted_new + ) end # Changes the type of a column concurrently. @@ -555,11 +542,7 @@ module Gitlab check_trigger_permissions!(table) - if Database.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end @@ -772,38 +755,12 @@ module Gitlab EOF end - # Installs the triggers necessary to perform a concurrent column rename on - # MySQL. - def install_rename_triggers_for_mysql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_insert - BEFORE INSERT - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger}_update - BEFORE UPDATE - ON #{table} - FOR EACH ROW - SET NEW.#{new} = NEW.#{old} - EOF - end - # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") execute("DROP FUNCTION IF EXISTS #{trigger}()") end - # Removes the triggers used for renaming a MySQL column concurrently. - def remove_rename_triggers_for_mysql(trigger) - execute("DROP TRIGGER IF EXISTS #{trigger}_insert") - execute("DROP TRIGGER IF EXISTS #{trigger}_update") - end - # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) @@ -853,8 +810,6 @@ module Gitlab order: index.orders } - # These options are not supported by MySQL, so we only add them if - # they were previously set. options[:using] = index.using if index.using options[:where] = index.where if index.where @@ -894,26 +849,16 @@ module Gitlab end # This will replace the first occurrence of a string in a column with - # the replacement - # On postgresql we can use `regexp_replace` for that. - # On mysql we find the location of the pattern, and overwrite it - # with the replacement + # the replacement using `regexp_replace` def replace_sql(column, pattern, replacement) quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if Database.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) + replace = Arel::Nodes::NamedFunction.new( + "regexp_replace", [column, quoted_pattern, quoted_replacement] + ) - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_if_exists(*args) @@ -955,11 +900,7 @@ database (#{dbname}) using a super user and running: ALTER #{user} WITH SUPERUSER -For MySQL you instead need to run: - - GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%' - -Both queries will grant the user super user permissions, ensuring you don't run +This query will grant the user super user permissions, ensuring you don't run into similar problems in the future (e.g. when new tables are created). EOF end @@ -1062,10 +1003,6 @@ into similar problems in the future (e.g. when new tables are created). # This will include indexes using an expression on the column, for example: # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));` # - # For mysql, it falls back to the default ActiveRecord implementation that - # will not find custom indexes. But it will select by name without passing - # a column. - # # We can remove this when upgrading to Rails 5 with an updated `index_exists?`: # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882 # @@ -1076,10 +1013,8 @@ into similar problems in the future (e.g. when new tables are created). # does not find indexes without passing a column name. if indexes(table).map(&:name).include?(index.to_s) true - elsif Gitlab::Database.postgresql? - postgres_exists_by_name?(table, index) else - false + postgres_exists_by_name?(table, index) end end @@ -1095,8 +1030,26 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end - def mysql_compatible_index_length - Gitlab::Database.mysql? ? 20 : nil + private + + def validate_timestamp_column_name!(column_name) + return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name) + + raise <<~MESSAGE + Illegal timestamp column name! Got #{column_name}. + Must be one of: #{PERMITTED_TIMESTAMP_COLUMNS.to_a} + MESSAGE + end + + def validate_not_in_transaction!(method_name, modifier = nil) + return unless transaction_open? + + raise <<~ERROR + #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. + + You can disable transactions by calling `disable_ddl_transaction!` in the body of + your migration class + ERROR end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 60afa4bcd52..565f34b78b7 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -51,14 +51,10 @@ module Gitlab quoted_old_full_path = quote_string(old_full_path) quoted_old_wildcard_path = quote_string("#{old_full_path}/%") - filter = if Database.mysql? - "lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "OR routes.path LIKE '#{quoted_old_wildcard_path}'" - else - "routes.id IN "\ - "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" - end + filter = + "routes.id IN "\ + "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ + "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" replace_statement = replace_sql(Route.arel_table[:path], old_full_path, diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index 109ae7893da..ddbabc9098e 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -2,14 +2,9 @@ module Gitlab module Database - BINARY_TYPE = - if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveModel::Type::Binary - end + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + BINARY_TYPE = ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb index 10971d2b274..2a6f39c6a27 100644 --- a/lib/gitlab/database/subquery.rb +++ b/lib/gitlab/database/subquery.rb @@ -6,11 +6,7 @@ module Gitlab class << self def self_join(relation) t = relation.arel_table - # Work around a bug in Rails 5, where LIMIT causes trouble - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 - r = relation.limit(nil).arel - r.take(relation.limit_value) if relation.limit_value - t2 = r.as('t2') + t2 = relation.arel.as('t2') relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 01fd261404b..86e532766b1 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -7,7 +7,7 @@ module Gitlab CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md/i.freeze + IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md|doc\/.+/i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 17fbecbd097..d09dcdbb337 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -61,6 +61,10 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), 'realtime_changes_import_gitea' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), + 'merge_request_widget' ) ].freeze diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 7961d4bbd6e..61eb030563d 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -15,17 +15,18 @@ module Gitlab raise ArgumentError, 'Key needs to be specified' unless key lease = Gitlab::ExclusiveLease.new(key, timeout: ttl) + retried = false until uuid = lease.try_obtain # Keep trying until we obtain the lease. To prevent hammering Redis too # much we'll wait for a bit. sleep(sleep_sec) - break if (retries -= 1) < 0 + (retries -= 1) < 0 ? break : retried ||= true end raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid - yield + yield(retried) ensure Gitlab::ExclusiveLease.cancel(key, uuid) end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 2d1c9ac40ae..6b52d6e88e5 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -27,7 +27,15 @@ module Gitlab klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader moved = klass.copy_to(file, target_parent) - moved.markdown_link + + moved_markdown = moved.markdown_link + + # Prevents rewrite of plain links as embedded + if was_embedded?(markdown) + moved_markdown + else + moved_markdown.sub(/\A!/, "") + end end end @@ -43,6 +51,10 @@ module Gitlab referenced_files.compact.select(&:exists?) end + def was_embedded?(markdown) + markdown.starts_with?("!") + end + private def find_file(project, secret, file) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 44a62586a23..df9f33baec2 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -9,6 +9,7 @@ module Gitlab # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze BLANK_SHA = ('0' * 40).freeze + COMMIT_ID = /\A[0-9a-f]{40}\z/.freeze TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze @@ -65,6 +66,10 @@ module Gitlab ref == BLANK_SHA end + def commit_id?(ref) + COMMIT_ID.match?(ref) + end + def version Gitlab::Git::Version.git_version end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a7d9ba51277..27032602828 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -55,6 +55,10 @@ module Gitlab @name = @relative_path.split("/").last end + def to_s + "<#{self.class.name}: #{self.gl_project_path}>" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end @@ -873,13 +877,13 @@ module Gitlab def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self, + start_branch_name: nil, start_sha: nil, start_repository: self, force: false) wrapped_gitaly_errors do gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) end end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb index 86c9f33d82a..5c73c0c66a9 100644 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ b/lib/gitlab/git/rugged_impl/blob.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entry def tree_entry(repository, sha, path, limit) if use_rugged?(repository, :rugged_tree_entry) - rugged_tree_entry(repository, sha, path, limit) + execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) else super end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 971a33b2e99..0eff35ab1c4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -36,7 +36,7 @@ module Gitlab override :find_commit def find_commit(repo, commit_id) if use_rugged?(repo, :rugged_find_commit) - rugged_find(repo, commit_id) + execute_rugged_call(:rugged_find, repo, commit_id) else super end @@ -45,7 +45,7 @@ module Gitlab override :batch_by_oid def batch_by_oid(repo, oids) if use_rugged?(repo, :rugged_list_commits_by_oid) - rugged_batch_by_oid(repo, oids) + execute_rugged_call(:rugged_batch_by_oid, repo, oids) else super end @@ -68,7 +68,7 @@ module Gitlab override :commit_tree_entry def commit_tree_entry(path) if use_rugged?(@repository, :rugged_commit_tree_entry) - rugged_tree_entry(path) + execute_rugged_call(:rugged_tree_entry, path) else super end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 9268abdfed9..8fde93e71e2 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -48,7 +48,7 @@ module Gitlab override :ancestor? def ancestor?(from, to) if use_rugged?(self, :rugged_commit_is_ancestor) - rugged_is_ancestor?(from, to) + execute_rugged_call(:rugged_is_ancestor?, from, to) else super end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index f3721a3f1b7..389c9d32ccb 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -16,7 +16,7 @@ module Gitlab override :tree_entries def tree_entries(repository, sha, path, recursive) if use_rugged?(repository, :rugged_tree_entries) - tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) else super end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index 99091b03cd1..80b75689334 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -10,6 +10,29 @@ module Gitlab Gitlab::GitalyClient.can_use_disk?(repo.storage) end + + def execute_rugged_call(method_name, *args) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + start = Gitlab::Metrics::System.monotonic_time + + result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend + + duration = Gitlab::Metrics::System.monotonic_time - start + + if Gitlab::RuggedInstrumentation.active? + Gitlab::RuggedInstrumentation.increment_query_count + Gitlab::RuggedInstrumentation.query_time += duration + + Gitlab::RuggedInstrumentation.add_call_details( + feature: method_name, + args: args, + duration: duration, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) + end + + result + end + end end end end diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb index dac4ddd320f..545451f0dc9 100644 --- a/lib/gitlab/git_logger.rb +++ b/lib/gitlab/git_logger.rb @@ -1,13 +1,9 @@ # frozen_string_literal: true module Gitlab - class GitLogger < Gitlab::Logger + class GitLogger < JsonLogger def self.file_name_noext - 'githost' - end - - def format_message(severity, timestamp, progname, msg) - "#{timestamp.to_s(:long)} -> #{severity} -> #{msg}\n" + 'git_json' end end end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index d98b85fecc4..2a8bcd015a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -27,6 +27,29 @@ module Gitlab end end + def includes_branches? + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.branch_ref?(ref) + end + end + + def includes_tags? + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.tag_ref?(ref) + end + end + + def includes_default_branch? + # If the branch doesn't have a default branch yet, we presume the + # first branch pushed will be the default. + return true unless project.default_branch.present? + + enum_for(:changes_refs).any? do |_oldrev, _newrev, ref| + Gitlab::Git.branch_ref?(ref) && + Gitlab::Git.branch_name(ref) == project.default_branch + end + end + private def deserialize_changes(changes) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 0b6321c66ab..e6cbfb00f60 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -240,7 +240,7 @@ module Gitlab # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) - # Only count limits in request-response environments (not sidekiq for example) + # Only count limits in request-response environments return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only @@ -387,21 +387,20 @@ module Gitlab end def self.can_use_disk?(storage) - false - # cached_value = MUTEX.synchronize do - # @can_use_disk ||= {} - # @can_use_disk[storage] - # end + cached_value = MUTEX.synchronize do + @can_use_disk ||= {} + @can_use_disk[storage] + end - # return cached_value unless cached_value.nil? + return cached_value unless cached_value.nil? - # gitaly_filesystem_id = filesystem_id(storage) - # direct_filesystem_id = filesystem_id_from_disk(storage) + gitaly_filesystem_id = filesystem_id(storage) + direct_filesystem_id = filesystem_id_from_disk(storage) - # MUTEX.synchronize do - # @can_use_disk[storage] = gitaly_filesystem_id.present? && - # gitaly_filesystem_id == direct_filesystem_id - # end + MUTEX.synchronize do + @can_use_disk[storage] = gitaly_filesystem_id.present? && + gitaly_filesystem_id == direct_filesystem_id + end end def self.filesystem_id(storage) @@ -414,7 +413,7 @@ module Gitlab metadata_file = File.read(storage_metadata_file_path(storage)) metadata_hash = JSON.parse(metadata_file) metadata_hash['gitaly_filesystem_id'] - rescue Errno::ENOENT, JSON::ParserError + rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError nil end diff --git a/lib/gitlab/gitaly_client/notification_service.rb b/lib/gitlab/gitaly_client/notification_service.rb deleted file mode 100644 index 873c3e4086d..00000000000 --- a/lib/gitlab/gitaly_client/notification_service.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class NotificationService - # 'repository' is a Gitlab::Git::Repository - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @storage = repository.storage - end - - def post_receive - GitalyClient.call( - @storage, - :notification_service, - :post_receive, - Gitaly::PostReceiveRequest.new(repository: @gitaly_repo) - ) - end - end - end -end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 783c2ff0915..33ca428a942 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -325,11 +325,11 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force = false) + start_branch_name, start_repository, force = false, start_sha = nil) req_enum = Enumerator.new do |y| header = user_commit_files_request_header(user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) y.yield Gitaly::UserCommitFilesRequest.new(header: header) @@ -445,7 +445,7 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) Gitaly::UserCommitFilesRequestHeader.new( repository: @gitaly_repo, @@ -456,7 +456,8 @@ module Gitlab commit_author_email: encode_binary(author_email), start_branch_name: encode_binary(start_branch_name), start_repository: start_repository.gitaly_repository, - force: force + force: force, + start_sha: encode_binary(start_sha) ) end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index a61beafae0d..826b35d685c 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -40,7 +40,7 @@ module Gitlab # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. def initialize(token, per_page: 100, parallel: true) - @octokit = Octokit::Client.new( + @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, api_endpoint: api_endpoint @@ -139,7 +139,7 @@ module Gitlab begin yield - rescue Octokit::TooManyRequests + rescue ::Octokit::TooManyRequests raise_or_wait_for_rate_limit # This retry will only happen when running in sequential mode as we'll diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 0e7c9ee0d00..9d925581441 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -36,6 +36,7 @@ module Gitlab description: description_for(release), created_at: release.created_at, updated_at: release.updated_at, + released_at: release.published_at, project_id: project.id } end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 41ec8741eb1..92917028851 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -38,11 +38,6 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url end - - # Flag controls a GFM feature used across many routes. - # Pushing the flag from one place simplifies control - # and facilitates easy removal. - push_frontend_feature_flag(:gfm_embedded_metrics) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/client_env_logger.rb b/lib/gitlab/grape_logging/loggers/client_env_logger.rb new file mode 100644 index 00000000000..3acc6f6a2ef --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/client_env_logger.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This is a fork of +# https://github.com/aserafin/grape_logging/blob/master/lib/grape_logging/loggers/client_env.rb +# to use remote_ip instead of ip. +module Gitlab + module GrapeLogging + module Loggers + class ClientEnvLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + { remote_ip: request.env["HTTP_X_FORWARDED_FOR"] || request.env["REMOTE_ADDR"], ua: request.env["HTTP_USER_AGENT"] } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb index 18ea3a8d2f3..7e86b35a215 100644 --- a/lib/gitlab/grape_logging/loggers/perf_logger.rb +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -6,11 +6,30 @@ module Gitlab module Loggers class PerfLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) + gitaly_data.merge(rugged_data) + end + + def gitaly_data + gitaly_calls = Gitlab::GitalyClient.get_request_count + + return {} if gitaly_calls.zero? + { gitaly_calls: Gitlab::GitalyClient.get_request_count, gitaly_duration: Gitlab::GitalyClient.query_time_ms } end + + def rugged_data + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + return {} if rugged_calls.zero? + + { + rugged_calls: rugged_calls, + rugged_duration_ms: Gitlab::RuggedInstrumentation.query_time_ms + } + end end end end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index 2bcd25cd3cc..ec4b97eaca4 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -18,11 +18,7 @@ module Gitlab def check catch_timeout 10.seconds do - if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s - else - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s - end + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s end end end diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 41eab3658bc..84eb60f3a5d 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # This class is part of the Gitlab::HTTP wrapper. Depending on the value -# of the global setting allow_local_requests_from_hooks_and_services this adapter +# of the global setting allow_local_requests_from_web_hooks_and_services this adapter # will allow/block connection to internal IPs and/or urls. # # This functionality can be overridden by providing the setting the option @@ -38,7 +38,7 @@ module Gitlab end def allow_settings_local_requests? - Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end end end diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index 5b3f30d894a..aaade39dd62 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -6,9 +6,7 @@ module Gitlab # Inserts a raw row and returns the ID of the inserted row. # # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - # rubocop: disable CodeReuse/ActiveRecord + # relation - An ActiveRecord::Relation to use for finding the table name def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when @@ -16,12 +14,8 @@ module Gitlab result = Gitlab::Database .bulk_insert(relation.table_name, [attributes], return_ids: true) - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + result.first end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index f63a5ece71e..bb46bd657e8 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -4,7 +4,9 @@ module Gitlab module ImportExport extend self - # For every version update, the version history in import_export.md has to be kept up to date. + # For every version update the version history in these docs must be kept up to date: + # - development/import_export.md + # - user/project/settings/import_export.md VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 @@ -28,6 +30,14 @@ module Gitlab "project.bundle" end + def lfs_objects_filename + "lfs-objects.json" + end + + def lfs_objects_storage + "lfs-objects" + end + def config_file Rails.root.join('lib/gitlab/import_export/import_export.yml') end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 409243e68a5..42cd94add79 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -45,7 +45,7 @@ module Gitlab end def key_from_hash(value) - value.is_a?(Hash) ? value.keys.first : value + value.is_a?(Hash) ? value.first.first : value end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 01437c67fa9..bd0f3e70749 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -37,11 +37,11 @@ project_tree: - :user - merge_requests: - :metrics - - :suggestions - notes: - :author - events: - :push_event_payload + - :suggestions - merge_request_diff: - :merge_request_diff_commits - :merge_request_diff_files @@ -80,6 +80,10 @@ project_tree: - :ci_cd_settings - :error_tracking_setting - :metrics_setting + - boards: + - lists: + - label: + - :priorities # Only include the following attributes for the models specified. included_attributes: @@ -133,6 +137,7 @@ excluded_attributes: - :packages_enabled - :mirror_last_update_at - :mirror_last_successful_update_at + - :emails_disabled namespaces: - :runners_token - :runners_token_encrypted @@ -216,6 +221,8 @@ methods: - :action project_badges: - :type + lists: + - :list_type # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index b145f37c052..a92e3862361 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -27,7 +27,7 @@ module Gitlab # {:merge_requests=>[:merge_request_diff, :notes]} def process_model_objects(model_object_hash) json_config_hash = {} - current_key = model_object_hash.keys.first + current_key = model_object_hash.first.first model_object_hash.values.flatten.each do |model_object| @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash } diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 345c7880e30..1de8a5bf9ec 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -3,6 +3,10 @@ module Gitlab module ImportExport class LfsRestorer + include Gitlab::Utils::StrongMemoize + + attr_accessor :project, :shared + def initialize(project:, shared:) @project = project @shared = shared @@ -17,7 +21,7 @@ module Gitlab true rescue => e - @shared.error(e) + shared.error(e) false end @@ -29,16 +33,57 @@ module Gitlab lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) lfs_object.file = File.open(path) unless lfs_object.file&.exists? + lfs_object.save! if lfs_object.changed? - @project.all_lfs_objects << lfs_object + repository_types(oid).each do |repository_type| + LfsObjectsProject.create!( + project: project, + lfs_object: lfs_object, + repository_type: repository_type + ) + end + end + + def repository_types(oid) + # We allow support for imports created before the `lfs-objects.json` + # file was generated. In this case, the restorer will link an LFS object + # with a single `lfs_objects_projects` relation. + # + # This allows us backwards-compatibility without version bumping. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30830#note_192608870 + return ['project'] unless has_lfs_json? + + lfs_json[oid] end def lfs_file_paths @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") end + def has_lfs_json? + strong_memoize(:has_lfs_json) do + File.exist?(lfs_json_path) + end + end + + def lfs_json + return {} unless has_lfs_json? + + @lfs_json ||= + begin + json = IO.read(lfs_json_path) + ActiveSupport::JSON.decode(json) + rescue + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + def lfs_storage_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 954f6f00078..18c590e1ca9 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -5,25 +5,40 @@ module Gitlab class LfsSaver include Gitlab::ImportExport::CommandLineUtil + attr_accessor :lfs_json, :project, :shared + + BATCH_SIZE = 100 + def initialize(project:, shared:) @project = project @shared = shared + @lfs_json = {} end def save - @project.all_lfs_objects.each do |lfs_object| - save_lfs_object(lfs_object) + project.all_lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end + + append_lfs_json_for_batch(batch) if write_lfs_json_enabled? end + write_lfs_json if write_lfs_json_enabled? + true rescue => e - @shared.error(e) + shared.error(e) false end private + def write_lfs_json_enabled? + ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) + end + def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) @@ -45,12 +60,36 @@ module Gitlab copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) end + def append_lfs_json_for_batch(lfs_objects_batch) + lfs_objects_projects = LfsObjectsProject + .select('lfs_objects.oid, array_agg(distinct lfs_objects_projects.repository_type) as repository_types') + .joins(:lfs_object) + .where(project: project, lfs_object: lfs_objects_batch) + .group('lfs_objects.oid') + + lfs_objects_projects.each do |group| + oid = group.oid + + lfs_json[oid] ||= [] + lfs_json[oid] += group.repository_types + end + end + + def write_lfs_json + mkdir_p(shared.export_path) + File.write(lfs_json_path, lfs_json.to_json) + end + def destination_path_for_object(lfs_object) File.join(lfs_export_path, lfs_object.oid) end def lfs_export_path - File.join(@shared.export_path, 'lfs-objects') + File.join(shared.export_path, ImportExport.lfs_objects_storage) + end + + def lfs_json_path + File.join(shared.export_path, ImportExport.lfs_objects_filename) end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index a154de5419e..4e976cfca3a 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -35,7 +35,7 @@ module Gitlab end def include?(old_author_id) - map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + map.has_key?(old_author_id) && map[old_author_id] != default_user_id end private @@ -50,6 +50,8 @@ module Gitlab @project.project_members.destroy_all # rubocop: disable DestroyAll ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + rescue => e + raise e, "Error adding importer user to project members. #{e.message}" end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dec99c23a2d..91fe4e5d074 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -130,6 +130,7 @@ module Gitlab def visibility_level level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level + level = Gitlab::VisibilityLevel::PRIVATE if level == Gitlab::VisibilityLevel::INTERNAL && Gitlab::CurrentSettings.restricted_visibility_levels.include?(level) { 'visibility_level' => level } end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 1b545b1d049..0be49e27acb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -185,7 +185,7 @@ module Gitlab return unless EXISTING_OBJECT_CHECK.include?(@relation_name) return unless @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group&.id + @relation_hash['group_id'] = @project.namespace_id end def reset_tokens! diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb new file mode 100644 index 00000000000..e6a5facb2a5 --- /dev/null +++ b/lib/gitlab/instrumentation_helper.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module InstrumentationHelper + extend self + + KEYS = %i(gitaly_calls gitaly_duration rugged_calls rugged_duration_ms).freeze + + def add_instrumentation_data(payload) + gitaly_calls = Gitlab::GitalyClient.get_request_count + + if gitaly_calls > 0 + payload[:gitaly_calls] = gitaly_calls + payload[:gitaly_duration] = Gitlab::GitalyClient.query_time_ms + end + + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + if rugged_calls > 0 + payload[:rugged_calls] = rugged_calls + payload[:rugged_duration_ms] = Gitlab::RuggedInstrumentation.query_time_ms + end + end + end +end diff --git a/lib/gitlab/kubernetes/default_namespace.rb b/lib/gitlab/kubernetes/default_namespace.rb new file mode 100644 index 00000000000..c95362b024b --- /dev/null +++ b/lib/gitlab/kubernetes/default_namespace.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class DefaultNamespace + attr_reader :cluster, :project + + delegate :platform_kubernetes, to: :cluster + + ## + # Ideally we would just use an environment record here instead of + # passing a project and name/slug separately, but we need to be able + # to look up namespaces before the environment has been persisted. + def initialize(cluster, project:) + @cluster = cluster + @project = project + end + + def from_environment_name(name) + from_environment_slug(generate_slug(name)) + end + + def from_environment_slug(slug) + default_platform_namespace(slug) || default_project_namespace(slug) + end + + private + + def default_platform_namespace(slug) + return unless platform_kubernetes&.namespace.present? + + if cluster.managed? && cluster.namespace_per_environment? + "#{platform_kubernetes.namespace}-#{slug}" + else + platform_kubernetes.namespace + end + end + + def default_project_namespace(slug) + namespace_slug = "#{project.path}-#{project.id}".downcase + + if cluster.namespace_per_environment? + namespace_slug += "-#{slug}" + end + + Gitlab::NamespaceSanitizer.sanitize(namespace_slug) + end + + ## + # Environment slug can be predicted given an environment + # name, so even if the environment isn't persisted yet we + # still know what to look for. + def generate_slug(name) + Gitlab::Slug::Environment.new(name).generate + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 42c4745ff98..6e4286589c1 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -3,8 +3,8 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.12.3'.freeze - KUBECTL_VERSION = '1.11.7'.freeze + HELM_VERSION = '2.14.3'.freeze + KUBECTL_VERSION = '1.11.10'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze SERVICE_ACCOUNT = 'tiller'.freeze CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index 876994d2678..dcf22e7abb6 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -7,19 +7,24 @@ module Gitlab include BaseCommand include ClientCommand + attr_reader :predelete, :postdelete attr_accessor :name, :files - def initialize(name:, rbac:, files:) + def initialize(name:, rbac:, files:, predelete: nil, postdelete: nil) @name = name @files = files @rbac = rbac + @predelete = predelete + @postdelete = postdelete end def generate_script super + [ init_command, wait_for_tiller_command, - delete_command + predelete, + delete_command, + postdelete ].compact.join("\n") end @@ -38,17 +43,6 @@ module Gitlab command.shelljoin end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 9744a5f3d8a..f572bc43533 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -27,9 +27,9 @@ module Gitlab wait_for_tiller_command, repository_command, repository_update_command, - preinstall_command, + preinstall, install_command, - postinstall_command + postinstall ].compact.join("\n") end @@ -58,14 +58,6 @@ module Gitlab command.shelljoin end - def preinstall_command - preinstall.join("\n") if preinstall - end - - def postinstall_command - postinstall.join("\n") if postinstall - end - def install_flag ['--install'] end diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb new file mode 100644 index 00000000000..a35ffa34c58 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/reset_command.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class ResetCommand + include BaseCommand + include ClientCommand + + attr_reader :name, :files + + def initialize(name:, rbac:, files:) + @name = name + @files = files + @rbac = rbac + end + + def generate_script + super + [ + reset_helm_command, + delete_tiller_replicaset + ].join("\n") + end + + def rbac? + @rbac + end + + def pod_name + "uninstall-#{name}" + end + + private + + # This method can be delete once we upgrade Helm to > 12.13.0 + # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/27096#note_159695900 + # + # Tracking this method to be removed here: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/52791#note_199374155 + def delete_tiller_replicaset + delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def reset_helm_command + command = %w[helm reset] + optional_tls_flags + + command.shelljoin + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 1350924cd76..64317225ec6 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -128,7 +128,7 @@ module Gitlab private def validate_url! - return if Gitlab::CurrentSettings.allow_local_requests_from_hooks_and_services? + return if Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false) end diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb new file mode 100644 index 00000000000..981eb5681dc --- /dev/null +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module KubectlCmd + class << self + def delete(*args) + %w(kubectl delete).concat(args).shelljoin + end + + def apply_file(filename, *args) + raise ArgumentError, "filename is not present" unless filename.present? + + %w(kubectl apply -f).concat([filename], args).shelljoin + end + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index bbdd094e33b..b23efd64dee 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -101,7 +101,7 @@ module Gitlab # GitHub Rate Limit API returns 404 when the rate limit is # disabled. In this case we just want to return gracefully # instead of spitting out an error. - rescue Octokit::NotFound + rescue ::Octokit::NotFound nil end diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb deleted file mode 100644 index 0628e82e592..00000000000 --- a/lib/gitlab/metrics/dashboard/base_service.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -module Gitlab - module Metrics - module Dashboard - class BaseService < ::BaseService - PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - - def get_dashboard - return error('Insufficient permissions.', :unauthorized) unless allowed? - - success(dashboard: process_dashboard) - rescue NOT_FOUND_ERROR - error("#{dashboard_path} could not be found.", :not_found) - rescue PROCESSING_ERROR => e - error(e.message, :unprocessable_entity) - end - - # Summary of all known dashboards for the service. - # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] - def self.all_dashboard_paths(_project) - raise NotImplementedError - end - - # Returns an un-processed dashboard from the cache. - def raw_dashboard - Gitlab::Metrics::Dashboard::Cache.fetch(cache_key) { get_raw_dashboard } - end - - private - - # Determines whether users should be able to view - # dashboards at all. - def allowed? - Ability.allowed?(current_user, :read_environment, project) - end - - # Returns a new dashboard Hash, supplemented with DB info - def process_dashboard - Gitlab::Metrics::Dashboard::Processor - .new(project, params[:environment], raw_dashboard) - .process(insert_project_metrics: insert_project_metrics?) - end - - # @return [String] Relative filepath of the dashboard yml - def dashboard_path - params[:dashboard_path] - end - - # @return [Hash] an unmodified dashboard - def get_raw_dashboard - raise NotImplementedError - end - - # @return [String] - def cache_key - raise NotImplementedError - end - - # Determines whether custom metrics should be included - # in the processed output. - # @return [Boolean] - def insert_project_metrics? - false - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb new file mode 100644 index 00000000000..3c39a7c6911 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/defaults.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Central point for managing default attributes from within +# the metrics dashboard module. +module Gitlab + module Metrics + module Dashboard + module Defaults + DEFAULT_PANEL_TYPE = 'area-chart' + DEFAULT_PANEL_WEIGHT = 0 + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb b/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb deleted file mode 100644 index 81ed8922e17..00000000000 --- a/lib/gitlab/metrics/dashboard/dynamic_dashboard_service.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# Responsible for returning a filtered system dashboard -# containing only the default embedded metrics. In future, -# this class may be updated to support filtering to -# alternate metrics/panels. -# -# Why isn't this filtering in a processing stage? By filtering -# here, we ensure the dynamically-determined dashboard is cached. -# -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class DynamicDashboardService < Gitlab::Metrics::Dashboard::BaseService - # For the default filtering for embedded metrics, - # uses the 'id' key in dashboard-yml definition for - # identification. - DEFAULT_EMBEDDED_METRICS_IDENTIFIERS = %w( - system_metrics_kubernetes_container_memory_total - system_metrics_kubernetes_container_cores_total - ).freeze - - # Returns a new dashboard with only the matching - # metrics from the system dashboard, stripped of groups. - # @return [Hash] - def raw_dashboard - panels = panel_groups.each_with_object([]) do |group, panels| - matched_panels = group['panels'].select { |panel| matching_panel?(panel) } - - panels.concat(matched_panels) - end - - { 'panel_groups' => [{ 'panels' => panels }] } - end - - def cache_key - "dynamic_metrics_dashboard_#{metric_identifiers.join('_')}" - end - - private - - # Returns an array of the panels groups on the - # system dashboard - def panel_groups - Gitlab::Metrics::Dashboard::SystemDashboardService - .new(project, nil) - .raw_dashboard['panel_groups'] - end - - # Identifies a panel as "matching" if any metric ids in - # the panel is in the list of identifiers to collect. - def matching_panel?(panel) - panel['metrics'].any? do |metric| - metric_identifiers.include?(metric['id']) - end - end - - def metric_identifiers - DEFAULT_EMBEDDED_METRICS_IDENTIFIERS - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb new file mode 100644 index 00000000000..1739a4e6738 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Central point for managing errors from within the metrics +# dashboard module. Handles errors from dashboard retrieval +# and processing steps, as well as defines shared error classes. +module Gitlab + module Metrics + module Dashboard + module Errors + PanelNotFoundError = Class.new(StandardError) + + PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError + NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError + + def handle_errors(error) + case error + when PROCESSING_ERROR + error(error.message, :unprocessable_entity) + when NOT_FOUND_ERROR + error("#{dashboard_path} could not be found.", :not_found) + when PanelNotFoundError + error(error.message, :not_found) + else + raise error + end + end + + def panels_not_found!(opts) + raise PanelNotFoundError.new("No panels matching properties #{opts}") + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index d7491d1553d..66c4d662a6c 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -12,21 +12,37 @@ module Gitlab # @param project [Project] # @param user [User] # @param environment [Environment] - # @param opts - dashboard_path [String] Path at which the - # dashboard can be found. Nil values will - # default to the system dashboard. - # @param opts - embedded [Boolean] Determines whether the + # @param options - embedded [Boolean] Determines whether the # dashboard is to be rendered as part of an # issue or location other than the primary # metrics dashboard UI. Returns only the # Memory/CPU charts of the system dash. + # @param options - dashboard_path [String] Path at which the + # dashboard can be found. Nil values will + # default to the system dashboard. + # @param options - group [String] Title of the group + # to which a panel might belong. Used by + # embedded dashboards. + # @param options - title [String] Title of the panel. + # Used by embedded dashboards. + # @param options - y_label [String] Y-Axis label of + # a panel. Used by embedded dashboards. # @return [Hash] - def find(project, user, environment, dashboard_path: nil, embedded: false) - service_for_path(dashboard_path, embedded: embedded) - .new(project, user, environment: environment, dashboard_path: dashboard_path) + def find(project, user, environment, options = {}) + service_for(options) + .new(project, user, options.merge(environment: environment)) .get_dashboard end + # Returns a dashboard without any supplemental info. + # Returns only full, yml-defined dashboards. + # @return [Hash] + def find_raw(project, dashboard_path: nil) + service_for(dashboard_path: dashboard_path) + .new(project, nil, dashboard_path: dashboard_path) + .raw_dashboard + end + # Summary of all known dashboards. # @return [Array<Hash>] ex) [{ path: String, # display_name: String, @@ -46,27 +62,16 @@ module Gitlab private - def service_for_path(dashboard_path, embedded:) - return dynamic_service if embedded - return system_service if system_dashboard?(dashboard_path) - - project_service - end - def system_service - Gitlab::Metrics::Dashboard::SystemDashboardService + ::Metrics::Dashboard::SystemDashboardService end def project_service - Gitlab::Metrics::Dashboard::ProjectDashboardService - end - - def dynamic_service - Gitlab::Metrics::Dashboard::DynamicDashboardService + ::Metrics::Dashboard::ProjectDashboardService end - def system_dashboard?(filepath) - !filepath || system_service.system_dashboard?(filepath) + def service_for(options) + Gitlab::Metrics::Dashboard::ServiceSelector.call(options) end end end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb deleted file mode 100644 index 5a1c4ecf886..00000000000 --- a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -# Searches a projects repository for a metrics dashboard and formats the output. -# Expects any custom dashboards will be located in `.gitlab/dashboards` -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService - DASHBOARD_ROOT = ".gitlab/dashboards" - - class << self - def all_dashboard_paths(project) - file_finder(project) - .list_files_for(DASHBOARD_ROOT) - .map do |filepath| - { - path: filepath, - display_name: name_for_path(filepath), - default: false - } - end - end - - def file_finder(project) - Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') - end - - # Grabs the filepath after the base directory. - def name_for_path(filepath) - filepath.delete_prefix("#{DASHBOARD_ROOT}/") - end - end - - private - - # Searches the project repo for a custom-defined dashboard. - def get_raw_dashboard - yml = self.class.file_finder(project).read(dashboard_path) - - YAML.safe_load(yml) - end - - def cache_key - "project_#{project.id}_metrics_dashboard_#{dashboard_path}" - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb new file mode 100644 index 00000000000..934ba9145a2 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Responsible for determining which dashboard service should +# be used to fetch or generate a dashboard hash. +# The services can be considered in two categories - embeds +# and dashboards. Embeds are all portions of dashboards. +module Gitlab + module Metrics + module Dashboard + class ServiceSelector + SERVICES = ::Metrics::Dashboard + + class << self + include Gitlab::Utils::StrongMemoize + + # Returns a class which inherits from the BaseService + # class that can be used to obtain a dashboard. + # @return [Gitlab::Metrics::Dashboard::Services::BaseService] + def call(params) + return SERVICES::CustomMetricEmbedService if custom_metric_embed?(params) + return SERVICES::DynamicEmbedService if dynamic_embed?(params) + return SERVICES::DefaultEmbedService if params[:embedded] + return SERVICES::SystemDashboardService if system_dashboard?(params[:dashboard_path]) + return SERVICES::ProjectDashboardService if params[:dashboard_path] + + default_service + end + + private + + def default_service + SERVICES::SystemDashboardService + end + + def system_dashboard?(filepath) + SERVICES::SystemDashboardService.system_dashboard?(filepath) + end + + def custom_metric_embed?(params) + SERVICES::CustomMetricEmbedService.valid_params?(params) + end + + def dynamic_embed?(params) + SERVICES::DynamicEmbedService.valid_params?(params) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index 0db7b176e8d..514ed50e58d 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -5,11 +5,11 @@ module Gitlab module Dashboard module Stages class BaseStage + include Gitlab::Metrics::Dashboard::Defaults + DashboardProcessingError = Class.new(StandardError) LayoutError = Class.new(DashboardProcessingError) - DEFAULT_PANEL_TYPE = 'area-chart' - attr_reader :project, :environment, :dashboard def initialize(project, environment, dashboard) diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb index 221610a14d1..643be309992 100644 --- a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -97,7 +97,7 @@ module Gitlab end def new_metric(metric) - metric.queries.first.merge(metric_id: metric.id) + metric.to_metric_hash end end end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb deleted file mode 100644 index 82421572f4a..00000000000 --- a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -# Fetches the system metrics dashboard and formats the output. -# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. -module Gitlab - module Metrics - module Dashboard - class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService - SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' - SYSTEM_DASHBOARD_NAME = 'Default' - - class << self - def all_dashboard_paths(_project) - [{ - path: SYSTEM_DASHBOARD_PATH, - display_name: SYSTEM_DASHBOARD_NAME, - default: true - }] - end - - def system_dashboard?(filepath) - filepath == SYSTEM_DASHBOARD_PATH - end - end - - private - - def dashboard_path - SYSTEM_DASHBOARD_PATH - end - - # Returns the base metrics shipped with every GitLab service. - def get_raw_dashboard - yml = File.read(Rails.root.join(dashboard_path)) - - YAML.safe_load(yml) - end - - def cache_key - "metrics_dashboard_#{dashboard_path}" - end - - def insert_project_metrics? - true - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index b197e7ca86b..94f8b2e02b1 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -21,14 +21,26 @@ module Gitlab \/(?<environment>\d+) \/metrics (?<query> - \?[a-z0-9_=-]+ - (&[a-z0-9_=-]+)* + \?[a-zA-Z0-9%.()+_=-]+ + (&[a-zA-Z0-9%.()+_=-]+)* )? (?<anchor>\#[a-z0-9_-]+)? ) }x end + # Parses query params out from full url string into hash. + # + # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' + # --> { title: 'Title', group: 'Group' } + def parse_query(url) + query_string = URI.parse(url).query.to_s + + CGI.parse(query_string) + .transform_values { |value| value.first } + .symbolize_keys + end + # Builds a metrics dashboard url based on the passed in arguments def build_dashboard_url(*args) Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5138b37f83e..1eae0a7bf45 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -15,19 +15,14 @@ module Gitlab @last_step = nil @metrics = [] - - @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) - @last_major_gc = Delta.new(GC.stat[:major_gc_count]) end def sample sample_memory_usage sample_file_descriptors - sample_gc flush ensure - GC::Profiler.clear @metrics.clear end @@ -43,23 +38,6 @@ module Gitlab add_metric('file_descriptors', value: System.file_descriptor_count) end - def sample_gc - time = GC::Profiler.total_time * 1000.0 - stats = GC.stat.merge(total_time: time) - - # We want the difference of GC runs compared to the last sample, not the - # total amount since the process started. - stats[:minor_gc_count] = - @last_minor_gc.compared_with(stats[:minor_gc_count]) - - stats[:major_gc_count] = - @last_major_gc.compared_with(stats[:major_gc_count]) - - stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count] - - add_metric('gc_statistics', stats) - end - def add_metric(series, values, tags = {}) prefix = sidekiq? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 4e835f37c04..8a24d4f3663 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -15,7 +15,6 @@ module Gitlab puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), - puma_phase: ::Gitlab::Metrics.gauge(:puma_phase, 'Phase number (increased during phased restarts)'), puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), @@ -54,7 +53,6 @@ module Gitlab last_status = worker['last_status'] labels = { worker: "worker_#{worker['index']}" } - metrics[:puma_phase].set(labels, worker['phase']) set_worker_metrics(last_status, labels) if last_status.present? end end @@ -76,7 +74,6 @@ module Gitlab metrics[:puma_workers].set(labels, stats['workers']) metrics[:puma_running_workers].set(labels, stats['booted_workers']) metrics[:puma_stale_workers].set(labels, stats['old_workers']) - metrics[:puma_phase].set(labels, stats['phase']) end def set_worker_metrics(stats, labels = {}) diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index eef802caabb..3bfa3da35e0 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,8 +6,12 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + GC_REPORT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.05, 0.1, 0.5].freeze + def initialize(interval) - metrics[:process_start_time_seconds].set(labels.merge(worker_label), Time.now.to_i) + GC::Profiler.clear + + metrics[:process_start_time_seconds].set(labels, Time.now.to_i) super end @@ -30,18 +34,18 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum), - memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum), + file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels), + memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels), process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels), - total_time: ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) + gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) } GC.stat.keys.each do |key| - metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) + metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels) end metrics @@ -50,47 +54,41 @@ module Gitlab def sample start_time = System.monotonic_time - metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) - metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:process_cpu_seconds_total].set(labels, ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels, ::Gitlab::Metrics::System.max_open_file_descriptors) set_memory_usage_metrics sample_gc metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) - ensure - GC::Profiler.clear end private def sample_gc - # Collect generic GC stats. + # Observe all GC samples + sample_gc_reports.each do |report| + metrics[:gc_duration_seconds].observe(labels, report[:GC_TIME]) + end + + # Collect generic GC stats GC.stat.each do |key, value| metrics[key].set(labels, value) end + end - # Collect the GC time since last sample in float seconds. - metrics[:total_time].increment(labels, GC::Profiler.total_time) + def sample_gc_reports + GC::Profiler.enable + GC::Profiler.raw_data + ensure + GC::Profiler.clear end def set_memory_usage_metrics memory_usage = System.memory_usage - memory_labels = labels.merge(worker_label) - metrics[:memory_bytes].set(memory_labels, memory_usage) - metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) - end - - def worker_label - return { worker: 'sidekiq' } if Sidekiq.server? - return {} unless defined?(Unicorn::Worker) - - worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no - { worker: worker_no } - else - { worker: 'master' } - end + metrics[:memory_bytes].set(labels, memory_usage) + metrics[:process_resident_memory_bytes].set(labels, memory_usage) end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 01db507761b..2ee7144fe2f 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -50,7 +50,8 @@ module Gitlab def observe(key, duration) return unless current_transaction - metric_cache_operation_duration_seconds.observe(current_transaction.labels.merge({ operation: key }), duration / 1000.0) + metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key })) + metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0) current_transaction.increment(:cache_duration, duration, false) current_transaction.increment(:cache_count, 1, false) current_transaction.increment("cache_#{key}_duration".to_sym, duration, false) @@ -63,12 +64,20 @@ module Gitlab Transaction.current end + def metric_cache_operations_total + @metric_cache_operations_total ||= ::Gitlab::Metrics.counter( + :gitlab_cache_operations_total, + 'Cache operations', + Transaction::BASE_LABELS + ) + end + def metric_cache_operation_duration_seconds @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', - Transaction::BASE_LABELS.merge({ action: nil }), - [0.001, 0.01, 0.1, 1, 10] + {}, + [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] ) end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 38b32770e90..c06f106ffe1 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -32,11 +32,6 @@ module Gitlab # Returns the maximum depth starting from the base # A base object with no children has a maximum depth of `1` def max_descendants_depth - unless hierarchy_supported? - # This makes the return value consistent with the case where hierarchy is supported - return descendants_base.exists? ? 1 : nil - end - base_and_descendants(with_depth: true).maximum(DEPTH_COLUMN) end @@ -66,8 +61,6 @@ module Gitlab # each parent. # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - return ancestors_base unless hierarchy_supported? - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order @@ -81,10 +74,6 @@ module Gitlab # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree def base_and_descendants(with_depth: false) - unless hierarchy_supported? - return with_depth ? descendants_base.select("1 as #{DEPTH_COLUMN}", objects_table[Arel.star]) : descendants_base - end - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) end @@ -112,8 +101,6 @@ module Gitlab # If nested objects are not supported, ancestors_base is returned. # rubocop: disable CodeReuse/ActiveRecord def all_objects - return ancestors_base unless hierarchy_supported? - ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte @@ -135,10 +122,6 @@ module Gitlab private - def hierarchy_supported? - Gitlab::Database.postgresql? - end - # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb new file mode 100644 index 00000000000..2dd7d08a58b --- /dev/null +++ b/lib/gitlab/octokit/middleware.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Octokit + class Middleware + def initialize(app) + @app = app + end + + def call(env) + Gitlab::UrlBlocker.validate!(env[:url], { allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests? }) + + @app.call(env) + end + + private + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index a13b3f9e069..f96466b2b00 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -175,6 +175,10 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end + def project_wiki_git_route_regex + @project_wiki_git_route_regex ||= /#{PATH_REGEX_STR}\.wiki/.freeze + end + def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb deleted file mode 100644 index 3a27e26eaba..00000000000 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -# Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb -# PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb -module Gitlab - module PerformanceBar - module PeekQueryTracker - def sorted_queries - PEEK_DB_CLIENT.query_details - .sort { |a, b| b[:duration] <=> a[:duration] } - end - - def results - super.merge(queries: sorted_queries) - end - - private - - def setup_subscribers - super - - # Reset each counter when a new request starts - before_request do - PEEK_DB_CLIENT.query_details = [] - end - - subscribe('sql.active_record') do |_, start, finish, _, data| - if Gitlab::SafeRequestStore.store[:peek_enabled] - unless data[:cached] - backtrace = Gitlab::Profiler.clean_backtrace(caller) - track_query(data[:sql].strip, data[:binds], backtrace, start, finish) - end - end - end - end - - def track_query(raw_query, bindings, backtrace, start, finish) - duration = (finish - start) * 1000.0 - query_info = { duration: duration.round(3), sql: raw_query, backtrace: backtrace } - - PEEK_DB_CLIENT.query_details << query_info - end - end - end -end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 890228e5e78..ec7671f9a8b 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -21,6 +21,9 @@ module Gitlab lib/gitlab/profiler.rb lib/gitlab/correlation_id.rb lib/gitlab/webpack/dev_server_middleware.rb + lib/gitlab/sidekiq_status/ + lib/gitlab/sidekiq_logging/ + lib/gitlab/sidekiq_middleware/ ].freeze # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) @@ -166,7 +169,7 @@ module Gitlab [model, times.count, times.sum] end - summarised_load_times.sort_by(&:last).reverse.each do |(model, query_count, time)| + summarised_load_times.sort_by(&:last).reverse_each do |(model, query_count, time)| logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb new file mode 100644 index 00000000000..a9270cd536e --- /dev/null +++ b/lib/gitlab/project_authorizations.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +# This class relies on Common Table Expressions to efficiently get all data, +# including data for nested groups. +module Gitlab + class ProjectAuthorizations + attr_reader :user + + # user - The User object for which to calculate the authorizations. + def initialize(user) + @user = user + end + + def calculate + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + projects = Project.arel_table + links = ProjectGroupLink.arel_table + + relations = [ + # The project a user has direct access to. + user.projects.select_for_project_authorization, + + # The personal projects of the user. + user.personal_projects.select_as_maintainer_for_project_authorization, + + # Projects that belong directly to any of the groups the user has + # access to. + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects), + + # Projects shared with any of the namespaces the user has access to. + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + ] + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end + + private + + # Builds a recursive CTE that gets all the groups the current user has + # access to, including any nested groups. + def recursive_cte + cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) + members = Member.arel_table + namespaces = Namespace.arel_table + + # Namespaces the user is a member of. + cte << user.groups + .select([namespaces[:id], members[:access_level]]) + .except(:order) + + # Sub groups of any groups the user is a member of. + cte << Group.select([ + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) + .joins(join_cte(cte)) + .joins(join_members) + .except(:order) + + cte + end + + # Builds a LEFT JOIN to join optional memberships onto the CTE. + def join_members + members = Member.arel_table + namespaces = Namespace.arel_table + + cond = members[:source_id] + .eq(namespaces[:id]) + .and(members[:source_type].eq('Namespace')) + .and(members[:requested_at].eq(nil)) + .and(members[:user_id].eq(user.id)) + + Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) + end + + # Builds an INNER JOIN to join namespaces onto the CTE. + def join_cte(cte) + namespaces = Namespace.arel_table + cond = cte.table[:id].eq(namespaces[:parent_id]) + + Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) + end + + def greatest(left, right, column_alias) + sql_function('GREATEST', [left, right], column_alias) + end + + def least(left, right, column_alias) + sql_function('LEAST', [left, right], column_alias) + end + + def sql_function(name, args, column_alias) + alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) + end + + def alias_as_column(value, alias_to) + Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb deleted file mode 100644 index 2372a316ab0..00000000000 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when supporting nested groups. - # - # This class relies on Common Table Expressions to efficiently get all data, - # including data for nested groups. As a result this class can only be used - # on PostgreSQL. - class WithNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - cte = recursive_cte - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([links[:project_id], - least(cte_alias[:access_level], - links[:group_access], - 'access_level')]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] - - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) - end - - private - - # Builds a recursive CTE that gets all the groups the current user has - # access to, including any nested groups. - def recursive_cte - cte = Gitlab::SQL::RecursiveCTE.new(:namespaces_cte) - members = Member.arel_table - namespaces = Namespace.arel_table - - # Namespaces the user is a member of. - cte << user.groups - .select([namespaces[:id], members[:access_level]]) - .except(:order) - - # Sub groups of any groups the user is a member of. - cte << Group.select([namespaces[:id], - greatest(members[:access_level], - cte.table[:access_level], 'access_level')]) - .joins(join_cte(cte)) - .joins(join_members) - .except(:order) - - cte - end - - # Builds a LEFT JOIN to join optional memberships onto the CTE. - def join_members - members = Member.arel_table - namespaces = Namespace.arel_table - - cond = members[:source_id] - .eq(namespaces[:id]) - .and(members[:source_type].eq('Namespace')) - .and(members[:requested_at].eq(nil)) - .and(members[:user_id].eq(user.id)) - - Arel::Nodes::OuterJoin.new(members, Arel::Nodes::On.new(cond)) - end - - # Builds an INNER JOIN to join namespaces onto the CTE. - def join_cte(cte) - namespaces = Namespace.arel_table - cond = cte.table[:id].eq(namespaces[:parent_id]) - - Arel::Nodes::InnerJoin.new(cte.table, Arel::Nodes::On.new(cond)) - end - - def greatest(left, right, column_alias) - sql_function('GREATEST', [left, right], column_alias) - end - - def least(left, right, column_alias) - sql_function('LEAST', [left, right], column_alias) - end - - def sql_function(name, args, column_alias) - alias_as_column(Arel::Nodes::NamedFunction.new(name, args), column_alias) - end - - def alias_as_column(value, alias_to) - Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) - end - end - end -end diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb deleted file mode 100644 index 50b41b17649..00000000000 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ProjectAuthorizations - # Calculating new project authorizations when not supporting nested groups. - class WithoutNestedGroups - attr_reader :user - - # user - The User object for which to calculate the authorizations. - def initialize(user) - @user = user - end - - def calculate - relations = [ - # Projects the user is a direct member of - user.projects.select_for_project_authorization, - - # Personal projects - user.personal_projects.select_as_maintainer_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization - ] - - ProjectAuthorization - .unscoped - .select_from_union(relations) - end - end - end -end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0f3b97e2317..2669adb8455 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -29,6 +29,21 @@ module Gitlab end end + def formatted_count(scope) + case scope + when 'blobs' + blobs_count.to_s + when 'notes' + formatted_limited_count(limited_notes_count) + when 'wiki_blobs' + wiki_blobs_count.to_s + when 'commits' + commits_count.to_s + else + super + end + end + def users super.where(id: @project.team.members) # rubocop:disable CodeReuse/ActiveRecord end @@ -108,7 +123,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def notes_finder(type) - NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') + NotesFinder.new(@current_user, search: query, target_type: type, project: project).execute.user.order('updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord @@ -134,9 +149,11 @@ module Gitlab project.repository.commit(key) if Commit.valid_hash?(key) end + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation - project + Project.where(id: project).select(:id).reorder(nil) end + # rubocop: enabled CodeReuse/ActiveRecord def filter_milestones_by_project(milestones) return Milestone.none unless Ability.allowed?(@current_user, :read_milestone, @project) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 99885be8755..fa1d1203842 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -13,11 +13,23 @@ module Gitlab end def archive_path - Rails.root.join("vendor/project_templates/#{name}.tar.gz") + self.class.archive_directory.join(archive_filename) + end + + def archive_filename + "#{name}.tar.gz" end def clone_url - "https://gitlab.com/gitlab-org/project-templates/#{name}.git" + "#{preview}.git" + end + + def project_path + URI.parse(preview).path.sub(%r{\A/}, '') + end + + def uri_encoded_project_path + ERB::Util.url_encode(project_path) end def ==(other) @@ -54,7 +66,7 @@ module Gitlab end def archive_directory - Rails.root.join("vendor_directory/project_templates") + Rails.root.join("vendor/project_templates") end end end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 9cc21129547..ba2d33ee1c1 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -4,12 +4,9 @@ module Gitlab module Prometheus module QueryVariables def self.call(environment) - deployment_platform = environment.deployment_platform - namespace = deployment_platform&.kubernetes_namespace_for(environment.project) || '' - { ci_environment_slug: environment.slug, - kube_namespace: namespace, + kube_namespace: environment.deployment_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index f13156f898e..9fefffefcde 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,6 +3,7 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient + include Gitlab::Utils::StrongMemoize Error = Class.new(StandardError) QueryError = Class.new(Gitlab::PrometheusClient::Error) @@ -14,10 +15,17 @@ module Gitlab # Minimal value of the `step` parameter for `query_range` in seconds. QUERY_RANGE_MIN_STEP = 60 - attr_reader :rest_client, :headers + # Key translation between RestClient and Gitlab::HTTP (HTTParty) + RESTCLIENT_GITLAB_HTTP_KEYMAP = { + ssl_cert_store: :cert_store + }.freeze - def initialize(rest_client) - @rest_client = rest_client + attr_reader :api_url, :options + private :api_url, :options + + def initialize(api_url, options = {}) + @api_url = api_url.chomp('/') + @options = options end def ping @@ -27,14 +35,10 @@ module Gitlab def proxy(type, args) path = api_path(type) get(path, args) - rescue RestClient::ExceptionWithResponse => ex - if ex.response - ex.response - else - raise PrometheusClient::Error, "Network connection error" - end - rescue RestClient::Exception - raise PrometheusClient::Error, "Network connection error" + rescue Gitlab::HTTP::ResponseError => ex + raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code) + + handle_response(ex.response) end def query(query, time: Time.now) @@ -78,50 +82,58 @@ module Gitlab private def api_path(type) - ['api', 'v1', type].join('/') + [api_url, 'api', 'v1', type].join('/') end def json_api_get(type, args = {}) path = api_path(type) response = get(path, args) handle_response(response) - rescue RestClient::ExceptionWithResponse => ex - if ex.response - handle_exception_response(ex.response) - else - raise PrometheusClient::Error, "Network connection error" + rescue Gitlab::HTTP::ResponseError => ex + raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code) + + handle_response(ex.response) + end + + def gitlab_http_key(key) + RESTCLIENT_GITLAB_HTTP_KEYMAP[key] || key + end + + def mapped_options + options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h + end + + def http_options + strong_memoize(:http_options) do + { follow_redirects: false }.merge(mapped_options) end - rescue RestClient::Exception - raise PrometheusClient::Error, "Network connection error" end def get(path, args) - rest_client[path].get(params: args) + Gitlab::HTTP.get(path, { query: args }.merge(http_options) ) rescue SocketError - raise PrometheusClient::Error, "Can't connect to #{rest_client.url}" + raise PrometheusClient::Error, "Can't connect to #{api_url}" rescue OpenSSL::SSL::SSLError - raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" + raise PrometheusClient::Error, "#{api_url} contains invalid SSL data" rescue Errno::ECONNREFUSED raise PrometheusClient::Error, 'Connection refused' end def handle_response(response) - json_data = parse_json(response.body) - if response.code == 200 && json_data['status'] == 'success' - json_data['data'] || {} - else - raise PrometheusClient::Error, "#{response.code} - #{response.body}" - end - end + response_code = response.try(:code) + response_body = response.try(:body) + + raise PrometheusClient::Error, "#{response_code} - #{response_body}" unless response_code + + json_data = parse_json(response_body) if [200, 400].include?(response_code) - def handle_exception_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} - elsif response.code == 400 - json_data = parse_json(response.body) + case response_code + when 200 + json_data['data'] if response['status'] == 'success' + when 400 raise PrometheusClient::QueryError, json_data['error'] || 'Bad data received' else - raise PrometheusClient::Error, "#{response.code} - #{response.body}" + raise PrometheusClient::Error, "#{response_code} - #{response_body}" end end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 3137676ba4b..682edfc4259 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -4,7 +4,14 @@ module Gitlab class PushOptions VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { - keys: [:create, :merge_when_pipeline_succeeds, :target] + keys: [ + :create, + :description, + :merge_when_pipeline_succeeds, + :remove_source_branch, + :target, + :title + ] }, ci: { keys: [:skip] diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 93030fd454e..ebdae139315 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -3,8 +3,8 @@ module Gitlab module QuickActions class CommandDefinition - attr_accessor :name, :aliases, :description, :explanation, :params, - :condition_block, :parse_params_block, :action_block, :warning, :types + attr_accessor :name, :aliases, :description, :explanation, :execution_message, + :params, :condition_block, :parse_params_block, :action_block, :warning, :types def initialize(name, attributes = {}) @name = name @@ -13,6 +13,7 @@ module Gitlab @description = attributes[:description] || '' @warning = attributes[:warning] || '' @explanation = attributes[:explanation] || '' + @execution_message = attributes[:execution_message] || '' @params = attributes[:params] || [] @condition_block = attributes[:condition_block] @parse_params_block = attributes[:parse_params_block] @@ -48,13 +49,23 @@ module Gitlab end def execute(context, arg) - return if noop? || !available?(context) + return unless executable?(context) count_commands_executed_in(context) execute_block(action_block, context, arg) end + def execute_message(context, arg) + return unless executable?(context) + + if execution_message.respond_to?(:call) + execute_block(execution_message, context, arg) + else + execution_message + end + end + def to_h(context) desc = description if desc.respond_to?(:call) @@ -77,6 +88,10 @@ module Gitlab private + def executable?(context) + !noop? && available?(context) + end + def count_commands_executed_in(context) return unless context.respond_to?(:commands_executed_count=) diff --git a/lib/gitlab/quick_actions/commit_actions.rb b/lib/gitlab/quick_actions/commit_actions.rb index 1018910e8e9..49f5ddf24eb 100644 --- a/lib/gitlab/quick_actions/commit_actions.rb +++ b/lib/gitlab/quick_actions/commit_actions.rb @@ -16,6 +16,13 @@ module Gitlab _("Tags this commit to %{tag_name}.") % { tag_name: tag_name } end end + execution_message do |tag_name, message| + if message.present? + _("Tagged this commit to %{tag_name} with \"%{message}\".") % { tag_name: tag_name, message: message } + else + _("Tagged this commit to %{tag_name}.") % { tag_name: tag_name } + end + end params 'v1.2.3 <message>' parse_params do |tag_name_and_message| tag_name_and_message.split(' ', 2) diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index ecb2169151e..5abbd377642 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -66,6 +66,35 @@ module Gitlab @explanation = block_given? ? block : text end + # Allows to provide a message about quick action execution result, success or failure. + # This message is shown after quick action execution and after saving the note. + # + # Example: + # + # execution_message do |arguments| + # "Added label(s) #{arguments.join(' ')}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + # + # Note: The execution_message won't be executed unless the condition block returns true. + # execution_message block is executed always after the command block has run, + # for this reason if the condition block doesn't return true after the command block has + # run you need to set the @execution_message variable inside the command block instead as + # shown in the following example. + # + # Example using instance variable: + # + # command :command_key do |arguments| + # # Awesome code block + # @execution_message[:command_key] = 'command_key executed successfully' + # end + # + def execution_message(text = '', &block) + @execution_message = block_given? ? block : text + end + # Allows to define type(s) that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # @@ -121,10 +150,16 @@ module Gitlab # comment. # It accepts aliases and takes a block. # + # You can also set the @execution_message instance variable, on conflicts with + # execution_message method the instance variable has precedence. + # # Example: # # command :my_command, :alias_for_my_command do |arguments| # # Awesome code block + # @updates[:my_command] = 'foo' + # + # @execution_message[:my_command] = 'my_command executed successfully' # end def command(*command_names, &block) define_command(CommandDefinition, *command_names, &block) @@ -158,6 +193,7 @@ module Gitlab description: @description, warning: @warning, explanation: @explanation, + execution_message: @execution_message, params: @params, condition_block: @condition_block, parse_params_block: @parse_params_block, @@ -173,6 +209,7 @@ module Gitlab @description = nil @explanation = nil + @execution_message = nil @params = nil @condition_block = nil @warning = nil diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index f7f89d4e897..e5d99ebee35 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -12,10 +12,16 @@ module Gitlab included do # Issue, MergeRequest, Epic: quick actions definitions desc do - "Close this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + _('Close this %{quick_action_target}') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end explanation do - "Closes this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Closes this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Closed this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -28,10 +34,16 @@ module Gitlab end desc do - "Reopen this #{quick_action_target.to_ability_name.humanize(capitalize: false)}" + _('Reopen this %{quick_action_target}') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end explanation do - "Reopens this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Reopens this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Reopened this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -45,7 +57,10 @@ module Gitlab desc _('Change title') explanation do |title_param| - _("Changes the title to \"%{title_param}\".") % { title_param: title_param } + _('Changes the title to "%{title_param}".') % { title_param: title_param } + end + execution_message do |title_param| + _('Changed the title to "%{title_param}".') % { title_param: title_param } end params '<New title>' types Issuable @@ -61,7 +76,10 @@ module Gitlab explanation do |labels_param| labels = find_label_references(labels_param) - "Adds #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + if labels.any? + _("Adds %{labels} %{label_text}.") % + { labels: labels.join(' '), label_text: 'label'.pluralize(labels.count) } + end end params '~label1 ~"label 2"' types Issuable @@ -71,21 +89,15 @@ module Gitlab find_labels.any? end command :label do |labels_param| - label_ids = find_label_ids(labels_param) - - if label_ids.any? - @updates[:add_label_ids] ||= [] - @updates[:add_label_ids] += label_ids - - @updates[:add_label_ids].uniq! - end + run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end desc _('Remove all or specific label(s)') explanation do |labels_param = nil| - if labels_param.present? - labels = find_label_references(labels_param) - "Removes #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? + label_references = labels_param.present? ? find_label_references(labels_param) : [] + if label_references.any? + _("Removes %{label_references} %{label_text}.") % + { label_references: label_references.join(' '), label_text: 'label'.pluralize(label_references.count) } else _('Removes all labels.') end @@ -99,7 +111,9 @@ module Gitlab end command :unlabel do |labels_param = nil| if labels_param.present? - label_ids = find_label_ids(labels_param) + labels = find_labels(labels_param) + label_ids = labels.map(&:id) + label_references = labels_to_reference(labels, :name) if label_ids.any? @updates[:remove_label_ids] ||= [] @@ -109,7 +123,10 @@ module Gitlab end else @updates[:label_ids] = [] + label_references = [] end + + @execution_message[:unlabel] = remove_label_message(label_references) end desc _('Replace all label(s)') @@ -125,18 +142,12 @@ module Gitlab current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent) end command :relabel do |labels_param| - label_ids = find_label_ids(labels_param) - - if label_ids.any? - @updates[:label_ids] ||= [] - @updates[:label_ids] += label_ids - - @updates[:label_ids].uniq! - end + run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids) end - desc _('Add a todo') - explanation _('Adds a todo.') + desc _('Add a To Do') + explanation _('Adds a To Do.') + execution_message _('Added a To Do.') types Issuable condition do quick_action_target.persisted? && @@ -146,8 +157,9 @@ module Gitlab @updates[:todo_event] = 'add' end - desc _('Mark to do as done') - explanation _('Marks to do as done.') + desc _('Mark To Do as done') + explanation _('Marks To Do as done.') + execution_message _('Marked To Do as done.') types Issuable condition do quick_action_target.persisted? && @@ -159,7 +171,12 @@ module Gitlab desc _('Subscribe') explanation do - "Subscribes to this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Subscribes to this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Subscribed to this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -172,7 +189,12 @@ module Gitlab desc _('Unsubscribe') explanation do - "Unsubscribes from this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." + _('Unsubscribes from this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + end + execution_message do + _('Unsubscribed from this %{quick_action_target}.') % + { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end types Issuable condition do @@ -187,6 +209,9 @@ module Gitlab explanation do |name| _("Toggles :%{name}: emoji award.") % { name: name } if name end + execution_message do |name| + _("Toggled :%{name}: emoji award.") % { name: name } if name + end params ':emoji:' types Issuable condition do @@ -215,6 +240,41 @@ module Gitlab substitution :tableflip do |comment| "#{comment} #{TABLEFLIP}" end + + private + + def run_label_command(labels:, command:, updates_key:) + return if labels.empty? + + @updates[updates_key] ||= [] + @updates[updates_key] += labels.map(&:id) + @updates[updates_key].uniq! + + label_references = labels_to_reference(labels, :name) + @execution_message[command] = case command + when :relabel + _('Replaced all labels with %{label_references} %{label_text}.') % + { + label_references: label_references.join(' '), + label_text: 'label'.pluralize(label_references.count) + } + when :label + _('Added %{label_references} %{label_text}.') % + { + label_references: label_references.join(' '), + label_text: 'label'.pluralize(labels.count) + } + end + end + + def remove_label_message(label_references) + if label_references.any? + _("Removed %{label_references} %{label_text}.") % + { label_references: label_references.join(' '), label_text: 'label'.pluralize(label_references.count) } + else + _('Removed all labels.') + end + end end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 85e62f950c8..da28fbf5be0 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -12,6 +12,9 @@ module Gitlab explanation do |due_date| _("Sets the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date end + execution_message do |due_date| + _("Set the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date + end params '<in 2 days | this Friday | December 31st>' types Issue condition do @@ -27,6 +30,7 @@ module Gitlab desc _('Remove due date') explanation _('Removes the due date.') + execution_message _('Removed the due date.') types Issue condition do quick_action_target.persisted? && @@ -49,22 +53,26 @@ module Gitlab current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) && quick_action_target.project.boards.count == 1 end - # rubocop: disable CodeReuse/ActiveRecord command :board_move do |target_list_name| - label_ids = find_label_ids(target_list_name) + labels = find_labels(target_list_name) + label_ids = labels.map(&:id) - if label_ids.size == 1 + if label_ids.size > 1 + message = _('Failed to move this issue because only a single label can be provided.') + elsif !Label.on_project_board?(quick_action_target.project_id, label_ids.first) + message = _('Failed to move this issue because label was not found.') + else label_id = label_ids.first - # Ensure this label corresponds to a list on the board - next unless Label.on_project_boards(quick_action_target.project_id).where(id: label_id).exists? - @updates[:remove_label_ids] = - quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) + quick_action_target.labels.on_project_boards(quick_action_target.project_id).where.not(id: label_id).pluck(:id) # rubocop: disable CodeReuse/ActiveRecord @updates[:add_label_ids] = [label_id] + + message = _("Moved issue to %{label} column in the board.") % { label: labels_to_reference(labels).first } end + + @execution_message[:board_move] = message end - # rubocop: enable CodeReuse/ActiveRecord desc _('Mark this issue as a duplicate of another issue') explanation do |duplicate_reference| @@ -81,7 +89,13 @@ module Gitlab if canonical_issue.present? @updates[:canonical_issue_id] = canonical_issue.id + + message = _("Marked this issue as a duplicate of %{duplicate_param}.") % { duplicate_param: duplicate_param } + else + message = _('Failed to mark this issue as a duplicate because referenced issue was not found.') end + + @execution_message[:duplicate] = message end desc _('Move this issue to another project.') @@ -99,12 +113,21 @@ module Gitlab if target_project.present? @updates[:target_project] = target_project + + message = _("Moved this issue to %{path_to_project}.") % { path_to_project: target_project_path } + else + message = _("Failed to move this issue because target project doesn't exist.") end + + @execution_message[:move] = message end - desc _('Make issue confidential.') + desc _('Make issue confidential') explanation do - _('Makes this issue confidential') + _('Makes this issue confidential.') + end + execution_message do + _('Made this issue confidential.') end types Issue condition do @@ -114,12 +137,19 @@ module Gitlab @updates[:confidential] = true end - desc _('Create a merge request.') + desc _('Create a merge request') explanation do |branch_name = nil| if branch_name - _("Creates branch '%{branch_name}' and a merge request to resolve this issue") % { branch_name: branch_name } + _("Creates branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } + else + _('Creates a branch and a merge request to resolve this issue.') + end + end + execution_message do |branch_name = nil| + if branch_name + _("Created branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } else - "Creates a branch and a merge request to resolve this issue" + _('Created a branch and a merge request to resolve this issue.') end end params "<branch name>" diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index e1579cfddc0..533c74ba9b4 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -9,12 +9,9 @@ module Gitlab included do # Issue, MergeRequest: quick actions definitions desc _('Assign') - # rubocop: disable CodeReuse/ActiveRecord explanation do |users| - users = quick_action_target.allows_multiple_assignees? ? users : users.take(1) - "Assigns #{users.map(&:to_reference).to_sentence}." + _('Assigns %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end - # rubocop: enable CodeReuse/ActiveRecord params do quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '@user' end @@ -26,7 +23,10 @@ module Gitlab extract_users(assignee_param) end command :assign do |users| - next if users.empty? + if users.empty? + @execution_message[:assign] = _("Failed to assign a user because no user was found.") + next + end if quick_action_target.allows_multiple_assignees? @updates[:assignee_ids] ||= quick_action_target.assignees.map(&:id) @@ -34,6 +34,8 @@ module Gitlab else @updates[:assignee_ids] = [users.first.id] end + + @execution_message[:assign] = _('Assigned %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end desc do @@ -44,9 +46,14 @@ module Gitlab end end explanation do |users = nil| - assignees = quick_action_target.assignees - assignees &= users if users.present? && quick_action_target.allows_multiple_assignees? - "Removes #{'assignee'.pluralize(assignees.size)} #{assignees.map(&:to_reference).to_sentence}." + assignees = assignees_for_removal(users) + _("Removes %{assignee_text} %{assignee_references}.") % + { assignee_text: 'assignee'.pluralize(assignees.size), assignee_references: assignees.map(&:to_reference).to_sentence } + end + execution_message do |users = nil| + assignees = assignees_for_removal(users) + _("Removed %{assignee_text} %{assignee_references}.") % + { assignee_text: 'assignee'.pluralize(assignees.size), assignee_references: assignees.map(&:to_reference).to_sentence } end params do quick_action_target.allows_multiple_assignees? ? '@user1 @user2' : '' @@ -74,6 +81,9 @@ module Gitlab explanation do |milestone| _("Sets the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone end + execution_message do |milestone| + _("Set the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone + end params '%"milestone"' types Issue, MergeRequest condition do @@ -92,6 +102,9 @@ module Gitlab explanation do _("Removes %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } end + execution_message do + _("Removed %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } + end types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -116,17 +129,22 @@ module Gitlab extract_references(issuable_param, :merge_request).first end command :copy_metadata do |source_issuable| - if source_issuable.present? && source_issuable.project.id == quick_action_target.project.id + if can_copy_metadata?(source_issuable) @updates[:add_label_ids] = source_issuable.labels.map(&:id) @updates[:milestone_id] = source_issuable.milestone.id if source_issuable.milestone + + @execution_message[:copy_metadata] = _("Copied labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end end desc _('Set time estimate') explanation do |time_estimate| - time_estimate = Gitlab::TimeTrackingFormatter.output(time_estimate) - - _("Sets time estimate to %{time_estimate}.") % { time_estimate: time_estimate } if time_estimate + formatted_time_estimate = format_time_estimate(time_estimate) + _("Sets time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate + end + execution_message do |time_estimate| + formatted_time_estimate = format_time_estimate(time_estimate) + _("Set time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate end params '<1w 3d 2h 14m>' types Issue, MergeRequest @@ -144,18 +162,12 @@ module Gitlab desc _('Add or subtract spent time') explanation do |time_spent, time_spent_date| - if time_spent - if time_spent > 0 - verb = _('Adds') - value = time_spent - else - verb = _('Subtracts') - value = -time_spent - end - - _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: Gitlab::TimeTrackingFormatter.output(value) } - end + spend_time_message(time_spent, time_spent_date, false) end + execution_message do |time_spent, time_spent_date| + spend_time_message(time_spent, time_spent_date, true) + end + params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' types Issue, MergeRequest condition do @@ -176,6 +188,7 @@ module Gitlab desc _('Remove time estimate') explanation _('Removes time estimate.') + execution_message _('Removed time estimate.') types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -187,6 +200,7 @@ module Gitlab desc _('Remove spent time') explanation _('Removes spent time.') + execution_message _('Removed spent time.') condition do quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) @@ -197,7 +211,8 @@ module Gitlab end desc _("Lock the discussion") - explanation _("Locks the discussion") + explanation _("Locks the discussion.") + execution_message _("Locked the discussion.") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -209,7 +224,8 @@ module Gitlab end desc _("Unlock the discussion") - explanation _("Unlocks the discussion") + explanation _("Unlocks the discussion.") + execution_message _("Unlocked the discussion.") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -219,6 +235,47 @@ module Gitlab command :unlock do @updates[:discussion_locked] = false end + + private + + def assignee_users_sentence(users) + if quick_action_target.allows_multiple_assignees? + users + else + [users.first] + end.map(&:to_reference).to_sentence + end + + def assignees_for_removal(users) + assignees = quick_action_target.assignees + if users.present? && quick_action_target.allows_multiple_assignees? + assignees & users + else + assignees + end + end + + def can_copy_metadata?(source_issuable) + source_issuable.present? && source_issuable.project_id == quick_action_target.project_id + end + + def format_time_estimate(time_estimate) + Gitlab::TimeTrackingFormatter.output(time_estimate) + end + + def spend_time_message(time_spent, time_spent_date, paste_tense) + return unless time_spent + + if time_spent > 0 + verb = paste_tense ? _('Added') : _('Adds') + value = time_spent + else + verb = paste_tense ? _('Subtracted') : _('Subtracts') + value = -time_spent + end + + _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: format_time_estimate(value) } + end end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index bade59182a1..e9127095a0d 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -8,8 +8,9 @@ module Gitlab included do # MergeRequest only quick actions definitions - desc 'Merge (when the pipeline succeeds)' - explanation 'Merges this merge request when the pipeline succeeds.' + desc _('Merge (when the pipeline succeeds)') + explanation _('Merges this merge request when the pipeline succeeds.') + execution_message _('Scheduled to merge this merge request when the pipeline succeeds.') types MergeRequest condition do last_diff_sha = params && params[:merge_request_diff_head_sha] @@ -22,10 +23,22 @@ module Gitlab desc 'Toggle the Work In Progress status' explanation do - verb = quick_action_target.work_in_progress? ? 'Unmarks' : 'Marks' noun = quick_action_target.to_ability_name.humanize(capitalize: false) - "#{verb} this #{noun} as Work In Progress." + if quick_action_target.work_in_progress? + _("Unmarks this %{noun} as Work In Progress.") + else + _("Marks this %{noun} as Work In Progress.") + end % { noun: noun } end + execution_message do + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if quick_action_target.work_in_progress? + _("Unmarked this %{noun} as Work In Progress.") + else + _("Marked this %{noun} as Work In Progress.") + end % { noun: noun } + end + types MergeRequest condition do quick_action_target.respond_to?(:work_in_progress?) && @@ -36,9 +49,12 @@ module Gitlab @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end - desc 'Set target branch' + desc _('Set target branch') explanation do |branch_name| - "Sets target branch to #{branch_name}." + _('Sets target branch to %{branch_name}.') % { branch_name: branch_name } + end + execution_message do |branch_name| + _('Set target branch to %{branch_name}.') % { branch_name: branch_name } end params '<Local branch name>' types MergeRequest diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e43147a3f37..e6372a42dda 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -46,6 +46,18 @@ module Gitlab "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', and spaces, but it cannot start or end with '/'" end + def environment_scope_regex_chars + "#{environment_name_regex_chars}\\*" + end + + def environment_scope_regex + @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/.freeze + end + + def environment_scope_regex_message + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces" + end + def kubernetes_namespace_regex /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ end @@ -94,6 +106,12 @@ module Gitlab }mx end + # Based on Jira's project key format + # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + def jira_issue_key_regex + @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + end + def jira_transition_id_regex @jira_transition_id_regex ||= /\d+/ end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index e40c366ed02..75503ee1789 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -23,6 +23,37 @@ module Gitlab end end + # Caches and strongly memoizes the method as a Redis Set. + # + # This only works for methods that do not take any arguments. The method + # should return an Array of Strings to be cached. + # + # In addition to overriding the named method, a "name_include?" method is + # defined. This uses the "SISMEMBER" query to efficiently check membership + # without needing to load the entire set into memory. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_as_redis_set(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + cache_method_output_as_redis_set(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + + define_method("#{name}_include?") do |value| + # If the cache isn't populated, we can't rely on it + return redis_set_cache.include?(name, value) if redis_set_cache.exist?(name) + + # Since we have to pull all branch names to populate the cache, use + # the data we already have to answer the query just this once + __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + end + end + # Caches truthy values from the method. All values are strongly memoized, # and cached in RequestStore. # @@ -84,6 +115,11 @@ module Gitlab raise NotImplementedError end + # RepositorySetCache to be used. Should be overridden by the including class + def redis_set_cache + raise NotImplementedError + end + # List of cached methods. Should be overridden by the including class def cached_methods raise NotImplementedError @@ -100,6 +136,18 @@ module Gitlab end end + # Caches and strongly memoizes the supplied block as a Redis Set. The result + # will be provided as a sorted array. + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output_as_redis_set(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + redis_set_cache.fetch(name, &block).sort + end + end + # Caches truthy values from the supplied block. All values are strongly # memoized, and cached in RequestStore. # @@ -154,6 +202,7 @@ module Gitlab clear_memoization(memoizable_name(name)) end + expire_redis_set_method_caches(methods) expire_request_store_method_caches(methods) end @@ -169,6 +218,10 @@ module Gitlab end end + def expire_redis_set_method_caches(methods) + methods.each { |name| redis_set_cache.expire(name) } + end + # All cached repository methods depend on the existence of a Git repository, # so if the repository doesn't exist, we already know not to call it. def fallback_early?(method_name) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb new file mode 100644 index 00000000000..fb634328a95 --- /dev/null +++ b/lib/gitlab/repository_set_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Interface to the Redis-backed cache store for keys that use a Redis set +module Gitlab + class RepositorySetCache + attr_reader :repository, :namespace, :expires_in + + def initialize(repository, extra_namespace: nil, expires_in: 2.weeks) + @repository = repository + @namespace = "#{repository.full_path}:#{repository.project.id}" + @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace + @expires_in = expires_in + end + + def cache_key(type) + [type, namespace, 'set'].join(':') + end + + def expire(key) + with { |redis| redis.del(cache_key(key)) } + end + + def exist?(key) + with { |redis| redis.exists(cache_key(key)) } + end + + def read(key) + with { |redis| redis.smembers(cache_key(key)) } + end + + def write(key, value) + full_key = cache_key(key) + + with do |redis| + redis.multi do + redis.del(full_key) + + # Splitting into groups of 1000 prevents us from creating a too-long + # Redis command + value.in_groups_of(1000, false) { |subset| redis.sadd(full_key, subset) } + + redis.expire(full_key, expires_in) + end + end + + value + end + + def fetch(key, &block) + if exist?(key) + read(key) + else + write(key, yield) + end + end + + def include?(key, value) + with { |redis| redis.sismember(cache_key(key), value) } + end + + private + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 64593153686..033e451dbee 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -6,6 +6,21 @@ module Gitlab module RequestProfiler PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze + def all + Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| + Profile.new(File.basename(path)) + end.select(&:valid?) + end + module_function :all + + def find(name) + file_path = File.join(PROFILES_DIR, name) + return unless File.exist?(file_path) + + Profile.new(name) + end + module_function :find + def profile_token Rails.cache.fetch('profile-token') do Devise.friendly_token diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 7615f6f443b..99958d7a211 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'ruby-prof' +require 'memory_profiler' module Gitlab module RequestProfiler @@ -28,22 +29,73 @@ module Gitlab end def call_with_profiling(env) + case env['HTTP_X_PROFILE_MODE'] + when 'execution', nil + call_with_call_stack_profiling(env) + when 'memory' + call_with_memory_profiling(env) + else + raise ActionController::BadRequest, invalid_profile_mode(env) + end + end + + def invalid_profile_mode(env) + <<~HEREDOC + Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}. + Supported profile mode request header: + - X-Profile-Mode: execution + - X-Profile-Mode: memory + HEREDOC + end + + def call_with_call_stack_profiling(env) ret = nil - result = RubyProf::Profile.profile do + report = RubyProf::Profile.profile do ret = catch(:warden) do @app.call(env) end end - printer = RubyProf::CallStackPrinter.new(result) - file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}.html" + generate_report(env, 'execution', 'html') do |file| + printer = RubyProf::CallStackPrinter.new(report) + printer.print(file) + end + + handle_request_ret(ret) + end + + def call_with_memory_profiling(env) + ret = nil + report = MemoryProfiler.report do + ret = catch(:warden) do + @app.call(env) + end + end + + generate_report(env, 'memory', 'txt') do |file| + report.pretty_print(to_file: file) + end + + handle_request_ret(ret) + end + + def generate_report(env, report_type, extension) + file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\ + "_#{report_type}.#{extension}" file_path = "#{PROFILES_DIR}/#{file_name}" FileUtils.mkdir_p(PROFILES_DIR) - File.open(file_path, 'wb') do |file| - printer.print(file) + + begin + File.open(file_path, 'wb') do |file| + yield(file) + end + rescue + FileUtils.rm(file_path) end + end + def handle_request_ret(ret) if ret.is_a?(Array) ret else diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index 46996ef8c51..76c675658b1 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -3,42 +3,40 @@ module Gitlab module RequestProfiler class Profile - attr_reader :name, :time, :request_path + attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type alias_method :to_param, :name - def self.all - Dir["#{PROFILES_DIR}/*.html"].map do |path| - new(File.basename(path)) - end - end - - def self.find(name) - name_dup = name.dup - name_dup << '.html' unless name.end_with?('.html') - - file_path = "#{PROFILES_DIR}/#{name_dup}" - return unless File.exist?(file_path) - - new(name_dup) - end - def initialize(name) @name = name + @file_path = File.join(PROFILES_DIR, name) set_attributes end - def content - File.read("#{PROFILES_DIR}/#{name}") + def valid? + @request_path.present? + end + + def content_type + case type + when 'html' + 'text/html' + when 'txt' + 'text/plain' + end end private def set_attributes - _, path, timestamp = name.split(/(.*)_(\d+)\.html$/) - @request_path = path.tr('|', '/') - @time = Time.at(timestamp.to_i).utc + matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/) + return unless matches + + @request_path = matches[:path].tr('|', '/') + @time = Time.at(matches[:timestamp].to_i).utc + @profile_mode = matches[:profile_mode] || 'unknown' + @type = matches[:type] end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb new file mode 100644 index 00000000000..8bb8c547ae1 --- /dev/null +++ b/lib/gitlab/rugged_instrumentation.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module RuggedInstrumentation + def self.query_time + SafeRequestStore[:rugged_query_time] ||= 0 + end + + def self.query_time=(duration) + SafeRequestStore[:rugged_query_time] = duration + end + + def self.query_time_ms + (self.query_time * 1000).round(2) + end + + def self.query_count + SafeRequestStore[:rugged_call_count] ||= 0 + end + + def self.increment_query_count + SafeRequestStore[:rugged_call_count] ||= 0 + SafeRequestStore[:rugged_call_count] += 1 + end + + def self.active? + SafeRequestStore.active? + end + + def self.peek_enabled? + SafeRequestStore[:peek_enabled] + end + + def self.add_call_details(details) + return unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] ||= [] + Gitlab::SafeRequestStore[:rugged_call_details] << details + end + + def self.list_call_details + return [] unless peek_enabled? + + Gitlab::SafeRequestStore[:rugged_call_details] || [] + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7c1e6b1baff..ce4c1611687 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,29 @@ module Gitlab without_count ? collection.without_count : collection end + def formatted_count(scope) + case scope + when 'projects' + formatted_limited_count(limited_projects_count) + when 'issues' + formatted_limited_count(limited_issues_count) + when 'merge_requests' + formatted_limited_count(limited_merge_requests_count) + when 'milestones' + formatted_limited_count(limited_milestones_count) + when 'users' + formatted_limited_count(limited_users_count) + end + end + + def formatted_limited_count(count) + if count >= COUNT_LIMIT + "#{COUNT_LIMIT - 1}+" + else + count.to_s + end + end + def limited_projects_count @limited_projects_count ||= limited_count(projects) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 764db14d720..005cb3112b8 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -39,9 +39,14 @@ module Gitlab # development and test. If you need development and test to behave # just the same as production you can use this instead of # track_exception. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) if enabled? - extra[:issue_url] = issue_url if issue_url + extra = build_extra_data(exception, issue_url, extra) context # Make sure we've set everything we know in the context Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -58,5 +63,15 @@ module Gitlab locale: I18n.locale } end + + def self.build_extra_data(exception, issue_url, extra) + exception.try(:sentry_extra_data)&.tap do |data| + extra.merge!(data) if data.is_a?(Hash) + end + + extra.merge({ issue_url: issue_url }.compact) + end + + private_class_method :build_extra_data end end diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 159ce27e702..cbd89b7629f 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -96,12 +96,7 @@ module Gitlab private def raw_explain(query) - explain = - if Gitlab::Database.postgresql? - "EXPLAIN ANALYZE #{query};" - else - "EXPLAIN #{query};" - end + explain = "EXPLAIN ANALYZE #{query};" ActiveRecord::Base.connection.execute(explain) end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index fdc0d518c59..60782306ade 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -15,9 +15,9 @@ module Gitlab yield - Sidekiq.logger.info log_job_done(started_at, base_payload) + Sidekiq.logger.info log_job_done(job, started_at, base_payload) rescue => job_exception - Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + Sidekiq.logger.warn log_job_done(job, started_at, base_payload, job_exception) raise end @@ -28,15 +28,26 @@ module Gitlab "#{payload['class']} JID-#{payload['jid']}" end + def add_instrumentation_keys!(job, output_payload) + output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) + end + def log_job_start(started_at, payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' + # Old gitlab-shell messages don't provide enqueued_at/created_at attributes + enqueued_at = payload['enqueued_at'] || payload['created_at'] + if enqueued_at + payload['scheduling_latency_s'] = elapsed_by_absolute_time(Time.iso8601(enqueued_at)) + end + payload end - def log_job_done(started_at, payload, job_exception = nil) + def log_job_done(job, started_at, payload, job_exception = nil) payload = payload.dup + add_instrumentation_keys!(job, payload) payload['duration'] = elapsed(started_at) payload['completed_at'] = Time.now.utc @@ -78,6 +89,10 @@ module Gitlab end end + def elapsed_by_absolute_time(start) + (Time.now.utc - start).to_f.round(3) + end + def elapsed(start) (current_time - start).round(3) end diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb new file mode 100644 index 00000000000..979a3fce7e6 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class InstrumentationLogger + def call(worker, job, queue) + yield + + # The Sidekiq logger is called outside the middleware block, so + # we need to modify the job hash to pass along this information + # since RequestStore is only active in the Sidekiq middleware. + # + # Modifying the job hash in a middleware is permitted by Sidekiq + # because Sidekiq keeps a pristine copy of the original hash + # before sending it to the middleware: + # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 + ::Gitlab::InstrumentationHelper.add_instrumentation_data(job) + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 671d795ec33..49c4fdc3033 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -14,9 +14,12 @@ module Gitlab # shut Sidekiq down MUTEX = Mutex.new + attr_reader :worker + def call(worker, job, queue) yield + @worker = worker current_rss = get_rss return unless MAX_RSS > 0 && current_rss > MAX_RSS @@ -25,9 +28,11 @@ module Gitlab # Return if another thread is already waiting to shut Sidekiq down next unless MUTEX.try_lock - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}" - Sidekiq.logger.warn "Sidekiq worker PID-#{pid} will stop fetching new jobs in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later" + warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ + " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") + + warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ + " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") # Wait `GRACE_TIME` to give the memory intensive job time to finish. # Then, tell Sidekiq to stop fetching new jobs. @@ -59,24 +64,28 @@ module Gitlab def wait_and_signal_pgroup(time, signal, explanation) return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, 0) end def wait_and_signal(time, signal, explanation) - Sidekiq.logger.warn "waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) sleep(time) - Sidekiq.logger.warn "sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})" + warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) Process.kill(signal, pid) end def pid Process.pid end + + def warn(message, signal: nil) + Sidekiq.logger.warn(class: worker.class.name, pid: pid, signal: signal, message: message) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/metrics.rb b/lib/gitlab/sidekiq_middleware/metrics.rb new file mode 100644 index 00000000000..3dc9521ee8b --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/metrics.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class Metrics + # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq + # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + + def initialize + @metrics = init_metrics + end + + def call(_worker, job, queue) + labels = create_labels(queue) + @metrics[:sidekiq_running_jobs].increment(labels, 1) + + if job['retry_count'].present? + @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) + end + + realtime = Benchmark.realtime do + yield + end + + @metrics[:sidekiq_jobs_completion_seconds].observe(labels, realtime) + rescue Exception # rubocop: disable Lint/RescueException + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) + raise + ensure + @metrics[:sidekiq_running_jobs].increment(labels, -1) + end + + private + + def init_metrics + { + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete sidekiq job', buckets: SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :livesum) + } + end + + def create_labels(queue) + { + queue: queue + } + end + end + end +end diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb new file mode 100644 index 00000000000..1b87d3bb626 --- /dev/null +++ b/lib/gitlab/slug/environment.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# An environment name is not necessarily suitable for use in URLs, DNS +# or other third-party contexts, so provide a slugified version. A slug has +# the following properties: +# * contains only lowercase letters (a-z), numbers (0-9), and '-' +# * begins with a letter +# * has a maximum length of 24 bytes (OpenShift limitation) +# * cannot end with `-` +module Gitlab + module Slug + class Environment + attr_reader :name + + def initialize(name) + @name = name + end + + def generate + # Lowercase letters and numbers only + slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-') + + # Must start with a letter + slugified = 'env-' + slugified unless slugified.match?(/^[a-z]/) + + # Repeated dashes are invalid (OpenShift limitation) + slugified.squeeze!('-') + + slugified = + if slugified.size > 24 || slugified != name + # Maximum length: 24 characters (OpenShift limitation) + shorten_and_add_suffix(slugified) + else + # Cannot end with a dash (Kubernetes label limitation) + slugified.chomp('-') + end + + slugified + end + + private + + def shorten_and_add_suffix(slug) + slug = slug[0..16] + slug << '-' unless slug.ends_with?('-') + slug << suffix + end + + # Slugifying a name may remove the uniqueness guarantee afforded by it being + # based on name (which must be unique). To compensate, we add a predictable + # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, + # but the chance of collisions is vanishingly small + def suffix + Digest::SHA2.hexdigest(name.to_s).to_i(16).to_s(36).last(6) + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e360b552f89..ac3b219e0c7 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -22,6 +22,17 @@ module Gitlab end end + def formatted_count(scope) + case scope + when 'snippet_titles' + snippet_titles_count.to_s + when 'snippet_blobs' + snippet_blobs_count.to_s + else + super + end + end + def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end diff --git a/lib/gitlab/snowplow_tracker.rb b/lib/gitlab/snowplow_tracker.rb new file mode 100644 index 00000000000..9f12513e09e --- /dev/null +++ b/lib/gitlab/snowplow_tracker.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module SnowplowTracker + NAMESPACE = 'cf' + + class << self + def track_event(category, action, label: nil, property: nil, value: nil, context: nil) + tracker&.track_struct_event(category, action, label, property, value, context, Time.now.to_i) + end + + private + + def tracker + return unless enabled? + + @tracker ||= ::SnowplowTracker::Tracker.new(emitter, subject, NAMESPACE, Gitlab::CurrentSettings.snowplow_site_id) + end + + def subject + ::SnowplowTracker::Subject.new + end + + def emitter + ::SnowplowTracker::Emitter.new(Gitlab::CurrentSettings.snowplow_collector_hostname) + end + + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + end + end +end diff --git a/lib/gitlab/submodule_links.rb b/lib/gitlab/submodule_links.rb index a6c0369d864..18fd604a3b0 100644 --- a/lib/gitlab/submodule_links.rb +++ b/lib/gitlab/submodule_links.rb @@ -9,7 +9,7 @@ module Gitlab end def for(submodule, sha) - submodule_url = submodule_url_for(sha)[submodule.path] + submodule_url = submodule_url_for(sha, submodule.path) SubmoduleHelper.submodule_links_for_url(submodule.id, submodule_url, repository) end @@ -17,10 +17,15 @@ module Gitlab attr_reader :repository - def submodule_url_for(sha) - strong_memoize(:"submodule_links_for_#{sha}") do + def submodule_urls_for(sha) + strong_memoize(:"submodule_urls_for_#{sha}") do repository.submodule_urls_for(sha) end end + + def submodule_url_for(sha, path) + urls = submodule_urls_for(sha) + urls && urls[path] + end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index f6b2e2acf16..9c35d200dcb 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -45,18 +45,21 @@ module Gitlab ascii_only: ascii_only ) + normalized_hostname = uri.normalized_host hostname = uri.hostname port = get_port(uri) address_info = get_address_info(hostname, port) return [uri, nil] unless address_info - protected_uri_with_hostname = enforce_uri_hostname(address_info, uri, hostname, dns_rebind_protection) + ip_address = ip_address(address_info) + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) validate_local_request( + normalized_hostname: normalized_hostname, address_info: address_info, allow_localhost: allow_localhost, allow_local_network: allow_local_network @@ -83,10 +86,7 @@ module Gitlab # # The original hostname is used to validate the SSL, given in that scenario # we'll be making the request to the IP address, instead of using the hostname. - def enforce_uri_hostname(addrs_info, uri, hostname, dns_rebind_protection) - address = addrs_info.first - ip_address = address&.ip_address - + def enforce_uri_hostname(ip_address, uri, hostname, dns_rebind_protection) return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != hostname uri = uri.dup @@ -94,6 +94,10 @@ module Gitlab [uri, hostname] end + def ip_address(address_info) + address_info.first&.ip_address + end + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) validate_html_tags(uri) if enforce_sanitization @@ -111,11 +115,30 @@ module Gitlab addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end rescue SocketError + # In the test suite we use a lot of mocked urls that are either invalid or + # don't exist. In order to avoid modifying a ton of tests and factories + # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS + # is not true + return if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' + + # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1) + # we block the url + raise BlockedUrlError, "Host cannot be resolved or invalid" end - def validate_local_request(address_info:, allow_localhost:, allow_local_network:) + def validate_local_request( + normalized_hostname:, + address_info:, + allow_localhost:, + allow_local_network:) return if allow_local_network && allow_localhost + ip_whitelist, domain_whitelist = + Gitlab::CurrentSettings.outbound_local_requests_whitelist_arrays + + return if local_domain_whitelisted?(domain_whitelist, normalized_hostname) || + local_ip_whitelisted?(ip_whitelist, ip_address(address_info)) + unless allow_localhost validate_localhost(address_info) validate_loopback(address_info) @@ -231,6 +254,16 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end + def local_ip_whitelisted?(ip_whitelist, ip_string) + ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) + + ip_whitelist.any? { |ip| ip.include?(ip_obj) } + end + + def local_domain_whitelisted?(domain_whitelist, domain_string) + domain_whitelist.include?(domain_string) + end + def config Gitlab.config end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 055e01a9399..353298e67b3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -6,7 +6,9 @@ module Gitlab class << self def data(force_refresh: false) - Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do + uncached_data + end end def uncached_data @@ -98,9 +100,7 @@ module Gitlab .merge(services_usage) .merge(approximate_counts) }.tap do |data| - if Feature.enabled?(:group_overview_security_dashboard) - data[:counts][:user_preferences] = user_preferences_usage - end + data[:counts][:user_preferences] = user_preferences_usage end end # rubocop: enable CodeReuse/ActiveRecord @@ -128,10 +128,22 @@ module Gitlab } end + # @return [Hash<Symbol, Integer>] def usage_counters - { - web_ide_commits: Gitlab::UsageDataCounters::WebIdeCommitsCounter.total_count - } + usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) } + end + + # @return [Array<#totals>] An array of objects that respond to `#totals` + def usage_data_counters + [ + Gitlab::UsageDataCounters::WikiPageCounter, + Gitlab::UsageDataCounters::WebIdeCounter, + Gitlab::UsageDataCounters::NoteCounter, + Gitlab::UsageDataCounters::SnippetCounter, + Gitlab::UsageDataCounters::SearchCounter, + Gitlab::UsageDataCounters::CycleAnalyticsCounter, + Gitlab::UsageDataCounters::SourceCodeCounter + ] end def components_usage_data @@ -176,8 +188,8 @@ module Gitlab {} # augmented in EE end - def count(relation, fallback: -1) - relation.count + def count(relation, count_by: nil, fallback: -1) + count_by ? relation.count(count_by) : relation.count rescue ActiveRecord::StatementInvalid fallback end diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb new file mode 100644 index 00000000000..2b52571c3cc --- /dev/null +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class BaseCounter + extend RedisCounter + + UnknownEvent = Class.new(StandardError) + + class << self + def redis_key(event) + Gitlab::Sentry.track_exception(UnknownEvent, extra: { event: event }) unless known_events.include?(event.to_s) + + "USAGE_#{prefix}_#{event}".upcase + end + + def count(event) + increment(redis_key event) + end + + def read(event) + total_count(redis_key event) + end + + def totals + known_events.map { |e| ["#{prefix}_#{e}".to_sym, read(e)] }.to_h + end + + private + + def known_events + self::KNOWN_EVENTS + end + + def prefix + self::PREFIX + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb b/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb new file mode 100644 index 00000000000..1ff4296ef65 --- /dev/null +++ b/lib/gitlab/usage_data_counters/cycle_analytics_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class CycleAnalyticsCounter < BaseCounter + KNOWN_EVENTS = %w[views].freeze + PREFIX = 'cycle_analytics' + end +end diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb new file mode 100644 index 00000000000..672450ec82b --- /dev/null +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class NoteCounter < BaseCounter + KNOWN_EVENTS = %w[create].freeze + PREFIX = 'note' + COUNTABLE_TYPES = %w[Snippet Commit MergeRequest].freeze + + class << self + def redis_key(event, noteable_type) + "#{super(event)}_#{noteable_type}".upcase + end + + def count(event, noteable_type) + return unless countable?(noteable_type) + + increment(redis_key(event, noteable_type)) + end + + def read(event, noteable_type) + return 0 unless countable?(noteable_type) + + total_count(redis_key(event, noteable_type)) + end + + def totals + COUNTABLE_TYPES.map do |countable_type| + [:"#{countable_type.underscore}_comment", read(:create, countable_type)] + end.to_h + end + + private + + def countable?(noteable_type) + COUNTABLE_TYPES.include?(noteable_type.to_s) + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index 123b8e1bef1..75d5a75e3a4 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -3,17 +3,15 @@ module Gitlab module UsageDataCounters module RedisCounter - def increment + def increment(redis_counter_key) + return unless Gitlab::CurrentSettings.usage_ping_enabled + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end - def total_count + def total_count(redis_counter_key) Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i } end - - def redis_counter_key - raise NotImplementedError - end end end end diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb new file mode 100644 index 00000000000..5f0735347e1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class SearchCounter + extend RedisCounter + + NAVBAR_SEARCHES_COUNT_KEY = 'NAVBAR_SEARCHES_COUNT' + + class << self + def increment_navbar_searches_count + increment(NAVBAR_SEARCHES_COUNT_KEY) + end + + def total_navbar_searches_count + total_count(NAVBAR_SEARCHES_COUNT_KEY) + end + + def totals + { + navbar_searches: total_navbar_searches_count + } + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/snippet_counter.rb b/lib/gitlab/usage_data_counters/snippet_counter.rb new file mode 100644 index 00000000000..e4d234ce4d9 --- /dev/null +++ b/lib/gitlab/usage_data_counters/snippet_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class SnippetCounter < BaseCounter + KNOWN_EVENTS = %w[create update].freeze + PREFIX = 'snippet' + end +end diff --git a/lib/gitlab/usage_data_counters/source_code_counter.rb b/lib/gitlab/usage_data_counters/source_code_counter.rb new file mode 100644 index 00000000000..8a1771a7bd1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/source_code_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class SourceCodeCounter < BaseCounter + KNOWN_EVENTS = %w[pushes].freeze + PREFIX = 'source_code' + end +end diff --git a/lib/gitlab/usage_data_counters/web_ide_commits_counter.rb b/lib/gitlab/usage_data_counters/web_ide_commits_counter.rb deleted file mode 100644 index 62236fa07a3..00000000000 --- a/lib/gitlab/usage_data_counters/web_ide_commits_counter.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - class WebIdeCommitsCounter - extend RedisCounter - - def self.redis_counter_key - 'WEB_IDE_COMMITS_COUNT' - end - end - end -end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb new file mode 100644 index 00000000000..0718c1dd761 --- /dev/null +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class WebIdeCounter + extend RedisCounter + + COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' + MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT' + VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT' + + class << self + def increment_commits_count + increment(COMMITS_COUNT_KEY) + end + + def total_commits_count + total_count(COMMITS_COUNT_KEY) + end + + def increment_merge_requests_count + increment(MERGE_REQUEST_COUNT_KEY) + end + + def total_merge_requests_count + total_count(MERGE_REQUEST_COUNT_KEY) + end + + def increment_views_count + increment(VIEWS_COUNT_KEY) + end + + def total_views_count + total_count(VIEWS_COUNT_KEY) + end + + def totals + { + web_ide_commits: total_commits_count, + web_ide_views: total_views_count, + web_ide_merge_requests: total_merge_requests_count + } + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb new file mode 100644 index 00000000000..9cfe0be5bab --- /dev/null +++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class WikiPageCounter < BaseCounter + KNOWN_EVENTS = %w[create update delete].freeze + PREFIX = 'wiki_pages' + end +end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 16ec8a8bb28..c66ce0434a4 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -22,7 +22,7 @@ module Gitlab end def force_utf8(str) - str.force_encoding(Encoding::UTF_8) + str.dup.force_encoding(Encoding::UTF_8) end def ensure_utf8_size(str, bytes:) @@ -131,5 +131,12 @@ module Gitlab data end end + + def string_to_ip_object(str) + return unless str + + IPAddr.new(str) + rescue IPAddr::InvalidAddressError + end end end diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb new file mode 100644 index 00000000000..620d71a7814 --- /dev/null +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require_dependency 'gitlab/utils' + +module Gitlab + module Utils + module SanitizeNodeLink + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + ATTRS_TO_SANITIZE = %w(href src data-src).freeze + + def remove_unsafe_links(env, remove_invalid_links: true) + node = env[:node] + + sanitize_node(node: node, remove_invalid_links: remove_invalid_links) + + # HTML entities such as <video></video> have scannable attrs in + # children elements, which also need to be sanitized. + # + node.children.each do |child_node| + sanitize_node(node: child_node, remove_invalid_links: remove_invalid_links) + end + end + + # Remove all invalid scheme characters before checking against the + # list of unsafe protocols. + # + # See https://tools.ietf.org/html/rfc3986#section-3.1 + # + def safe_protocol?(scheme) + return false unless scheme + + scheme = scheme + .strip + .downcase + .gsub(/[^A-Za-z\+\.\-]+/, '') + + UNSAFE_PROTOCOLS.none?(scheme) + end + + private + + def sanitize_node(node:, remove_invalid_links: true) + ATTRS_TO_SANITIZE.each do |attr| + next unless node.has_attribute?(attr) + + begin + node[attr] = node[attr].strip + uri = Addressable::URI.parse(node[attr]) + + next unless uri.scheme + next if safe_protocol?(uri.scheme) + + node.remove_attribute(attr) + rescue Addressable::URI::InvalidURIError + node.remove_attribute(attr) if remove_invalid_links + end + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 46a7b5b982a..3b77fe838ae 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -221,7 +221,7 @@ module Gitlab end def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") diff --git a/lib/gitlab/zoom_link_extractor.rb b/lib/gitlab/zoom_link_extractor.rb index d9994898a08..7ac14eb2d4f 100644 --- a/lib/gitlab/zoom_link_extractor.rb +++ b/lib/gitlab/zoom_link_extractor.rb @@ -17,5 +17,9 @@ module Gitlab def links @text.scan(ZOOM_REGEXP) end + + def match? + ZOOM_REGEXP.match?(@text) + end end end diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb deleted file mode 100644 index 99be51bc8c6..00000000000 --- a/lib/gt_one_coercion.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -class GtOneCoercion < Virtus::Attribute - def coerce(value) - [1, value.to_i].max - end -end diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb deleted file mode 100644 index f36610abf8f..00000000000 --- a/lib/mysql_zero_date.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# Disable NO_ZERO_DATE mode for mysql in rails 5. -# We use zero date as a default value -# (config/initializers/active_record_mysql_timestamp.rb), in -# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) -# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode -# is not sufficient. - -require 'active_record/connection_adapters/abstract_mysql_adapter' - -module MysqlZeroDate - def configure_connection - super - - @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables - end -end - -ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb deleted file mode 100644 index 581cc6a37b4..00000000000 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -module Peek - module Rblineprof - module CustomControllerHelpers - extend ActiveSupport::Concern - - # This will become useless once https://github.com/peek/peek-rblineprof/pull/5 - # is merged - def pygmentize(file_name, code, lexer = nil) - if lexer.present? - Gitlab::Highlight.highlight(file_name, code) - else - "<pre>#{Rack::Utils.escape_html(code)}</pre>" - end - end - - # rubocop:disable all - def inject_rblineprof - ret = nil - profile = lineprof(rblineprof_profiler_regex) do - ret = yield - end - - if response.content_type =~ %r|text/html| - sort = params[:lineprofiler_sort] - mode = params[:lineprofiler_mode] || 'cpu' - min = (params[:lineprofiler_min] || 5).to_i * 1000 - summary = params[:lineprofiler_summary] - - # Sort each file by the longest calculated time - per_file = profile.map do |file, lines| - total, child, excl, total_cpu, child_cpu, excl_cpu = lines[0] - - wall = summary == 'exclusive' ? excl : total - cpu = summary == 'exclusive' ? excl_cpu : total_cpu - idle = summary == 'exclusive' ? (excl - excl_cpu) : (total - total_cpu) - - [ - file, lines, - wall, cpu, idle, - sort == 'idle' ? idle : sort == 'cpu' ? cpu : wall - ] - end.sort_by{ |a,b,c,d,e,f| -f } - - output = ["<div class='modal-dialog modal-xl'><div class='modal-content'>"] - output << "<div class='modal-header'>" - output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" - output << "<button class='close' type='button' data-dismiss='modal' aria-label='close'><span aria-hidden='true'>×</span></button>" - output << "</div>" - output << "<div class='modal-body'>" - - per_file.each do |file_name, lines, file_wall, file_cpu, file_idle, file_sort| - output << "<div class='peek-rblineprof-file'><div class='heading'>" - - show_src = file_sort > min - tmpl = show_src ? "<a href='#' class='js-lineprof-file'>%s</a>" : "%s" - - if mode == 'cpu' - output << sprintf("<span class='duration'>% 8.1fms + % 8.1fms</span> #{tmpl}", file_cpu / 1000.0, file_idle / 1000.0, file_name.sub(Rails.root.to_s + '/', '')) - else - output << sprintf("<span class='duration'>% 8.1fms</span> #{tmpl}", file_wall/1000.0, file_name.sub(Rails.root.to_s + '/', '')) - end - - output << "</div>" # .heading - - next unless show_src - - output << "<div class='data'>" - code = [] - times = [] - File.readlines(file_name).each_with_index do |line, i| - code << line - wall, cpu, calls = lines[i + 1] - - if calls && calls > 0 - if mode == 'cpu' - idle = wall - cpu - times << sprintf("% 8.1fms + % 8.1fms (% 5d)", cpu / 1000.0, idle / 1000.0, calls) - else - times << sprintf("% 8.1fms (% 5d)", wall / 1000.0, calls) - end - else - times << ' ' - end - end - output << "<pre class='duration'>#{times.join("\n")}</pre>" - # The following line was changed from - # https://github.com/peek/peek-rblineprof/blob/8d3b7a283a27de2f40abda45974516693d882258/lib/peek/rblineprof/controller_helpers.rb#L125 - # This will become useless once https://github.com/peek/peek-rblineprof/pull/16 - # is merged and is implemented. - output << "<pre class='code highlight white'>#{pygmentize(file_name, code.join, 'ruby')}</pre>" - output << "</div></div>" # .data then .peek-rblineprof-file - end - - output << "</div></div></div>" - - response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output.join}</div>".html_safe - end - - ret - end - - private - - def human_description(lineprofiler_param) - case lineprofiler_param - when 'app' - 'app/ & lib/' - when 'views' - 'app/view/' - when 'gems' - 'vendor/gems' - when 'all' - 'everything in Rails.root' - when 'stdlib' - 'everything in the Ruby standard library' - else - 'app/, config/, lib/, vendor/ & plugin/' - end - end - end - end -end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb new file mode 100644 index 00000000000..2d78818630d --- /dev/null +++ b/lib/peek/views/active_record.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Peek + module Views + class ActiveRecord < DetailedView + private + + def setup_subscribers + super + + subscribe('sql.active_record') do |_, start, finish, _, data| + if Gitlab::SafeRequestStore.store[:peek_enabled] + unless data[:cached] + detail_store << { + duration: finish - start, + sql: data[:sql].strip, + backtrace: Gitlab::Profiler.clean_backtrace(caller) + } + end + end + end + end + end + end +end diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb new file mode 100644 index 00000000000..f4ca1cb5075 --- /dev/null +++ b/lib/peek/views/detailed_view.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Peek + module Views + class DetailedView < View + def results + { + duration: formatted_duration, + calls: calls, + details: details + } + end + + def detail_store + ::Gitlab::SafeRequestStore["#{key}_call_details"] ||= [] + end + + private + + def duration + detail_store.map { |entry| entry[:duration] }.sum # rubocop:disable CodeReuse/ActiveRecord + end + + def calls + detail_store.count + end + + def call_details + detail_store + end + + def format_call_details(call) + call.merge(duration: (call[:duration] * 1000).round(3)) + end + + def details + call_details + .sort { |a, b| b[:duration] <=> a[:duration] } + .map(&method(:format_call_details)) + end + + def formatted_duration + ms = duration * 1000 + + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + end + end +end diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index 30f95a10024..6ad6ddfd89d 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -2,7 +2,9 @@ module Peek module Views - class Gitaly < View + class Gitaly < DetailedView + private + def duration ::Gitlab::GitalyClient.query_time end @@ -11,36 +13,14 @@ module Peek ::Gitlab::GitalyClient.get_request_count end - def results - { - duration: formatted_duration, - calls: calls, - details: details - } - end - - private - - def details + def call_details ::Gitlab::GitalyClient.list_call_details - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) end def format_call_details(call) pretty_request = call[:request]&.reject { |k, v| v.blank? }.to_h.pretty_inspect - call.merge(duration: (call[:duration] * 1000).round(3), - request: pretty_request || {}) - end - - def formatted_duration - ms = duration * 1000 - if ms >= 1000 - "%.2fms" % ms - else - "%.0fms" % ms - end + super.merge(request: pretty_request || {}) end def setup_subscribers diff --git a/lib/peek/views/redis.rb b/lib/peek/views/redis_detailed.rb index 73de8672fa4..f36f581d5e9 100644 --- a/lib/peek/views/redis.rb +++ b/lib/peek/views/redis_detailed.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'redis' -require 'peek-redis' module Gitlab module Peek @@ -17,16 +16,21 @@ module Gitlab private def add_call_details(duration, args) + return unless peek_enabled? # redis-rb passes an array (e.g. [:get, key]) return unless args.length == 1 detail_store << { cmd: args.first, duration: duration, - backtrace: Gitlab::Profiler.clean_backtrace(caller) + backtrace: ::Gitlab::Profiler.clean_backtrace(caller) } end + def peek_enabled? + Gitlab::SafeRequestStore.store[:peek_enabled] + end + def detail_store ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] end @@ -36,26 +40,17 @@ end module Peek module Views - module RedisDetailed + class RedisDetailed < DetailedView REDACTED_MARKER = "<redacted>" - def results - super.merge(details: details) - end - - def details - detail_store - .sort { |a, b| b[:duration] <=> a[:duration] } - .map(&method(:format_call_details)) + def key + 'redis' end - def detail_store - ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] - end + private def format_call_details(call) - call.merge(cmd: format_command(call[:cmd]), - duration: (call[:duration] * 1000).round(3)) + super.merge(cmd: format_command(call[:cmd])) end def format_command(cmd) @@ -76,11 +71,3 @@ end class Redis::Client prepend Gitlab::Peek::RedisInstrumented end - -module Peek - module Views - class Redis < View - prepend Peek::Views::RedisDetailed - end - end -end diff --git a/lib/peek/views/rugged.rb b/lib/peek/views/rugged.rb new file mode 100644 index 00000000000..18b3f422852 --- /dev/null +++ b/lib/peek/views/rugged.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Peek + module Views + class Rugged < DetailedView + def results + return {} unless calls > 0 + + super + end + + private + + def duration + ::Gitlab::RuggedInstrumentation.query_time + end + + def calls + ::Gitlab::RuggedInstrumentation.query_count + end + + def call_details + ::Gitlab::RuggedInstrumentation.list_call_details + end + + def format_call_details(call) + super.merge(args: format_args(call[:args])) + end + + def format_args(args) + args.map do |arg| + # ActiveSupport::JSON recursively calls as_json on all + # instance variables, and if that instance variable points to + # something that refers back to the same instance, we can wind + # up in an infinite loop. Currently this only seems to happen with + # Gitlab::Git::Repository and ::Repository. + if arg.instance_variables.present? + arg.to_s + else + arg + end + end + end + end + end +end diff --git a/lib/prometheus/cleanup_multiproc_dir_service.rb b/lib/prometheus/cleanup_multiproc_dir_service.rb new file mode 100644 index 00000000000..6418b4de166 --- /dev/null +++ b/lib/prometheus/cleanup_multiproc_dir_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Prometheus + class CleanupMultiprocDirService + include Gitlab::Utils::StrongMemoize + + def execute + FileUtils.rm_rf(old_metrics) if old_metrics + end + + private + + def old_metrics + strong_memoize(:old_metrics) do + Dir[File.join(multiprocess_files_dir, '*.db')] if multiprocess_files_dir + end + end + + def multiprocess_files_dir + ::Prometheus::Client.configuration.multiprocess_files_dir + end + end +end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb new file mode 100644 index 00000000000..e0f7e7e0a9e --- /dev/null +++ b/lib/prometheus/pid_provider.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Prometheus + module PidProvider + extend self + + def worker_id + if Sidekiq.server? + 'sidekiq' + elsif defined?(Unicorn::Worker) + unicorn_worker_id + elsif defined?(::Puma) + puma_worker_id + else + unknown_process_id + end + end + + private + + def unicorn_worker_id + if matches = process_name.match(/unicorn.*worker\[([0-9]+)\]/) + "unicorn_#{matches[1]}" + elsif process_name =~ /unicorn/ + "unicorn_master" + else + unknown_process_id + end + end + + def puma_worker_id + if matches = process_name.match(/puma.*cluster worker ([0-9]+):/) + "puma_#{matches[1]}" + elsif process_name =~ /puma/ + "puma_master" + else + unknown_process_id + end + end + + def unknown_process_id + "process_#{Process.pid}" + end + + def process_name + $0 + end + end +end diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index 24d8eac200c..60d79b52680 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -14,6 +14,7 @@ module Quality finders frontend graphql + haml_lint helpers initializers javascripts diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb index 4022e8ff946..07cca1c8d1e 100644 --- a/lib/sentry/client.rb +++ b/lib/sentry/client.rb @@ -67,7 +67,7 @@ module Sentry def handle_request_exceptions yield - rescue HTTParty::Error => e + rescue Gitlab::HTTP::Error => e Gitlab::Sentry.track_acceptable_exception(e) raise_error 'Error when connecting to Sentry' rescue Net::OpenTimeout diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb index 93cb192087a..1ed5d5dc3f5 100644 --- a/lib/serializers/json.rb +++ b/lib/serializers/json.rb @@ -1,32 +1,16 @@ # frozen_string_literal: true module Serializers - # This serializer exports data as JSON, - # it is designed to be used with interwork compatibility between MySQL and PostgreSQL - # implementations, as used version of MySQL does not support native json type - # - # Secondly, the loader makes the resulting hash to have deep indifferent access + # Make the resulting hash have deep indifferent access class JSON class << self def dump(obj) - # MySQL stores data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - obj = ActiveSupport::JSON.encode(obj) - end - obj end def load(data) return if data.nil? - # On MySQL we store data as text - # look at ./config/initializers/ar_mysql_jsonb_support.rb - if Gitlab::Database.mysql? - data = ActiveSupport::JSON.decode(data) - end - Gitlab::Utils.deep_indifferent_access(data) end end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index ab46c47d8f5..1f8c915140c 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -28,7 +28,7 @@ sudo -u git -H git pull origin master echo 'Deploy: Bundle and migrate' # change it to your needs -sudo -u git -H bundle --without aws development test mysql --deployment +sudo -u git -H bundle --without aws development test --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 32df74f104a..bccb94ff0bf 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -67,6 +67,13 @@ if ! cd "$app_root" ; then echo "Failed to cd into $app_root, exiting!"; exit 1 fi +# Select the web server to use +if [ -z "$EXPERIMENTAL_PUMA" ]; then + use_web_server="unicorn" +else + use_web_server="puma" +fi + ### Init Script functions @@ -256,7 +263,7 @@ start_gitlab() { check_stale_pids if [ "$web_status" != "0" ]; then - echo "Starting GitLab web server" + echo "Starting GitLab web server ($use_web_server)" fi if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" @@ -281,7 +288,7 @@ start_gitlab() { # Remove old socket if it exists rm -f "$rails_socket" 2>/dev/null # Start the web server - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web start + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web start fi # If sidekiq is already running, don't start it again. @@ -343,7 +350,7 @@ stop_gitlab() { if [ "$web_status" = "0" ]; then echo "Shutting down GitLab web server" - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web stop + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" @@ -447,7 +454,7 @@ reload_gitlab(){ exit 1 fi printf "Reloading GitLab web server configuration... " - RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web reload + RAILS_ENV=$RAILS_ENV USE_WEB_SERVER=$use_web_server bin/web reload echo "Done." echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 467711fb74e..08c8df9b044 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.parse('2.21.0') + @required_version ||= Gitlab::VersionInfo.parse('2.22.0') end def self.current_version diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake new file mode 100644 index 00000000000..1cac7520227 --- /dev/null +++ b/lib/tasks/frontend.rake @@ -0,0 +1,21 @@ +unless Rails.env.production? + namespace :frontend do + desc 'GitLab | Frontend | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb') + ENV['NO_KNAPSACK'] = 'true' + t.pattern = args[:pattern] + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Frontend | Run JavaScript tests' + task tests: ['yarn:check'] do + sh "yarn test" do |ok, res| + abort('rake frontend:tests failed') unless ok + end + end + end + + desc 'GitLab | Frontend | Shortcut for frontend:fixtures and frontend:tests' + task frontend: ['frontend:fixtures', 'frontend:tests'] +end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 88172e26c67..4d854cd178d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -127,6 +127,58 @@ namespace :gitlab do end end + namespace :sessions do + desc "GitLab | Cleanup | Sessions | Clean ActiveSession lookup keys" + task active_sessions_lookup_keys: :gitlab_environment do + session_key_pattern = "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:*" + last_save_check = Time.at(0) + wait_time = 10.seconds + cursor = 0 + total_users_scanned = 0 + + Gitlab::Redis::SharedState.with do |redis| + begin + cursor, keys = redis.scan(cursor, match: session_key_pattern) + total_users_scanned += keys.count + + if last_save_check < Time.now - 1.second + while redis.info('persistence')['rdb_bgsave_in_progress'] == '1' + puts "BGSAVE in progress, waiting #{wait_time} seconds" + sleep(wait_time) + end + last_save_check = Time.now + end + + keys.each do |key| + user_id = key.split(':').last + + lookup_key_count = redis.scard(key) + + session_ids = ActiveSession.session_ids_for_user(user_id) + entries = ActiveSession.raw_active_session_entries(session_ids, user_id) + session_ids_and_entries = session_ids.zip(entries) + + inactive_session_ids = session_ids_and_entries.map do |session_id, session| + session_id if session.nil? + end.compact + + redis.pipelined do |conn| + inactive_session_ids.each do |session_id| + conn.srem(key, session_id) + end + end + + if inactive_session_ids + puts "deleted #{inactive_session_ids.count} out of #{lookup_key_count} lookup keys for User ##{user_id}" + end + end + end while cursor.to_i != 0 + + puts "--- All done! Total number of scanned users: #{total_users_scanned}" + end + end + end + def remove? ENV['REMOVE'] == 'true' end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 4e7a8adbef6..1961f64659c 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -26,26 +26,19 @@ namespace :gitlab do task drop_tables: :environment do connection = ActiveRecord::Base.connection - # If MySQL, turn off foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=0') if Gitlab::Database.mysql? - - # connection.tables is deprecated in MySQLAdapter, but in PostgreSQLAdapter - # data_sources returns both views and tables, so use #tables instead - tables = Gitlab::Database.mysql? ? connection.data_sources : connection.tables + # In PostgreSQLAdapter, data_sources returns both views and tables, so use + # #tables instead + tables = connection.tables # Removes the entry from the array tables.delete 'schema_migrations' # Truncate schema_migrations to ensure migrations re-run - connection.execute('TRUNCATE schema_migrations') if connection.data_source_exists? 'schema_migrations' + connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html - # MySQL: http://dev.mysql.com/doc/refman/5.7/en/drop-table.html # Add `IF EXISTS` because cascade could have already deleted a table. tables.each { |t| connection.execute("DROP TABLE IF EXISTS #{connection.quote_table_name(t)} CASCADE") } - - # If MySQL, re-enable foreign key checks - connection.execute('SET FOREIGN_KEY_CHECKS=1') if Gitlab::Database.mysql? end desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' @@ -77,5 +70,13 @@ namespace :gitlab do Gitlab::DowntimeCheck.new.check_and_print(migrations) end + + desc 'Sets up EE specific database functionality' + + if Gitlab.ee? + task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + else + task :setup_ee + end end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index e876b23d43f..5d86d6e466c 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -31,7 +31,6 @@ namespace :gitlab do terminate_all_connections unless Rails.env.production? Rake::Task["db:reset"].invoke - Rake::Task["add_limits_mysql"].invoke Rake::Task["setup_postgresql"].invoke Rake::Task["db:seed_fu"].invoke rescue Gitlab::TaskAbortedByUserError @@ -46,8 +45,6 @@ namespace :gitlab do # method terminates all the connections so that a subsequent DROP # will work. def self.terminate_all_connections - return false unless Gitlab::Database.postgresql? - cmd = <<~SQL SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index e058e9fe069..fdcd34320b1 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,25 +5,42 @@ namespace :gitlab do end desc "GitLab | Update project templates" - task :update_project_templates do - include Gitlab::ImportExport::CommandLineUtil + task :update_project_templates, [] => :environment do |_task, args| + # we need an instance method from Gitlab::ImportExport::CommandLineUtil and don't + # want to include it in the task, as this would affect subsequent tasks as well + downloader = Class.new do + extend Gitlab::ImportExport::CommandLineUtil + + def self.call(uploader, upload_path) + download_or_copy_upload(uploader, upload_path) + end + end + + template_names = args.extras.to_set if Rails.env.production? - puts "This rake task is not meant fo production instances".red - exit(1) + raise "This rake task is not meant for production instances" end admin = User.find_by(admin: true) unless admin - puts "No admin user could be found".red - exit(1) + raise "No admin user could be found" end - Gitlab::ProjectTemplate.all.each do |template| + tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}" + puts "Creating temporary namespace #{tmp_namespace_path}" + tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path) + + templates = if template_names.empty? + Gitlab::ProjectTemplate.all + else + Gitlab::ProjectTemplate.all.select { |template| template_names.include?(template.name) } + end + + templates.each do |template| params = { - import_url: template.clone_url, - namespace_id: admin.namespace.id, + namespace_id: tmp_namespace.id, path: template.name, skip_wiki: true } @@ -32,33 +49,67 @@ namespace :gitlab do project = Projects::CreateService.new(admin, params).execute unless project.persisted? - puts project.errors.messages - exit(1) + raise "Failed to create project: #{project.errors.messages}" end - loop do - if project.finished? - puts "Import finished for #{template.name}" - break + uri_encoded_project_path = template.uri_encoded_project_path + + # extract a concrete commit for signing off what we actually downloaded + # this way we do the right thing even if the repository gets updated in the meantime + get_commits_response = Gitlab::HTTP.get("https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/commits", + query: { page: 1, per_page: 1 } + ) + raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success? + + commit_sha = get_commits_response.parsed_response.dig(0, 'id') + + project_archive_uri = "https://gitlab.com/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}" + commit_message = <<~MSG + Initialized from '#{template.title}' project template + + Template repository: #{template.preview} + Commit SHA: #{commit_sha} + MSG + + Dir.mktmpdir do |tmpdir| + Dir.chdir(tmpdir) do + Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz']) + Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz']) + extracted_project_basename = Dir['*/'].first + Dir.chdir(extracted_project_basename) do + Gitlab::TaskHelpers.run_command!(%w(git init)) + Gitlab::TaskHelpers.run_command!(%w(git add .)) + Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message]) + + # Hacky workaround to push to the project in a way that works with both GDK and the test environment + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab::TaskHelpers.run_command!(['git', 'remote', 'add', 'origin', "file://#{project.repository.raw.path}"]) + end + Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master']) + end end + end - if project.failed? - puts "Failed to import from #{project_params[:import_url]}".red - exit(1) - end + project.reset - puts "Waiting for the import to finish" + Projects::ImportExport::ExportService.new(project, admin).execute + downloader.call(project.export_file, template.archive_path) - sleep(5) - project.reset + unless Projects::DestroyService.new(project, admin).execute + puts "Failed to destroy project #{template.name} (but namespace will be cleaned up later)" end - Projects::ImportExport::ExportService.new(project, admin).execute - download_or_copy_upload(project.export_file, template.archive_path) - Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end - puts "Done".green + + success = true + ensure + if tmp_namespace + puts "Destroying temporary namespace #{tmp_namespace_path}" + tmp_namespace.destroy + end + + puts "Done".green if success end def update(template) diff --git a/lib/tasks/gitlab/uploads/legacy.rake b/lib/tasks/gitlab/uploads/legacy.rake new file mode 100644 index 00000000000..18fb8afe455 --- /dev/null +++ b/lib/tasks/gitlab/uploads/legacy.rake @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :uploads do + namespace :legacy do + desc "GitLab | Uploads | Migrate all legacy attachments" + task migrate: :environment do + class Upload < ApplicationRecord + self.table_name = 'uploads' + + include ::EachBatch + end + + migration = 'LegacyUploadsMigrator'.freeze + batch_size = 5000 + delay_interval = 5.minutes.to_i + + Upload.where(uploader: 'AttachmentUploader').each_batch(of: batch_size) do |relation, index| + start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + delay = index * delay_interval + + BackgroundMigrationWorker.perform_in(delay, migration, [start_id, end_id]) + end + end + end + end +end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 2dc14183fa3..36590010406 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,15 +1,8 @@ unless Rails.env.production? namespace :karma do + # alias exists for legacy reasons desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - task fixtures: ['karma:rspec_fixtures'] - - desc 'GitLab | Karma | Generate fixtures using RSpec' - RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args| - args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') - ENV['NO_KNAPSACK'] = 'true' - t.pattern = args[:pattern] - t.rspec_opts = '--format documentation' - end + task fixtures: ['frontend:fixtures'] desc 'GitLab | Karma | Run JavaScript tests' task tests: ['yarn:check'] do diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake deleted file mode 100644 index c77fa49d586..00000000000 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ /dev/null @@ -1,17 +0,0 @@ -require Rails.root.join('db/migrate/limits_to_mysql') -require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') -require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') -require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') -require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') -require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') - -desc "GitLab | Add limits to strings in mysql database" -task add_limits_mysql: :environment do - puts "Adding limits to schema.rb for mysql" - LimitsToMysql.new.up - MarkdownCacheLimitsToMysql.new.up - MergeRequestDiffFileLimitsToMysql.new.up - LimitsCiBuildTraceChunksRawDataForMysql.new.up - IncreaseMysqlTextLimitForGpgKeys.new.up - PrometheusMetricsLimitsToMysql.new.up -end diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake index 56b81106c5f..4ec4fdd281f 100644 --- a/lib/tasks/services.rake +++ b/lib/tasks/services.rake @@ -86,7 +86,7 @@ namespace :services do doc_start = Time.now doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') - result = ERB.new(services_template, 0, '>') + result = ERB.new(services_template, trim_mode: '>') .result(OpenStruct.new(services: services).instance_eval { binding }) File.open(doc_path, 'w') do |f| diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index c881ad4cf12..bf18332a8eb 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -2,8 +2,6 @@ return if Rails.env.production? -Rake::Task["spec"].clear if Rake::Task.task_defined?('spec') - namespace :spec do desc 'GitLab | RSpec | Run unit tests' RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| @@ -26,63 +24,8 @@ namespace :spec do t.rspec_opts = args[:rspec_opts] end - desc '[Deprecated] Use the "bin/rspec --tag api" instead' - task :api do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @api) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use the "spec:system" task instead' - task :feature do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @feature) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/models" instead' - task :models do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @models) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/services" instead' - task :services do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @services) - ] - run_commands(cmds) - end - - desc '[Deprecated] Use "bin/rspec spec/lib" instead' - task :lib do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec --tag @lib) - ] - run_commands(cmds) - end -end - -desc "GitLab | Run specs" -task :spec do - cmds = [ - %w(rake gitlab:setup), - %w(rspec spec) - ] - run_commands(cmds) -end - -def run_commands(cmds) - cmds.each do |cmd| - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") + desc 'Run the code examples in spec/requests/api' + RSpec::Core::RakeTask.new(:api) do |t| + t.pattern = 'spec/requests/api/**/*_spec.rb' end end |