diff options
Diffstat (limited to 'app/models')
36 files changed, 456 insertions, 196 deletions
diff --git a/app/models/active_session.rb b/app/models/active_session.rb index 345767179eb..fdd210f0fba 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -93,12 +93,12 @@ class ActiveSession end def self.list_sessions(user) - sessions_from_ids(session_ids_for_user(user)) + sessions_from_ids(session_ids_for_user(user.id)) end - def self.session_ids_for_user(user) + def self.session_ids_for_user(user_id) Gitlab::Redis::SharedState.with do |redis| - redis.smembers(lookup_key_name(user.id)) + redis.smembers(lookup_key_name(user_id)) end end @@ -129,7 +129,7 @@ class ActiveSession end def self.cleaned_up_lookup_entries(redis, user) - session_ids = session_ids_for_user(user) + session_ids = session_ids_for_user(user.id) entries = raw_active_session_entries(session_ids, user.id) # remove expired keys. diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 8e558487c1c..a769a8f07fd 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -41,6 +41,11 @@ class ApplicationSetting < ApplicationRecord validates :uuid, presence: true + validates :outbound_local_requests_whitelist, + length: { maximum: 1_000, message: N_('is too long (maximum is 1000 entries)') } + + validates :outbound_local_requests_whitelist, qualified_domain_array: true, allow_blank: true + validates :session_expire_delay, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index df4caed175d..1e612bd0e78 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -2,6 +2,7 @@ module ApplicationSettingImplementation extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace | # or @@ -96,7 +97,9 @@ module ApplicationSettingImplementation diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, commit_email_hostname: default_commit_email_hostname, protected_ci_variables: false, - local_markdown_version: 0 + local_markdown_version: 0, + outbound_local_requests_whitelist: [], + raw_blob_request_limit: 300 } end @@ -131,31 +134,52 @@ module ApplicationSettingImplementation end def domain_whitelist_raw - self.domain_whitelist&.join("\n") + array_to_string(self.domain_whitelist) end def domain_blacklist_raw - self.domain_blacklist&.join("\n") + array_to_string(self.domain_blacklist) end def domain_whitelist_raw=(values) - self.domain_whitelist = [] - self.domain_whitelist = values.split(DOMAIN_LIST_SEPARATOR) - self.domain_whitelist.reject! { |d| d.empty? } - self.domain_whitelist + self.domain_whitelist = domain_strings_to_array(values) end def domain_blacklist_raw=(values) - self.domain_blacklist = [] - self.domain_blacklist = values.split(DOMAIN_LIST_SEPARATOR) - self.domain_blacklist.reject! { |d| d.empty? } - self.domain_blacklist + self.domain_blacklist = domain_strings_to_array(values) end def domain_blacklist_file=(file) self.domain_blacklist_raw = file.read end + def outbound_local_requests_whitelist_raw + array_to_string(self.outbound_local_requests_whitelist) + end + + def outbound_local_requests_whitelist_raw=(values) + self.outbound_local_requests_whitelist = domain_strings_to_array(values) + end + + def outbound_local_requests_whitelist_arrays + strong_memoize(:outbound_local_requests_whitelist_arrays) do + ip_whitelist = [] + domain_whitelist = [] + + self.outbound_local_requests_whitelist.each do |str| + ip_obj = Gitlab::Utils.string_to_ip_object(str) + + if ip_obj + ip_whitelist << ip_obj + else + domain_whitelist << str + end + end + + [ip_whitelist, domain_whitelist] + end + end + def repository_storages Array(read_attribute(:repository_storages)) end @@ -255,6 +279,17 @@ module ApplicationSettingImplementation private + def array_to_string(arr) + arr&.join("\n") + end + + def domain_strings_to_array(values) + values + .split(DOMAIN_LIST_SEPARATOR) + .reject(&:empty?) + .uniq + end + def ensure_uuid! return if uuid? diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 635fcc86166..07813e03f3a 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -40,6 +40,7 @@ module Ci has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent + has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id Ci::JobArtifact.file_types.each do |key, value| has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id @@ -48,6 +49,7 @@ module Ci has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build accepts_nested_attributes_for :runner_session + accepts_nested_attributes_for :job_variables delegate :url, to: :runner_session, prefix: true, allow_nil: true delegate :terminal_specification, to: :runner_session, allow_nil: true @@ -331,10 +333,10 @@ module Ci end # rubocop: disable CodeReuse/ServiceClass - def play(current_user) + def play(current_user, job_variables_attributes = nil) Ci::PlayBuildService .new(project, current_user) - .execute(self) + .execute(self, job_variables_attributes) end # rubocop: enable CodeReuse/ServiceClass @@ -432,6 +434,7 @@ module Ci Gitlab::Ci::Variables::Collection.new .concat(persisted_variables) .concat(scoped_variables) + .concat(job_variables) .concat(persisted_environment_variables) .to_runner_variables end @@ -531,6 +534,14 @@ module Ci trace.exist? end + def has_live_trace? + trace.live_trace_exist? + end + + def has_archived_trace? + trace.archived_trace_exist? + end + def artifacts_file job_artifacts_archive&.file end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index f80e98e5bca..e132cb045e2 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -176,6 +176,10 @@ module Ci end end + def self.archived_trace_exists_for?(job_id) + where(job_id: job_id).trace.take&.file&.file&.exists? + end + private def file_format_adapter_class diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb new file mode 100644 index 00000000000..862a0bc1299 --- /dev/null +++ b/app/models/ci/job_variable.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class JobVariable < ApplicationRecord + extend Gitlab::Ci::Model + include NewHasVariable + + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + alias_attribute :secret_value, :value + + validates :key, uniqueness: { scope: :job_id } + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2262282e647..1c76f401690 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -229,15 +229,15 @@ module Ci # # ref - The name (or names) of the branch(es)/tag(s) to limit the list of # pipelines to. + # sha - The commit SHA (or mutliple SHAs) to limit the list of pipelines to. # limit - This limits a backlog search, default to 100. - def self.newest_first(ref: nil, limit: 100) + def self.newest_first(ref: nil, sha: nil, limit: 100) relation = order(id: :desc) relation = relation.where(ref: ref) if ref + relation = relation.where(sha: sha) if sha if limit ids = relation.limit(limit).select(:id) - # MySQL does not support limit in subquery - ids = ids.pluck(:id) if Gitlab::Database.mysql? relation = relation.where(id: ids) end @@ -248,10 +248,14 @@ module Ci newest_first(ref: ref).pluck(:status).first end - def self.latest_successful_for(ref) + def self.latest_successful_for_ref(ref) newest_first(ref: ref).success.take end + def self.latest_successful_for_sha(sha) + newest_first(sha: sha).success.take + end + def self.latest_successful_for_refs(refs) relation = newest_first(ref: refs).success diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 07d00503861..43ff874ac23 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -264,7 +264,7 @@ module Ci private def cleanup_runner_queue - Gitlab::Redis::Queues.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) end end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 6ae8c3bd7f3..6533b7a186e 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.6.0'.freeze + VERSION = '0.7.0'.freeze self.table_name = 'clusters_applications_runners' diff --git a/app/models/commit.rb b/app/models/commit.rb index be37fa2e76f..0889ce7e287 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -346,7 +346,7 @@ class Commit if commits_in_merge_request.present? message_body << "" - commits_in_merge_request.reverse.each do |commit_in_merge| + commits_in_merge_request.reverse_each do |commit_in_merge| message_body << "#{commit_in_merge.short_id} #{commit_in_merge.title}" end end diff --git a/app/models/concerns/case_sensitivity.rb b/app/models/concerns/case_sensitivity.rb index c93b6589ee7..abddbf1c7e3 100644 --- a/app/models/concerns/case_sensitivity.rb +++ b/app/models/concerns/case_sensitivity.rb @@ -40,14 +40,10 @@ module CaseSensitivity end def lower_value(value) - return value if Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new('LOWER', [Arel::Nodes.build_quoted(value)]) end def lower_column(column) - return column if Gitlab::Database.mysql? - column.lower end end diff --git a/app/models/concerns/descendant.rb b/app/models/concerns/descendant.rb deleted file mode 100644 index 4c436522122..00000000000 --- a/app/models/concerns/descendant.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Descendant - extend ActiveSupport::Concern - - class_methods do - def supports_nested_objects? - Gitlab::Database.postgresql? - end - end -end diff --git a/app/models/concerns/new_has_variable.rb b/app/models/concerns/new_has_variable.rb new file mode 100644 index 00000000000..429bf496872 --- /dev/null +++ b/app/models/concerns/new_has_variable.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module NewHasVariable + extend ActiveSupport::Concern + include HasVariable + + included do + attr_encrypted :value, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + insecure_mode: false + end +end diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index af387c99f3d..0648b4a78e1 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -47,7 +47,7 @@ module ProtectedRef def access_levels_for_ref(ref, action:, protected_refs: nil) self.matching(ref, protected_refs: protected_refs) - .map(&:"#{action}_access_levels").flatten + .flat_map(&:"#{action}_access_levels") end # Returns all protected refs that match the given ref name. diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index e4fe46d722a..9cd7b8d6258 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -1,5 +1,26 @@ # frozen_string_literal: true +# This module makes it possible to handle items as a list, where the order of items can be easily altered +# Requirements: +# +# - Only works for ActiveRecord models +# - relative_position integer field must present on the model +# - This module uses GROUP BY: the model should have a parent relation, example: project -> issues, project is the parent relation (issues table has a parent_id column) +# +# Setup like this in the body of your class: +# +# include RelativePositioning +# +# # base query used for the position calculation +# def self.relative_positioning_query_base(issue) +# where(deleted: false) +# end +# +# # column that should be used in GROUP BY +# def self.relative_positioning_parent_column +# :project_id +# end +# module RelativePositioning extend ActiveSupport::Concern @@ -93,7 +114,7 @@ module RelativePositioning return move_after(before) unless after return move_before(after) unless before - # If there is no place to insert an issue we need to create one by moving the before issue closer + # If there is no place to insert an item we need to create one by moving the before item closer # to its predecessor. This process will recursively move all the predecessors until we have a place if (after.relative_position - before.relative_position) < 2 before.move_before @@ -108,11 +129,11 @@ module RelativePositioning pos_after = before.next_relative_position if before.shift_after? - issue_to_move = self.class.in_parents(parent_ids).find_by!(relative_position: pos_after) - issue_to_move.move_after - @positionable_neighbours = [issue_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables + item_to_move = self.class.relative_positioning_query_base(self).find_by!(relative_position: pos_after) + item_to_move.move_after + @positionable_neighbours = [item_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables - pos_after = issue_to_move.relative_position + pos_after = item_to_move.relative_position end self.relative_position = self.class.position_between(pos_before, pos_after) @@ -123,11 +144,11 @@ module RelativePositioning pos_before = after.prev_relative_position if after.shift_before? - issue_to_move = self.class.in_parents(parent_ids).find_by!(relative_position: pos_before) - issue_to_move.move_before - @positionable_neighbours = [issue_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables + item_to_move = self.class.relative_positioning_query_base(self).find_by!(relative_position: pos_before) + item_to_move.move_before + @positionable_neighbours = [item_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables - pos_before = issue_to_move.relative_position + pos_before = item_to_move.relative_position end self.relative_position = self.class.position_between(pos_before, pos_after) @@ -141,13 +162,13 @@ module RelativePositioning self.relative_position = self.class.position_between(min_relative_position || START_POSITION, MIN_POSITION) end - # Indicates if there is an issue that should be shifted to free the place + # Indicates if there is an item that should be shifted to free the place def shift_after? next_pos = next_relative_position next_pos && (next_pos - relative_position) == 1 end - # Indicates if there is an issue that should be shifted to free the place + # Indicates if there is an item that should be shifted to free the place def shift_before? prev_pos = prev_relative_position prev_pos && (relative_position - prev_pos) == 1 @@ -159,7 +180,7 @@ module RelativePositioning def save_positionable_neighbours return unless @positionable_neighbours - status = @positionable_neighbours.all? { |issue| issue.save(touch: false) } + status = @positionable_neighbours.all? { |item| item.save(touch: false) } @positionable_neighbours = nil status @@ -170,16 +191,15 @@ module RelativePositioning # When calculating across projects, this is much more efficient than # MAX(relative_position) without the GROUP BY, due to index usage: # https://gitlab.com/gitlab-org/gitlab-ce/issues/54276#note_119340977 - relation = self.class - .in_parents(parent_ids) + relation = self.class.relative_positioning_query_base(self) .order(Gitlab::Database.nulls_last_order('position', 'DESC')) + .group(self.class.relative_positioning_parent_column) .limit(1) - .group(self.class.parent_column) relation = yield relation if block_given? relation - .pluck(self.class.parent_column, Arel.sql("#{calculation}(relative_position) AS position")) + .pluck(self.class.relative_positioning_parent_column, Arel.sql("#{calculation}(relative_position) AS position")) .first&. last end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 9becab632f3..116e8967651 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -33,29 +33,12 @@ module Routable # # Returns a single object, or nil. def find_by_full_path(path, follow_redirects: false) - # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so - # any literal matches come first, for this we have to use "BINARY". - # Without this there's still no guarantee in what order MySQL will return - # rows. - # - # Why do we do this? - # - # Even though we have Rails validation on Route for unique paths - # (case-insensitive), there are old projects in our DB (and possibly - # clients' DBs) that have the same path with different cases. - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/18603. Also note that - # our unique index is case-sensitive in Postgres. - binary = Gitlab::Database.mysql? ? 'BINARY' : '' - order_sql = Arel.sql("(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") + order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)") found = where_full_path_in([path]).reorder(order_sql).take return found if found if follow_redirects - if Gitlab::Database.postgresql? - joins(:redirect_routes).find_by("LOWER(redirect_routes.path) = LOWER(?)", path) - else - joins(:redirect_routes).find_by(redirect_routes: { path: path }) - end + joins(:redirect_routes).find_by("LOWER(redirect_routes.path) = LOWER(?)", path) end end @@ -67,27 +50,13 @@ module Routable # # Returns an ActiveRecord::Relation. def where_full_path_in(paths) - wheres = [] - cast_lower = Gitlab::Database.postgresql? + return none if paths.empty? - paths.each do |path| - path = connection.quote(path) - - where = - if cast_lower - "(LOWER(routes.path) = LOWER(#{path}))" - else - "(routes.path = #{path})" - end - - wheres << where + wheres = paths.map do |path| + "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))" end - if wheres.empty? - none - else - joins(:route).where(wheres.join(' OR ')) - end + joins(:route).where(wheres.join(' OR ')) end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index b42adad94ba..8b536a123fc 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -15,7 +15,8 @@ module Taskable INCOMPLETE_PATTERN = /(\[[\s]\])/.freeze ITEM_PATTERN = %r{ ^ - \s*(?:[-+*]|(?:\d+\.)) # list prefix required - task item has to be always in a list + (?:(?:>\s{0,4})*) # optional blockquote characters + \s*(?:[-+*]|(?:\d+\.)) # list prefix required - task item has to be always in a list \s+ # whitespace prefix has to be always presented for a list item (\[\s\]|\[[xX]\]) # checkbox (\s.+) # followed by whitespace and some text. diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 39e12ac2b06..facd81dde80 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -86,4 +86,9 @@ class ContainerRepository < ApplicationRecord def self.build_root_repository(project) self.new(project: project, name: '') end + + def self.find_by_path!(path) + self.find_by!(project: path.repository_project, + name: path.repository_name) + end end diff --git a/app/models/cycle_analytics/group_level.rb b/app/models/cycle_analytics/group_level.rb index 508cde0ca00..a41e1375484 100644 --- a/app/models/cycle_analytics/group_level.rb +++ b/app/models/cycle_analytics/group_level.rb @@ -11,9 +11,7 @@ module CycleAnalytics end def summary - @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, - from: options[:from], - current_user: options[:current_user]).data + @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, options: options).data end def permissions(*) diff --git a/app/models/deployment.rb b/app/models/deployment.rb index b69cda4f2f9..68586e7a1fd 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -128,17 +128,8 @@ class Deployment < ApplicationRecord merge_requests = merge_requests.where("merge_request_metrics.merged_at >= ?", previous_deployment.finished_at) end - # Need to use `map` instead of `select` because MySQL doesn't allow `SELECT`ing from the same table - # that we're updating. - merge_request_ids = - if Gitlab::Database.postgresql? - merge_requests.select(:id) - elsif Gitlab::Database.mysql? - merge_requests.map(&:id) - end - MergeRequest::Metrics - .where(merge_request_id: merge_request_ids, first_deployed_to_production_at: nil) + .where(merge_request_id: merge_requests.select(:id), first_deployed_to_production_at: nil) .update_all(first_deployed_to_production_at: finished_at) end diff --git a/app/models/deployment_metrics.rb b/app/models/deployment_metrics.rb index cfe762ca25e..2056c8bc59c 100644 --- a/app/models/deployment_metrics.rb +++ b/app/models/deployment_metrics.rb @@ -44,18 +44,7 @@ class DeploymentMetrics end end - # TODO remove fallback case to deployment_platform_cluster. - # Otherwise we will continue to pay the performance penalty described in - # https://gitlab.com/gitlab-org/gitlab-ce/issues/63475 - # - # Removal issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/64105 def cluster_prometheus - cluster_with_fallback = cluster || deployment_platform_cluster - - cluster_with_fallback.application_prometheus if cluster_with_fallback&.application_prometheus_available? - end - - def deployment_platform_cluster - deployment.environment.deployment_platform&.cluster + cluster.application_prometheus if cluster&.application_prometheus_available? end end diff --git a/app/models/group.rb b/app/models/group.rb index 9520db1bc0a..74eb556b1b5 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -10,7 +10,6 @@ class Group < Namespace include Referable include SelectForProjectAuthorization include LoadedInGroupList - include Descendant include GroupDescendant include TokenAuthenticatable include WithUploads @@ -144,6 +143,12 @@ class Group < Namespace notification_settings(hierarchy_order: hierarchy_order).where(user: user) end + def notification_email_for(user) + # Finds the closest notification_setting with a `notification_email` + notification_settings = notification_settings_for(user, hierarchy_order: :asc) + notification_settings.find { |n| n.notification_email.present? }&.notification_email + end + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -382,7 +387,7 @@ class Group < Namespace variables = Ci::GroupVariable.where(group: list_of_ids) variables = variables.unprotected unless project.protected_for?(ref) variables = variables.group_by(&:group_id) - list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten + list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact end def group_member(user) @@ -416,6 +421,10 @@ class Group < Namespace super || ::Gitlab::CurrentSettings.default_project_creation end + def subgroup_creation_level + super || ::Gitlab::Access::OWNER_SUBGROUP_ACCESS + end + private def update_two_factor_requirement diff --git a/app/models/issue.rb b/app/models/issue.rb index 12d30389910..164858dc432 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -91,11 +91,11 @@ class Issue < ApplicationRecord end end - class << self - alias_method :in_parents, :in_projects + def self.relative_positioning_query_base(issue) + in_projects(issue.parent_ids) end - def self.parent_column + def self.relative_positioning_parent_column :project_id end @@ -131,7 +131,7 @@ class Issue < ApplicationRecord when 'due_date' then order_due_date_asc when 'due_date_asc' then order_due_date_asc when 'due_date_desc' then order_due_date_desc - when 'relative_position' then order_relative_position_asc + when 'relative_position' then order_relative_position_asc.with_order_id_desc else super end diff --git a/app/models/label.rb b/app/models/label.rb index b83e0862bab..dd403562bfa 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -33,7 +33,7 @@ class Label < ApplicationRecord default_scope { order(title: :asc) } - scope :templates, -> { where(template: true) } + scope :templates, -> { where(template: true, type: [Label.name, nil]) } scope :with_title, ->(title) { where(title: title) } scope :with_lists_and_board, -> { joins(lists: :board).merge(List.movable) } scope :on_project_boards, ->(project_id) { with_lists_and_board.where(boards: { project_id: project_id }) } diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b8d7348268a..058350b16ce 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -332,8 +332,6 @@ class Namespace < ApplicationRecord end def force_share_with_group_lock_on_descendants - return unless Group.supports_nested_objects? - # We can't use `descendants.update_all` since Rails will throw away the WITH # RECURSIVE statement. We also can't use WHERE EXISTS since we can't use # different table aliases, hence we're just using WHERE IN. Since we have a diff --git a/app/models/note.rb b/app/models/note.rb index 5c31cff9816..3f182c1f099 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -292,7 +292,7 @@ class Note < ApplicationRecord end def special_role=(role) - raise "Role is undefined, #{role} not found in #{SpecialRole.values}" unless SpecialRole.values.include?(role) + raise "Role is undefined, #{role} not found in #{SpecialRole.values}" unless SpecialRole.value?(role) @special_role = role end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index e6e491634ab..27c122d3559 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -22,7 +22,7 @@ class PagesDomain < ApplicationRecord validate :validate_pages_domain validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? } - validate :validate_intermediates, if: ->(domain) { domain.certificate.present? } + validate :validate_intermediates, if: ->(domain) { domain.certificate.present? && domain.certificate_changed? } attr_encrypted :key, mode: :per_attribute_iv_and_salt, diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index f69f0e2dccb..7ae431eaad7 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -7,6 +7,7 @@ class PersonalAccessToken < ApplicationRecord add_authentication_token_field :token, digest: true REDIS_EXPIRY_TIME = 3.minutes + TOKEN_LENGTH = 20 serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize diff --git a/app/models/project.rb b/app/models/project.rb index 8dfe2212282..cca7da8c49a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -357,6 +357,7 @@ class Project < ApplicationRecord scope :sorted_by_activity, -> { reorder(Arel.sql("GREATEST(COALESCE(last_activity_at, '1970-01-01'), COALESCE(last_repository_updated_at, '1970-01-01')) DESC")) } scope :sorted_by_stars_desc, -> { reorder(star_count: :desc) } scope :sorted_by_stars_asc, -> { reorder(star_count: :asc) } + scope :sorted_by_name_asc_limited, ->(limit) { reorder(name: :asc).limit(limit) } scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) } scope :personal, ->(user) { where(namespace_id: user.namespace_id) } @@ -441,22 +442,6 @@ class Project < ApplicationRecord without_deleted.find_by_id(id) end - # Paginates a collection using a `WHERE id < ?` condition. - # - # before - A project ID to use for filtering out projects with an equal or - # greater ID. If no ID is given, all projects are included. - # - # limit - The maximum number of rows to include. - def self.paginate_in_descending_order_using_id( - before: nil, - limit: Kaminari.config.default_per_page - ) - relation = order_id_desc.limit(limit) - relation = relation.where('projects.id < ?', before) if before - - relation - end - def self.eager_load_namespace_and_owner includes(namespace: :owner) end @@ -734,16 +719,27 @@ class Project < ApplicationRecord repository.commits_by(oids: oids) end - # ref can't be HEAD, can only be branch/tag name or SHA - def latest_successful_build_for(job_name, ref = default_branch) - latest_pipeline = ci_pipelines.latest_successful_for(ref) + # ref can't be HEAD, can only be branch/tag name + def latest_successful_build_for_ref(job_name, ref = default_branch) + return unless ref + + latest_pipeline = ci_pipelines.latest_successful_for_ref(ref) + return unless latest_pipeline + + latest_pipeline.builds.latest.with_artifacts_archive.find_by(name: job_name) + end + + def latest_successful_build_for_sha(job_name, sha) + return unless sha + + latest_pipeline = ci_pipelines.latest_successful_for_sha(sha) return unless latest_pipeline latest_pipeline.builds.latest.with_artifacts_archive.find_by(name: job_name) end - def latest_successful_build_for!(job_name, ref = default_branch) - latest_successful_build_for(job_name, ref) || raise(ActiveRecord::RecordNotFound.new("Couldn't find job #{job_name}")) + def latest_successful_build_for_ref!(job_name, ref = default_branch) + latest_successful_build_for_ref(job_name, ref) || raise(ActiveRecord::RecordNotFound.new("Couldn't find job #{job_name}")) end def merge_base_commit(first_commit_id, second_commit_id) @@ -1496,12 +1492,20 @@ class Project < ApplicationRecord !namespace.share_with_group_lock end - def pipeline_for(ref, sha = nil) + def pipeline_for(ref, sha = nil, id = nil) + if id.present? + pipelines_for(ref, sha).find_by(id: id) + else + pipelines_for(ref, sha).take + end + end + + def pipelines_for(ref, sha = nil) sha ||= commit(ref).try(:sha) return unless sha - ci_pipelines.order(id: :desc).find_by(sha: sha, ref: ref) + ci_pipelines.order(id: :desc).where(sha: sha, ref: ref) end def latest_successful_pipeline_for_default_branch @@ -1510,12 +1514,12 @@ class Project < ApplicationRecord end @latest_successful_pipeline_for_default_branch = - ci_pipelines.latest_successful_for(default_branch) + ci_pipelines.latest_successful_for_ref(default_branch) end def latest_successful_pipeline_for(ref = nil) if ref && ref != default_branch - ci_pipelines.latest_successful_for(ref) + ci_pipelines.latest_successful_for_ref(ref) else latest_successful_pipeline_for_default_branch end @@ -1869,16 +1873,24 @@ class Project < ApplicationRecord end def append_or_update_attribute(name, value) - old_values = public_send(name.to_s) # rubocop:disable GitlabSecurity/PublicSend + if Project.reflect_on_association(name).try(:macro) == :has_many + # if this is 1-to-N relation, update the parent object + value.each do |item| + item.update!( + Project.reflect_on_association(name).foreign_key => id) + end + + # force to drop relation cache + public_send(name).reset # rubocop:disable GitlabSecurity/PublicSend - if Project.reflect_on_association(name).try(:macro) == :has_many && old_values.any? - update_attribute(name, old_values + value) + # succeeded + true else + # if this is another relation or attribute, update just object update_attribute(name, value) end - - rescue ActiveRecord::RecordNotSaved => e - handle_update_attribute_error(e, value) + rescue ActiveRecord::RecordInvalid => e + raise e, "Failed to set #{name}: #{e.message}" end # Tries to set repository as read_only, checking for existing Git transfers in progress beforehand @@ -2267,18 +2279,6 @@ class Project < ApplicationRecord ContainerRepository.build_root_repository(self).has_tags? end - def handle_update_attribute_error(ex, value) - if ex.message.start_with?('Failed to replace') - if value.respond_to?(:each) - invalid = value.detect(&:invalid?) - - raise ex, ([ex.message] + invalid.errors.full_messages).join(' ') if invalid - end - end - - raise ex - end - def fetch_branch_allows_collaboration(user, branch_name = nil) return false unless user diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index f39f54f0434..e11d0c48b4b 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -1,10 +1,6 @@ # frozen_string_literal: true class ProjectAutoDevops < ApplicationRecord - include IgnorableColumn - - ignore_column :domain - belongs_to :project, inverse_of: :auto_devops enum deploy_strategy: { diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 62aec4351db..4edf263433f 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -1,24 +1,47 @@ # frozen_string_literal: true +require 'slack-notifier' module ChatMessage class PipelineMessage < BaseMessage + MAX_VISIBLE_JOBS = 10 + + attr_reader :user attr_reader :ref_type attr_reader :ref attr_reader :status + attr_reader :detailed_status attr_reader :duration + attr_reader :finished_at attr_reader :pipeline_id + attr_reader :failed_stages + attr_reader :failed_jobs + + attr_reader :project + attr_reader :commit + attr_reader :committer + attr_reader :pipeline def initialize(data) super + @user = data[:user] @user_name = data.dig(:user, :username) || 'API' pipeline_attributes = data[:object_attributes] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @ref = pipeline_attributes[:ref] @status = pipeline_attributes[:status] + @detailed_status = pipeline_attributes[:detailed_status] @duration = pipeline_attributes[:duration].to_i + @finished_at = pipeline_attributes[:finished_at] ? Time.parse(pipeline_attributes[:finished_at]).to_i : nil @pipeline_id = pipeline_attributes[:id] + @failed_jobs = Array(data[:builds]).select { |b| b[:status] == 'failed' }.reverse # Show failed jobs from oldest to newest + @failed_stages = @failed_jobs.map { |j| j[:stage] }.uniq + + @project = Project.find(data[:project][:id]) + @commit = project.commit_by(oid: data[:commit][:id]) + @committer = commit.committer + @pipeline = Ci::Pipeline.find(pipeline_id) end def pretext @@ -28,38 +51,145 @@ module ChatMessage def attachments return message if markdown - [{ text: format(message), color: attachment_color }] + return [{ text: format(message), color: attachment_color }] unless fancy_notifications? + + [{ + fallback: format(message), + color: attachment_color, + author_name: user_combined_name, + author_icon: user_avatar, + author_link: author_url, + title: s_("ChatMessage|Pipeline #%{pipeline_id} %{humanized_status} in %{duration}") % + { + pipeline_id: pipeline_id, + humanized_status: humanized_status, + duration: pretty_duration(duration) + }, + title_link: pipeline_url, + fields: attachments_fields, + footer: project.name, + footer_icon: project.avatar_url, + ts: finished_at + }] end def activity { - title: "Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status}", - subtitle: "in #{project_link}", - text: "in #{pretty_duration(duration)}", + title: s_("ChatMessage|Pipeline %{pipeline_link} of %{ref_type} %{branch_link} by %{user_combined_name} %{humanized_status}") % + { + pipeline_link: pipeline_link, + ref_type: ref_type, + branch_link: branch_link, + user_combined_name: user_combined_name, + humanized_status: humanized_status + }, + subtitle: s_("ChatMessage|in %{project_link}") % { project_link: project_link }, + text: s_("ChatMessage|in %{duration}") % { duration: pretty_duration(duration) }, image: user_avatar || '' } end private + def fancy_notifications? + Feature.enabled?(:fancy_pipeline_slack_notifications, default_enabled: true) + end + + def failed_stages_field + { + title: s_("ChatMessage|Failed stage").pluralize(failed_stages.length), + value: Slack::Notifier::LinkFormatter.format(failed_stages_links), + short: true + } + end + + def failed_jobs_field + { + title: s_("ChatMessage|Failed job").pluralize(failed_jobs.length), + value: Slack::Notifier::LinkFormatter.format(failed_jobs_links), + short: true + } + end + + def yaml_error_field + { + title: s_("ChatMessage|Invalid CI config YAML file"), + value: pipeline.yaml_errors, + short: false + } + end + + def attachments_fields + fields = [ + { + title: ref_type == "tag" ? s_("ChatMessage|Tag") : s_("ChatMessage|Branch"), + value: Slack::Notifier::LinkFormatter.format(ref_name_link), + short: true + }, + { + title: s_("ChatMessage|Commit"), + value: Slack::Notifier::LinkFormatter.format(commit_link), + short: true + } + ] + + fields << failed_stages_field if failed_stages.any? + fields << failed_jobs_field if failed_jobs.any? + fields << yaml_error_field if pipeline.has_yaml_errors? + + fields + end + def message - "#{project_link}: Pipeline #{pipeline_link} of #{ref_type} #{branch_link} by #{user_combined_name} #{humanized_status} in #{pretty_duration(duration)}" + s_("ChatMessage|%{project_link}: Pipeline %{pipeline_link} of %{ref_type} %{branch_link} by %{user_combined_name} %{humanized_status} in %{duration}") % + { + project_link: project_link, + pipeline_link: pipeline_link, + ref_type: ref_type, + branch_link: branch_link, + user_combined_name: user_combined_name, + humanized_status: humanized_status, + duration: pretty_duration(duration) + } end def humanized_status - case status - when 'success' - 'passed' + if fancy_notifications? + case status + when 'success' + detailed_status == "passed with warnings" ? s_("ChatMessage|has passed with warnings") : s_("ChatMessage|has passed") + when 'failed' + s_("ChatMessage|has failed") + else + status + end else - status + case status + when 'success' + s_("ChatMessage|passed") + when 'failed' + s_("ChatMessage|failed") + else + status + end end end def attachment_color - if status == 'success' - 'good' + if fancy_notifications? + case status + when 'success' + detailed_status == 'passed with warnings' ? 'warning' : 'good' + else + 'danger' + end else - 'danger' + case status + when 'success' + 'good' + else + 'danger' + end end end @@ -71,16 +201,83 @@ module ChatMessage "[#{ref}](#{branch_url})" end + def project_url + project.web_url + end + def project_link - "[#{project_name}](#{project_url})" + "[#{project.name}](#{project_url})" + end + + def pipeline_failed_jobs_url + "#{project_url}/pipelines/#{pipeline_id}/failures" end def pipeline_url - "#{project_url}/pipelines/#{pipeline_id}" + if fancy_notifications? && failed_jobs.any? + pipeline_failed_jobs_url + else + "#{project_url}/pipelines/#{pipeline_id}" + end end def pipeline_link "[##{pipeline_id}](#{pipeline_url})" end + + def job_url(job) + "#{project_url}/-/jobs/#{job[:id]}" + end + + def job_link(job) + "[#{job[:name]}](#{job_url(job)})" + end + + def failed_jobs_links + failed = failed_jobs.slice(0, MAX_VISIBLE_JOBS) + truncated = failed_jobs.slice(MAX_VISIBLE_JOBS, failed_jobs.size) + + failed_links = failed.map { |job| job_link(job) } + + unless truncated.blank? + failed_links << s_("ChatMessage|and [%{count} more](%{pipeline_failed_jobs_url})") % { + count: truncated.size, + pipeline_failed_jobs_url: pipeline_failed_jobs_url + } + end + + failed_links.join(I18n.translate(:'support.array.words_connector')) + end + + def stage_link(stage) + # All stages link to the pipeline page + "[#{stage}](#{pipeline_url})" + end + + def failed_stages_links + failed_stages.map { |s| stage_link(s) }.join(I18n.translate(:'support.array.words_connector')) + end + + def commit_url + Gitlab::UrlBuilder.build(commit) + end + + def commit_link + "[#{commit.title}](#{commit_url})" + end + + def commits_page_url + "#{project_url}/commits/#{ref}" + end + + def ref_name_link + "[#{ref}](#{commits_page_url})" + end + + def author_url + return unless user && committer + + Gitlab::UrlBuilder.build(committer) + end end end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index e571700fd02..d08fcd8954d 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -31,7 +31,7 @@ class JiraService < IssueTrackerService # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 def self.reference_pattern(only_long: true) - @reference_pattern ||= /(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)/ + @reference_pattern ||= /(?<issue>\b#{Gitlab::Regex.jira_issue_key_regex})/ end def initialize_properties @@ -54,7 +54,7 @@ class JiraService < IssueTrackerService username: self.username, password: self.password, site: URI.join(url, '/').to_s, # Intended to find the root - context_path: url.path.chomp('/'), + context_path: url.path, auth_type: :basic, read_timeout: 120, use_cookies: true, @@ -103,6 +103,12 @@ class JiraService < IssueTrackerService "#{url}/secure/CreateIssue.jspa" end + alias_method :original_url, :url + + def url + original_url&.chomp('/') + end + def execute(push) # This method is a no-op, because currently JiraService does not # support any events. diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index af705b29f7a..6b5605f9999 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -31,7 +31,7 @@ class RemoteMirror < ApplicationRecord scope :enabled, -> { where(enabled: true) } scope :started, -> { with_update_status(:started) } - scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.hour.ago, 3.hours.ago) } state_machine :update_status, initial: :none do event :update_start do @@ -173,7 +173,7 @@ class RemoteMirror < ApplicationRecord result = URI.parse(url) result.password = '*****' if result.password - result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.user = '*****' if result.user && result.user != 'git' # tokens or other data may be saved as user result.to_s end diff --git a/app/models/repository.rb b/app/models/repository.rb index 187382ad182..a89f573e3d6 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -7,6 +7,9 @@ class Repository REF_KEEP_AROUND = 'keep-around'.freeze REF_ENVIRONMENTS = 'environments'.freeze + ARCHIVE_CACHE_TIME = 60 # Cache archives referred to by a (mutable) ref for 1 minute + ARCHIVE_CACHE_TIME_IMMUTABLE = 3600 # Cache archives referred to by an immutable reference for 1 hour + RESERVED_REFS_NAMES = %W[ heads tags diff --git a/app/models/user.rb b/app/models/user.rb index 0fd3daa3383..b439d1c0c16 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1259,6 +1259,11 @@ class User < ApplicationRecord end end + def notification_email_for(notification_group) + # Return group-specific email address if present, otherwise return global notification email address + notification_group&.notification_email_for(self) || notification_email + end + def notification_settings_for(source) if notification_settings.loaded? notification_settings.find do |notification| diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index f1326f4c8cb..b236250c24e 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -26,7 +26,7 @@ class UserPreference < ApplicationRecord def set_notes_filter(filter_id, issuable) # No need to update the column if the value is already set. - if filter_id && NOTES_FILTERS.values.include?(filter_id) + if filter_id && NOTES_FILTERS.value?(filter_id) field = notes_filter_field_for(issuable) self[field] = filter_id |