diff options
Diffstat (limited to 'app/models')
68 files changed, 1256 insertions, 500 deletions
diff --git a/app/models/appearance.rb b/app/models/appearance.rb index ff15689ecac..76cfe28742a 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -2,9 +2,8 @@ class Appearance < ActiveRecord::Base include CacheMarkdownField cache_markdown_field :description + cache_markdown_field :new_project_guidelines - validates :title, presence: true - validates :description, presence: true validates :logo, file_size: { maximum: 1.megabyte } validates :header_logo, file_size: { maximum: 1.megabyte } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 5e16badabec..253e213af81 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -153,11 +153,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { greater_than_or_equal_to: 0 } - validates :circuitbreaker_backoff_threshold, - :circuitbreaker_failure_count_threshold, - :circuitbreaker_failure_wait_time, + validates :circuitbreaker_failure_count_threshold, :circuitbreaker_failure_reset_time, :circuitbreaker_storage_timeout, + :circuitbreaker_check_interval, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -165,12 +164,26 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 1 } - validates_each :circuitbreaker_backoff_threshold do |record, attr, value| - if value.to_i >= record.circuitbreaker_failure_count_threshold - record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\ - "lower than the failure count threshold")) - end - end + validates :gitaly_timeout_default, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :gitaly_timeout_medium, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :gitaly_timeout_medium, + numericality: { less_than_or_equal_to: :gitaly_timeout_default }, + if: :gitaly_timeout_default + validates :gitaly_timeout_medium, + numericality: { greater_than_or_equal_to: :gitaly_timeout_fast }, + if: :gitaly_timeout_fast + + validates :gitaly_timeout_fast, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :gitaly_timeout_fast, + numericality: { less_than_or_equal_to: :gitaly_timeout_default }, + if: :gitaly_timeout_default SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } @@ -276,7 +289,8 @@ class ApplicationSetting < ActiveRecord::Base koding_url: nil, max_artifacts_size: Settings.artifacts['max_size'], max_attachment_size: Settings.gitlab['max_attachment_size'], - password_authentication_enabled: Settings.gitlab['password_authentication_enabled'], + password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'], + password_authentication_enabled_for_git: true, performance_bar_allowed_group_id: nil, rsa_key_restriction: 0, plantuml_enabled: false, @@ -295,10 +309,22 @@ class ApplicationSetting < ActiveRecord::Base sign_in_text: nil, signup_enabled: Settings.gitlab['signup_enabled'], terminal_max_session_time: 0, + throttle_unauthenticated_enabled: false, + throttle_unauthenticated_requests_per_period: 3600, + throttle_unauthenticated_period_in_seconds: 3600, + throttle_authenticated_web_enabled: false, + throttle_authenticated_web_requests_per_period: 7200, + throttle_authenticated_web_period_in_seconds: 3600, + throttle_authenticated_api_enabled: false, + throttle_authenticated_api_requests_per_period: 7200, + throttle_authenticated_api_period_in_seconds: 3600, two_factor_grace_period: 48, user_default_external: false, polling_interval_multiplier: 1, - usage_ping_enabled: Settings.gitlab['usage_ping_enabled'] + usage_ping_enabled: Settings.gitlab['usage_ping_enabled'], + gitaly_timeout_fast: 10, + gitaly_timeout_medium: 30, + gitaly_timeout_default: 55 } end @@ -465,6 +491,14 @@ class ApplicationSetting < ActiveRecord::Base has_attribute?(attr_name) ? public_send(attr_name) : FORBIDDEN_KEY_VALUE # rubocop:disable GitlabSecurity/PublicSend end + def allow_signup? + signup_enabled? && password_authentication_enabled_for_web? + end + + def password_authentication_enabled? + password_authentication_enabled_for_web? || password_authentication_enabled_for_git? + end + private def ensure_uuid! diff --git a/app/models/blob.rb b/app/models/blob.rb index ad0bc2e2ead..29e762724e3 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -76,12 +76,24 @@ class Blob < SimpleDelegator new(blob, project) end + def self.lazy(project, commit_id, path) + BatchLoader.for(commit_id: commit_id, path: path).batch do |items, loader| + project.repository.blobs_at(items.map(&:values)).each do |blob| + loader.call({ commit_id: blob.commit_id, path: blob.path }, blob) if blob + end + end + end + def initialize(blob, project = nil) @project = project super(blob) end + def inspect + "#<#{self.class.name} oid:#{id[0..8]} commit:#{commit_id[0..8]} path:#{path}>" + end + # Returns the data of the blob. # # If the blob is a text based blob the content is converted to UTF-8 and any @@ -95,7 +107,10 @@ class Blob < SimpleDelegator end def load_all_data! - super(project.repository) if project + # Endpoint needed: gitlab-org/gitaly#756 + Gitlab::GitalyClient.allow_n_plus_1_calls do + super(project.repository) if project + end end def no_highlighting? diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index ebeab87d8fc..ddd075e1fcb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,18 +1,26 @@ module Ci class Build < CommitStatus + prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue include Presentable include Importable + MissingDependenciesError = Class.new(StandardError) + belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' has_many :deployments, as: :deployable + has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @persisted_environment ||= Environment.find_by( @@ -31,7 +39,10 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } - scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) } + scope :with_artifacts, ->() do + where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', + '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id')) + end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, ArtifactUploader::LOCAL_STORE]) } @@ -39,8 +50,27 @@ module Ci scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } - mount_uploader :artifacts_file, ArtifactUploader - mount_uploader :artifacts_metadata, ArtifactUploader + scope :matches_tag_ids, -> (tag_ids) do + matcher = ::ActsAsTaggableOn::Tagging + .where(taggable_type: CommitStatus) + .where(context: 'tags') + .where('taggable_id = ci_builds.id') + .where.not(tag_id: tag_ids).select('1') + + where("NOT EXISTS (?)", matcher) + end + + scope :with_any_tags, -> do + matcher = ::ActsAsTaggableOn::Tagging + .where(taggable_type: CommitStatus) + .where(context: 'tags') + .where('taggable_id = ci_builds.id').select('1') + + where("EXISTS (?)", matcher) + end + + mount_uploader :legacy_artifacts_file, LegacyArtifactUploader, mount_on: :artifacts_file + mount_uploader :legacy_artifacts_metadata, LegacyArtifactUploader, mount_on: :artifacts_metadata acts_as_taggable @@ -105,12 +135,17 @@ module Ci end before_transition any => [:failed] do |build| + next unless build.project next if build.retries_max.zero? if build.retries_count < build.retries_max Ci::Build.retry(build, build.user) end end + + before_transition any => [:running] do |build| + build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') + end end def detailed_status(current_user) @@ -193,6 +228,10 @@ module Ci project.build_timeout end + def triggered_by?(current_user) + user == current_user + end + # A slugified version of the build ref, suitable for inclusion in URLs and # domain names. Rules: # @@ -240,7 +279,7 @@ module Ci @merge_request ||= begin - merge_requests = MergeRequest.includes(:merge_request_diff) + merge_requests = MergeRequest.includes(:latest_merge_request_diff) .where(source_branch: ref, source_project: pipeline.project) .reorder(iid: :desc) @@ -314,6 +353,7 @@ module Ci def execute_hooks return unless project + build_data = Gitlab::DataBuilder::Build.build(self) project.execute_hooks(build_data.dup, :job_hooks) project.execute_services(build_data.dup, :job_hooks) @@ -391,6 +431,7 @@ module Ci def keep_artifacts! self.update(artifacts_expire_at: nil) + self.job_artifacts.update_all(expire_at: nil) end def coverage_regex @@ -462,6 +503,20 @@ module Ci options[:dependencies]&.empty? end + def validates_dependencies! + dependencies.each do |dependency| + raise MissingDependenciesError unless dependency.valid_dependency? + end + end + + def valid_dependency? + return false unless complete? + return false if artifacts_expired? + return false if erased? + + true + end + def hide_secrets(trace) return unless trace @@ -478,11 +533,7 @@ module Ci private def update_artifacts_size - self.artifacts_size = if artifacts_file.exists? - artifacts_file.size - else - nil - end + self.artifacts_size = legacy_artifacts_file&.size end def erase_trace! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb new file mode 100644 index 00000000000..84fc6863567 --- /dev/null +++ b/app/models/ci/job_artifact.rb @@ -0,0 +1,36 @@ +module Ci + class JobArtifact < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :project + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + before_save :set_size, if: :file_changed? + + mount_uploader :file, JobArtifactUploader + + enum file_type: { + archive: 1, + metadata: 2 + } + + def self.artifacts_size_for(project) + self.where(project: project).sum(:size) + end + + def set_size + self.size = file.size + end + + def expire_in + expire_at - Time.now if expire_at + end + + def expire_in=(value) + self.expire_at = + if value + ChronicDuration.parse(value)&.seconds&.from_now + end + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ca65e81f27a..eebbf7c4218 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -40,7 +40,6 @@ module Ci validates :status, presence: { unless: :importing? } validate :valid_commit_sha, unless: :importing? - after_initialize :set_config_source, if: :new_record? after_create :keep_around_commits, unless: :importing? enum source: { @@ -66,8 +65,8 @@ module Ci state_machine :status, initial: :created do event :enqueue do - transition created: :pending - transition [:success, :failed, :canceled, :skipped] => :running + transition [:created, :skipped] => :pending + transition [:success, :failed, :canceled] => :running end event :run do @@ -149,34 +148,70 @@ module Ci end end - # ref can't be HEAD or SHA, can only be branch/tag name - scope :latest, ->(ref = nil) do - max_id = unscope(:select) - .select("max(#{quoted_table_name}.id)") - .group(:ref, :sha) + scope :internal, -> { where(source: internal_sources) } - if ref - where(ref: ref, id: max_id.where(ref: ref)) - else - where(id: max_id) - end + # Returns the pipelines in descending order (= newest first), optionally + # limited to a number of references. + # + # ref - The name (or names) of the branch(es)/tag(s) to limit the list of + # pipelines to. + def self.newest_first(ref = nil) + relation = order(id: :desc) + + ref ? relation.where(ref: ref) : relation end - scope :internal, -> { where(source: internal_sources) } def self.latest_status(ref = nil) - latest(ref).status + newest_first(ref).pluck(:status).first end def self.latest_successful_for(ref) - success.latest(ref).order(id: :desc).first + newest_first(ref).success.take end def self.latest_successful_for_refs(refs) - success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash| + relation = newest_first(refs).success + + relation.each_with_object({}) do |pipeline, hash| hash[pipeline.ref] ||= pipeline end end + # Returns a Hash containing the latest pipeline status for every given + # commit. + # + # The keys of this Hash are the commit SHAs, the values the statuses. + # + # commits - The list of commit SHAs to get the status for. + # ref - The ref to scope the data to (e.g. "master"). If the ref is not + # given we simply get the latest status for the commits, regardless + # of what refs their pipelines belong to. + def self.latest_status_per_commit(commits, ref = nil) + p1 = arel_table + p2 = arel_table.alias + + # This LEFT JOIN will filter out all but the newest row for every + # combination of (project_id, sha) or (project_id, sha, ref) if a ref is + # given. + cond = p1[:sha].eq(p2[:sha]) + .and(p1[:project_id].eq(p2[:project_id])) + .and(p1[:id].lt(p2[:id])) + + cond = cond.and(p1[:ref].eq(p2[:ref])) if ref + join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond) + + relation = select(:sha, :status) + .where(sha: commits) + .where(p2[:id].eq(nil)) + .joins(join.join_sources) + + relation = relation.where(ref: ref) if ref + + relation.each_with_object({}) do |row, hash| + hash[row[:sha]] = row[:status] + end + end + def self.truncate_sha(sha) sha[0...8] end @@ -300,8 +335,10 @@ module Ci def latest? return false unless ref + commit = project.commit(ref) return false unless commit + commit.sha == sha end @@ -327,7 +364,7 @@ module Ci end def has_kubernetes_active? - project.kubernetes_service&.active? + project.deployment_platform&.active? end def has_stage_seeds? @@ -409,7 +446,7 @@ module Ci end def notes - Note.for_commit_id(sha) + project.notes.for_commit_id(sha) end def process! @@ -469,7 +506,10 @@ module Ci end def latest_builds_with_artifacts - @latest_builds_with_artifacts ||= builds.latest.with_artifacts + # We purposely cast the builds to an Array here. Because we always use the + # rows if there are more than 0 this prevents us from having to run two + # queries: one to get the count and one to get the rows. + @latest_builds_with_artifacts ||= builds.latest.with_artifacts.to_a end private diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index c6509f89117..dcbb397fb78 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -1,6 +1,7 @@ module Ci class Runner < ActiveRecord::Base extend Gitlab::Ci::Model + include Gitlab::SQL::Pattern RUNNER_QUEUE_EXPIRY_TIME = 60.minutes ONLINE_CONTACT_TIMEOUT = 1.hour @@ -59,10 +60,7 @@ module Ci # # Returns an ActiveRecord::Relation. def self.search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:token].matches(pattern).or(t[:description].matches(pattern))) + fuzzy_search(query, [:token, :description]) end def self.contact_time_deadline @@ -114,7 +112,7 @@ module Ci def can_pick?(build) return false if self.ref_protected? && !build.protected? - assignable_for?(build.project) && accepting_tags?(build) + assignable_for?(build.project_id) && accepting_tags?(build) end def only_for?(project) @@ -173,8 +171,8 @@ module Ci end end - def assignable_for?(project) - is_shared? || projects.exists?(id: project.id) + def assignable_for?(project_id) + is_shared? || projects.exists?(id: project_id) end def accepting_tags?(build) diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 185d9473aab..55419189282 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -17,8 +17,7 @@ module Clusters # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true - # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration - has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true has_one :application_helm, class_name: 'Clusters::Applications::Helm' has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' @@ -29,15 +28,9 @@ module Clusters validates :name, cluster_name: true validate :restrict_modification, on: :update - # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3 - # We need callback here because `enabled` belongs to Clusters::Cluster - # Callbacks in Clusters::Platforms::Kubernetes will not be called after update - after_save :update_kubernetes_integration! - delegate :status, to: :provider, allow_nil: true delegate :status_reason, to: :provider, allow_nil: true delegate :on_creation?, to: :provider, allow_nil: true - delegate :update_kubernetes_integration!, to: :platform, allow_nil: true delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true delegate :installed?, to: :application_helm, prefix: true, allow_nil: true @@ -62,6 +55,10 @@ module Clusters end end + def created? + status_name == :created + end + def applications [ application_helm || build_application_helm, @@ -77,6 +74,10 @@ module Clusters return platform_kubernetes if kubernetes? end + def managed? + !user? + end + def first_project return @first_project if defined?(@first_project) diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 6dc1ee810d3..9160a169452 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -1,7 +1,12 @@ module Clusters module Platforms class Kubernetes < ActiveRecord::Base + include Gitlab::CurrentSettings + include Gitlab::Kubernetes + include ReactiveCaching + self.table_name = 'cluster_platforms_kubernetes' + self.reactive_cache_key = ->(kubernetes) { [kubernetes.class.model_name.singular, kubernetes.id] } belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster' @@ -29,19 +34,17 @@ module Clusters validates :api_url, url: true, presence: true validates :token, presence: true - # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes - after_destroy :destroy_kubernetes_integration! + validate :prevent_modification, on: :update + + after_save :clear_reactive_cache! alias_attribute :ca_pem, :ca_cert delegate :project, to: :cluster, allow_nil: true delegate :enabled?, to: :cluster, allow_nil: true + delegate :managed?, to: :cluster, allow_nil: true - class << self - def namespace_for_project(project) - "#{project.path}-#{project.id}" - end - end + alias_method :active?, :enabled? def actual_namespace if namespace.present? @@ -51,58 +54,138 @@ module Clusters end end - def default_namespace - self.class.namespace_for_project(project) if project + def predefined_variables + config = YAML.dump(kubeconfig) + + variables = [ + { key: 'KUBE_URL', value: api_url, public: true }, + { key: 'KUBE_TOKEN', value: token, public: false }, + { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true }, + { key: 'KUBECONFIG', value: config, public: false, file: true } + ] + + if ca_pem.present? + variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true } + variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true } + end + + variables end - def kubeclient - @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service? + # Constructs a list of terminals from the reactive cache + # + # Returns nil if the cache is empty, in which case you should try again a + # short time later + def terminals(environment) + with_reactive_cache do |data| + pods = filter_by_label(data[:pods], app: environment.slug) + terminals = pods.flat_map { |pod| terminals_for_pod(api_url, actual_namespace, pod) } + terminals.each { |terminal| add_terminal_auth(terminal, terminal_auth) } + end end - def update_kubernetes_integration! - raise 'Kubernetes service already configured' unless manages_kubernetes_service? + # Caches resources in the namespace so other calls don't need to block on + # network access + def calculate_reactive_cache + return unless enabled? && project && !project.pending_delete? - # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false - cluster.reload + # We may want to cache extra things in the future + { pods: read_pods } + end + + def kubeclient + @kubeclient ||= build_kubeclient! + end + + private - ensure_kubernetes_service&.update!( - active: enabled?, - api_url: api_url, - namespace: namespace, + def kubeconfig + to_kubeconfig( + url: api_url, + namespace: actual_namespace, token: token, - ca_pem: ca_cert - ) + ca_pem: ca_pem) end - def active? - manages_kubernetes_service? + def default_namespace + return unless project + + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') end - private + def build_kubeclient!(api_path: 'api', api_version: 'v1') + raise "Incomplete settings" unless api_url && actual_namespace - def enforce_namespace_to_lower_case - self.namespace = self.namespace&.downcase + unless (username && password) || token + raise "Either username/password or token is required to access API" + end + + ::Kubeclient::Client.new( + join_api_url(api_path), + api_version, + auth_options: kubeclient_auth_options, + ssl_options: kubeclient_ssl_options, + http_proxy_uri: ENV['http_proxy'] + ) end - # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class - def manages_kubernetes_service? - return true unless kubernetes_service&.active? + # Returns a hash of all pods in the namespace + def read_pods + kubeclient = build_kubeclient! + + kubeclient.get_pods(namespace: actual_namespace).as_json + rescue KubeException => err + raise err unless err.error_code == 404 + + [] + end + + def kubeclient_ssl_options + opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER } + + if ca_pem.present? + opts[:cert_store] = OpenSSL::X509::Store.new + opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) + end + + opts + end - kubernetes_service.api_url == api_url + def kubeclient_auth_options + { bearer_token: token } end - def destroy_kubernetes_integration! - return unless manages_kubernetes_service? + def join_api_url(api_path) + url = URI.parse(api_url) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [prefix, api_path].join("/") - kubernetes_service&.destroy! + url.to_s end - def kubernetes_service - @kubernetes_service ||= project&.kubernetes_service + def terminal_auth + { + token: token, + ca_pem: ca_pem, + max_session_time: current_application_settings.terminal_max_session_time + } end - def ensure_kubernetes_service - @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service + def enforce_namespace_to_lower_case + self.namespace = self.namespace&.downcase + end + + def prevent_modification + return unless managed? + + if api_url_changed? || token_changed? || ca_pem_changed? + errors.add(:base, "cannot modify managed cluster") + return false + end + + true end end end diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index ee2e43ee9dd..7fac32466ab 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -56,6 +56,7 @@ module Clusters before_transition any => [:creating] do |provider, transition| operation_id = transition.args.first raise ArgumentError.new('operation_id is required') unless operation_id.present? + provider.operation_id = operation_id end diff --git a/app/models/commit.rb b/app/models/commit.rb index 6dba154a6ea..307e4fcedfe 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -1,3 +1,4 @@ +# coding: utf-8 class Commit extend ActiveModel::Naming extend Gitlab::Cache::RequestCache @@ -25,7 +26,7 @@ class Commit DIFF_HARD_LIMIT_FILES = 1000 DIFF_HARD_LIMIT_LINES = 50000 - MIN_SHA_LENGTH = 7 + MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze def banzai_render_context(field) @@ -80,10 +81,11 @@ class Commit @raw = raw_commit @project = project + @statuses = {} end def id - @raw.id + raw.id end def ==(other) @@ -108,12 +110,12 @@ class Commit @link_reference_pattern ||= super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})/) end - def to_reference(from_project = nil, full: false) - commit_reference(from_project, id, full: full) + def to_reference(from = nil, full: false) + commit_reference(from, id, full: full) end - def reference_link_text(from_project = nil, full: false) - commit_reference(from_project, short_id, full: full) + def reference_link_text(from = nil, full: false) + commit_reference(from, short_id, full: full) end def diff_line_count @@ -236,11 +238,13 @@ class Commit end def status(ref = nil) - @statuses ||= {} - return @statuses[ref] if @statuses.key?(ref) - @statuses[ref] = pipelines.latest_status(ref) + @statuses[ref] = project.pipelines.latest_status_per_commit(id, ref)[id] + end + + def set_status_for_ref(ref, status) + @statuses[ref] = status end def signature @@ -358,7 +362,7 @@ class Commit @deltas ||= raw.deltas end - def diffs(diff_options = nil) + def diffs(diff_options = {}) Gitlab::Diff::FileCollection::Commit.new(self, diff_options: diff_options) end @@ -378,8 +382,8 @@ class Commit private - def commit_reference(from_project, referable_commit_id, full: false) - reference = project.to_reference(from_project, full: full) + def commit_reference(from, referable_commit_id, full: false) + reference = project.to_reference(from, full: full) if reference.present? "#{reference}#{self.class.reference_prefix}#{referable_commit_id}" diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb new file mode 100644 index 00000000000..dd93af9df64 --- /dev/null +++ b/app/models/commit_collection.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# A collection of Commit instances for a specific project and Git reference. +class CommitCollection + include Enumerable + + attr_reader :project, :ref, :commits + + # project - The project the commits belong to. + # commits - The Commit instances to store. + # ref - The name of the ref (e.g. "master"). + def initialize(project, commits, ref = nil) + @project = project + @commits = commits + @ref = ref + end + + def each(&block) + commits.each(&block) + end + + # Sets the pipeline status for every commit. + # + # Setting this status ahead of time removes the need for running a query for + # every commit we're displaying. + def with_pipeline_status + statuses = project.pipelines.latest_status_per_commit(map(&:id), ref) + + each do |commit| + commit.set_status_for_ref(ref, statuses[commit.id]) + end + + self + end + + def respond_to_missing?(message, inc_private = false) + commits.respond_to?(message, inc_private) + end + + # rubocop:disable GitlabSecurity/PublicSend + def method_missing(message, *args, &block) + commits.public_send(message, *args, &block) + end +end diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 84e2e8a5dd5..b93c111dabc 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -89,8 +89,8 @@ class CommitRange alias_method :id, :to_s - def to_reference(from_project = nil, full: false) - project_reference = project.to_reference(from_project, full: full) + def to_reference(from = nil, full: false) + project_reference = project.to_reference(from, full: full) if project_reference.present? project_reference + self.class.reference_prefix + self.id @@ -99,8 +99,8 @@ class CommitRange end end - def reference_link_text(from_project = nil) - project_reference = project.to_reference(from_project) + def reference_link_text(from = nil) + project_reference = project.to_reference(from) reference = ref_from + notation + ref_to if project_reference.present? diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 6b07dbdf3ea..c0263c0b4e2 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -17,6 +17,7 @@ class CommitStatus < ActiveRecord::Base validates :name, presence: true, unless: :importing? alias_attribute :author, :user + alias_attribute :pipeline_id, :commit_id scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) @@ -42,7 +43,8 @@ class CommitStatus < ActiveRecord::Base script_failure: 1, api_failure: 2, stuck_or_timeout_failure: 3, - runner_system_failure: 4 + runner_system_failure: 4, + missing_dependency_failure: 5 } ## @@ -103,26 +105,29 @@ class CommitStatus < ActiveRecord::Base end after_transition do |commit_status, transition| + next unless commit_status.project next if transition.loopback? commit_status.run_after_commit do - if pipeline + if pipeline_id if complete? || manual? - PipelineProcessWorker.perform_async(pipeline.id) + PipelineProcessWorker.perform_async(pipeline_id) else - PipelineUpdateWorker.perform_async(pipeline.id) + PipelineUpdateWorker.perform_async(pipeline_id) end end - StageUpdateWorker.perform_async(commit_status.stage_id) - ExpireJobCacheWorker.perform_async(commit_status.id) + StageUpdateWorker.perform_async(stage_id) + ExpireJobCacheWorker.perform_async(id) end end after_transition any => :failed do |commit_status| + next unless commit_status.project + commit_status.run_after_commit do MergeRequests::AddTodoWhenBuildFailsService - .new(pipeline.project, nil).execute(self) + .new(project, nil).execute(self) end end end diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb new file mode 100644 index 00000000000..0460439e9e6 --- /dev/null +++ b/app/models/concerns/artifact_migratable.rb @@ -0,0 +1,45 @@ +# Adapter class to unify the interface between mounted uploaders and the +# Ci::Artifact model +# Meant to be prepended so the interface can stay the same +module ArtifactMigratable + def artifacts_file + job_artifacts_archive&.file || legacy_artifacts_file + end + + def artifacts_metadata + job_artifacts_metadata&.file || legacy_artifacts_metadata + end + + def artifacts? + !artifacts_expired? && artifacts_file.exists? + end + + def artifacts_metadata? + artifacts? && artifacts_metadata.exists? + end + + def artifacts_file_changed? + job_artifacts_archive&.file_changed? || attribute_changed?(:artifacts_file) + end + + def remove_artifacts_file! + if job_artifacts_archive + job_artifacts_archive.destroy + else + remove_legacy_artifacts_file! + end + end + + def remove_artifacts_metadata! + if job_artifacts_metadata + job_artifacts_metadata.destroy + else + remove_legacy_artifacts_metadata! + end + end + + def artifacts_size + read_attribute(:artifacts_size).to_i + + job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i + end +end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 2ec70203710..10659030910 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -4,15 +4,26 @@ module Avatarable def avatar_path(only_path: true) return unless self[:avatar].present? - # If only_path is true then use the relative path of avatar. - # Otherwise use full path (including host). asset_host = ActionController::Base.asset_host - gitlab_host = only_path ? gitlab_config.relative_url_root : gitlab_config.url + use_asset_host = asset_host.present? - # If asset_host is set then it is expected that assets are handled by a standalone host. - # That means we do not want to get GitLab's relative_url_root option anymore. - host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host + # Avatars for private and internal groups and projects require authentication to be viewed, + # which means they can only be served by Rails, on the regular GitLab host. + # If an asset host is configured, we need to return the fully qualified URL + # instead of only the avatar path, so that Rails doesn't prefix it with the asset host. + if use_asset_host && respond_to?(:public?) && !public? + use_asset_host = false + only_path = false + end - [host, avatar.url].join + url_base = "" + if use_asset_host + url_base << asset_host unless only_path + else + url_base << gitlab_config.base_url unless only_path + url_base << gitlab_config.relative_url_root + end + + url_base + avatar.url end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index 9adc309a22b..d8394415362 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -98,6 +98,7 @@ module Awardable def create_award_emoji(name, current_user) return unless emoji_awardable? + award_emoji.create(name: normalize_name(name), user: current_user) end diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb new file mode 100644 index 00000000000..984c4f53bf7 --- /dev/null +++ b/app/models/concerns/bulk_member_access_load.rb @@ -0,0 +1,46 @@ +# Returns and caches in thread max member access for a resource +# +module BulkMemberAccessLoad + extend ActiveSupport::Concern + + included do + # Determine the maximum access level for a group of resources in bulk. + # + # Returns a Hash mapping resource ID -> maximum access level. + def max_member_access_for_resource_ids(resource_klass, resource_ids, memoization_index = self.id, &block) + raise 'Block is mandatory' unless block_given? + + resource_ids = resource_ids.uniq + key = max_member_access_for_resource_key(resource_klass, memoization_index) + access = {} + + if RequestStore.active? + RequestStore.store[key] ||= {} + access = RequestStore.store[key] + end + + # Look up only the IDs we need + resource_ids = resource_ids - access.keys + + return access if resource_ids.empty? + + resource_access = yield(resource_ids) + + access.merge!(resource_access) + + missing_resource_ids = resource_ids - resource_access.keys + + missing_resource_ids.each do |resource_id| + access[resource_id] = Gitlab::Access::NO_ACCESS + end + + access + end + + private + + def max_member_access_for_resource_key(klass, memoization_index) + "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}" + end + end +end diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb index f5cbb3becad..4b4d519f3df 100644 --- a/app/models/concerns/discussion_on_diff.rb +++ b/app/models/concerns/discussion_on_diff.rb @@ -32,6 +32,10 @@ module DiscussionOnDiff first_note.position.new_path end + def on_merge_request_commit? + for_merge_request? && commit_id.present? + end + # Returns an array of at most 16 highlighted lines above a diff note def truncated_diff_lines(highlight: true) lines = highlight ? highlighted_diff_lines : diff_lines diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb index 9585b5583dc..8a241e4374a 100644 --- a/app/models/concerns/has_variable.rb +++ b/app/models/concerns/has_variable.rb @@ -16,6 +16,10 @@ module HasVariable key: Gitlab::Application.secrets.db_key_base, algorithm: 'aes-256-cbc' + def key=(new_key) + super(new_key.to_s.strip) + end + def to_runner_variable { key: key, value: value, public: false } end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index c008fb91a16..5ca4a7086cb 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -122,9 +122,7 @@ module Issuable # # Returns an ActiveRecord::Relation. def search(query) - title = to_fuzzy_arel(:title, query) - - where(title) + fuzzy_search(query, [:title]) end # Searches for records with a matching title or description. @@ -135,10 +133,7 @@ module Issuable # # Returns an ActiveRecord::Relation. def full_search(query) - title = to_fuzzy_arel(:title, query) - description = to_fuzzy_arel(:description, query) - - where(title&.or(description)) + fuzzy_search(query, [:title, :description]) end def sort(method, excluded_labels: []) @@ -255,8 +250,10 @@ module Issuable participants(user).include?(user) end - def to_hook_data(user, old_labels: [], old_assignees: []) + def to_hook_data(user, old_associations: {}) changes = previous_changes + old_labels = old_associations.fetch(:labels, []) + old_assignees = old_associations.fetch(:assignees, []) if old_labels != labels changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)] @@ -270,6 +267,14 @@ module Issuable end end + if self.respond_to?(:total_time_spent) + old_total_time_spent = old_associations.fetch(:total_time_spent, nil) + + if old_total_time_spent != total_time_spent + changes[:total_time_spent] = [old_total_time_spent, total_time_spent] + end + end + Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes) end @@ -341,4 +346,11 @@ module Issuable def first_contribution? false end + + ## + # Overriden in MergeRequest + # + def wipless_title_changed(old_title) + old_title != title + end end diff --git a/app/models/concerns/manual_inverse_association.rb b/app/models/concerns/manual_inverse_association.rb new file mode 100644 index 00000000000..0fca8feaf89 --- /dev/null +++ b/app/models/concerns/manual_inverse_association.rb @@ -0,0 +1,17 @@ +module ManualInverseAssociation + extend ActiveSupport::Concern + + module ClassMethods + def manual_inverse_association(association, inverse) + define_method(association) do |*args| + super(*args).tap do |value| + next unless value + + child_association = value.association(inverse) + child_association.set_inverse_instance(self) + child_association.target = self + end + end + end + end +end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 1db6b2d2fa2..b43eaeaeea0 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -31,11 +31,11 @@ module Mentionable # # By default this will be the class name and the result of calling # `to_reference` on the object. - def gfm_reference(from_project = nil) + def gfm_reference(from = nil) # "MergeRequest" > "merge_request" > "Merge request" > "merge request" friendly_name = self.class.to_s.underscore.humanize.downcase - "#{friendly_name} #{to_reference(from_project)}" + "#{friendly_name} #{to_reference(from)}" end # The GFM reference to this Mentionable, which shouldn't be included in its #references. diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 710fc1ed647..7026f565706 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -86,6 +86,14 @@ module Milestoneish false end + def total_issue_time_spent + @total_issue_time_spent ||= issues.joins(:timelogs).sum(:time_spent) + end + + def human_total_issue_time_spent + Gitlab::TimeTrackingFormatter.output(total_issue_time_spent) + end + private def count_issues_by_state(user) diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index fde1cc44afa..e62f42e8e70 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -1,12 +1,6 @@ module ProtectedBranchAccess extend ActiveSupport::Concern - ALLOWED_ACCESS_LEVELS ||= [ - Gitlab::Access::MASTER, - Gitlab::Access::DEVELOPER, - Gitlab::Access::NO_ACCESS - ].freeze - included do include ProtectedRefAccess @@ -14,18 +8,6 @@ module ProtectedBranchAccess delegate :project, to: :protected_branch - validates :access_level, presence: true, inclusion: { - in: ALLOWED_ACCESS_LEVELS - } - - def self.human_access_levels - { - Gitlab::Access::MASTER => "Masters", - Gitlab::Access::DEVELOPER => "Developers + Masters", - Gitlab::Access::NO_ACCESS => "No one" - }.with_indifferent_access - end - def check_access(user) return false if access_level == Gitlab::Access::NO_ACCESS diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index c4f158e569a..80c9f7d4eb4 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -1,13 +1,35 @@ module ProtectedRefAccess extend ActiveSupport::Concern + ALLOWED_ACCESS_LEVELS = [ + Gitlab::Access::MASTER, + Gitlab::Access::DEVELOPER, + Gitlab::Access::NO_ACCESS + ].freeze + + HUMAN_ACCESS_LEVELS = { + Gitlab::Access::MASTER => "Masters".freeze, + Gitlab::Access::DEVELOPER => "Developers + Masters".freeze, + Gitlab::Access::NO_ACCESS => "No one".freeze + }.freeze + included do scope :master, -> { where(access_level: Gitlab::Access::MASTER) } scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } + + validates :access_level, presence: true, if: :role?, inclusion: { + in: ALLOWED_ACCESS_LEVELS + } end def humanize - self.class.human_access_levels[self.access_level] + HUMAN_ACCESS_LEVELS[self.access_level] + end + + # CE access levels are always role-based, + # where as EE allows groups and users too + def role? + true end def check_access(user) diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb index 78ac4f324e7..b782e85717e 100644 --- a/app/models/concerns/referable.rb +++ b/app/models/concerns/referable.rb @@ -7,7 +7,7 @@ module Referable # Returns the String necessary to reference this object in Markdown # - # from_project - Refering Project object + # from - Referring parent object # # This should be overridden by the including class. # @@ -17,12 +17,12 @@ module Referable # Issue.last.to_reference(other_project) # => "cross-project#1" # # Returns a String - def to_reference(_from_project = nil, full:) + def to_reference(_from = nil, full:) '' end - def reference_link_text(from_project = nil) - to_reference(from_project) + def reference_link_text(from = nil) + to_reference(from) end included do diff --git a/app/models/concerns/throttled_touch.rb b/app/models/concerns/throttled_touch.rb new file mode 100644 index 00000000000..ad0ff0f20d4 --- /dev/null +++ b/app/models/concerns/throttled_touch.rb @@ -0,0 +1,10 @@ +# ThrottledTouch can be used to throttle the number of updates triggered by +# calling "touch" on an ActiveRecord model. +module ThrottledTouch + # The amount of time to wait before "touch" can update a record again. + TOUCH_INTERVAL = 1.minute + + def touch(*args) + super if (Time.zone.now - updated_at) > TOUCH_INTERVAL + end +end diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb index 6eba87da1a1..4a65738214b 100644 --- a/app/models/diff_discussion.rb +++ b/app/models/diff_discussion.rb @@ -24,7 +24,11 @@ class DiffDiscussion < Discussion return unless for_merge_request? return {} if active? - noteable.version_params_for(position.diff_refs) + if on_merge_request_commit? + { commit_id: commit_id } + else + noteable.version_params_for(position.diff_refs) + end end def reply_attributes diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index d88a92dc027..b53d44cda95 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -17,8 +17,10 @@ class DiffNote < Note validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete validate :verify_supported + validate :diff_refs_match_commit, if: :for_commit? - before_validation :set_original_position, :update_position, on: :create + before_validation :set_original_position, on: :create + before_validation :update_position, on: :create, if: :on_text? before_validation :set_line_code after_save :keep_around_commits @@ -134,6 +136,12 @@ class DiffNote < Note errors.add(:position, "is invalid") end + def diff_refs_match_commit + return if self.original_position.diff_refs == self.commit.diff_refs + + errors.add(:commit_id, 'does not match the diff refs') + end + def keep_around_commits project.repository.keep_around(self.original_position.base_sha) project.repository.keep_around(self.original_position.start_sha) diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 437df923d2d..92482a1a875 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -11,6 +11,7 @@ class Discussion :author, :noteable, + :commit_id, :for_commit?, :for_merge_request?, diff --git a/app/models/email.rb b/app/models/email.rb index 2da8b050149..d6516761f0a 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -1,5 +1,6 @@ class Email < ActiveRecord::Base include Sortable + include Gitlab::SQL::Pattern belongs_to :user diff --git a/app/models/environment.rb b/app/models/environment.rb index 21a028e351c..bf69b4c50f0 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -138,11 +138,11 @@ class Environment < ActiveRecord::Base end def has_terminals? - project.deployment_service.present? && available? && last_deployment.present? + project.deployment_platform.present? && available? && last_deployment.present? end def terminals - project.deployment_service.terminals(self) if has_terminals? + project.deployment_platform.terminals(self) if has_terminals? end def has_metrics? diff --git a/app/models/epic.rb b/app/models/epic.rb index 62898a02e2d..286b855de3f 100644 --- a/app/models/epic.rb +++ b/app/models/epic.rb @@ -1,7 +1,11 @@ # Placeholder class for model that is implemented in EE -# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE +# It reserves '&' as a reference prefix, but the table does not exists in CE class Epic < ActiveRecord::Base - # TODO: this will be implemented as part of #3853 - def to_reference + def self.reference_prefix + '&' + end + + def self.reference_prefix_escaped + '&' end end diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb index 0bf18e529f0..2aaba2e4c90 100644 --- a/app/models/external_issue.rb +++ b/app/models/external_issue.rb @@ -38,13 +38,17 @@ class ExternalIssue @project.id end - def to_reference(_from_project = nil, full: nil) + def to_reference(_from = nil, full: nil) id end - def reference_link_text(from_project = nil) + def reference_link_text(from = nil) return "##{id}" if id =~ /^\d+$/ id end + + def notes + Note.none + end end diff --git a/app/models/fork_network_member.rb b/app/models/fork_network_member.rb index 6a9b52a1ef8..eb9417dc34f 100644 --- a/app/models/fork_network_member.rb +++ b/app/models/fork_network_member.rb @@ -4,4 +4,14 @@ class ForkNetworkMember < ActiveRecord::Base belongs_to :forked_from_project, class_name: 'Project' validates :fork_network, :project, presence: true + + after_destroy :cleanup_fork_network + + private + + def cleanup_fork_network + # Explicitly using `#count` makes sure we have the correct number if the + # relation was loaded in the fork_network. + fork_network.destroy if fork_network.fork_network_members.count == 0 + end end diff --git a/app/models/group.rb b/app/models/group.rb index 8cf632fb566..fddace03387 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper + include AfterCommitQueue include AccessRequestable include Avatarable include Referable @@ -50,20 +51,6 @@ class Group < Namespace Gitlab::Database.postgresql? end - # Searches for groups matching the given query. - # - # This method uses ILIKE on PostgreSQL and LIKE on MySQL. - # - # query - The search query as a String - # - # Returns an ActiveRecord::Relation. - def search(query) - table = Namespace.arel_table - pattern = "%#{query}%" - - where(table[:name].matches(pattern).or(table[:path].matches(pattern))) - end - def sort(method) if method == 'storage_size_desc' # storage_size is a virtual column so we need to @@ -97,7 +84,7 @@ class Group < Namespace end end - def to_reference(_from_project = nil, full: nil) + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -303,6 +290,18 @@ class Group < Namespace "#{parent.full_path}/#{path_was}" end + def group_member(user) + if group_members.loaded? + group_members.find { |gm| gm.user_id == user.id } + else + group_members.find_by(user_id: user) + end + end + + def hashed_storage?(_feature) + false + end + private def update_two_factor_requirement diff --git a/app/models/identity.rb b/app/models/identity.rb index ac8094b610e..ff811e19f8a 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -1,18 +1,27 @@ class Identity < ActiveRecord::Base include Sortable include CaseSensitivity + belongs_to :user validates :provider, presence: true - validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider } + validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider, case_sensitive: false } validates :user_id, uniqueness: { scope: :provider } + scope :with_provider, ->(provider) { where(provider: provider) } scope :with_extern_uid, ->(provider, extern_uid) do - extern_uid = Gitlab::LDAP::Person.normalize_dn(extern_uid) if provider.starts_with?('ldap') - where(extern_uid: extern_uid, provider: provider) + iwhere(extern_uid: normalize_uid(provider, extern_uid)).with_provider(provider) end def ldap? provider.starts_with?('ldap') end + + def self.normalize_uid(provider, uid) + if provider.to_s.starts_with?('ldap') + Gitlab::LDAP::Person.normalize_dn(uid) + else + uid.to_s + end + end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 3b3c7fb7f8b..33db197e612 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -9,6 +9,7 @@ class Issue < ActiveRecord::Base include FasterCacheKeys include RelativePositioning include TimeTrackable + include ThrottledTouch DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze @@ -49,7 +50,6 @@ class Issue < ActiveRecord::Base scope :public_only, -> { where(confidential: false) } after_save :expire_etag_cache - after_commit :update_project_counter_caches, on: :destroy attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -246,7 +246,12 @@ class Issue < ActiveRecord::Base def as_json(options = {}) super(options).tap do |json| - json[:subscribed] = subscribed?(options[:user], project) if options.key?(:user) && options[:user] + if options.key?(:sidebar_endpoints) && project + url_helper = Gitlab::Routing.url_helpers + + json.merge!(issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'), + toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self)) + end if options.key?(:labels) json[:labels] = labels.as_json( @@ -262,10 +267,6 @@ class Issue < ActiveRecord::Base true end - def update_project_counter_caches? - state_changed? || confidential_changed? - end - def update_project_counter_caches Projects::OpenIssuesCountService.new(project).refresh_cache end diff --git a/app/models/key.rb b/app/models/key.rb index f119b15c737..a3f8a5d6dc7 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -2,6 +2,7 @@ require 'digest/md5' class Key < ActiveRecord::Base include Gitlab::CurrentSettings + include AfterCommitQueue include Sortable belongs_to :user @@ -27,8 +28,10 @@ class Key < ActiveRecord::Base after_commit :add_to_shell, on: :create after_create :post_create_hook + after_create :refresh_user_cache after_commit :remove_from_shell, on: :destroy after_destroy :post_destroy_hook + after_destroy :refresh_user_cache def key=(value) value&.delete!("\n\r") @@ -76,6 +79,12 @@ class Key < ActiveRecord::Base ) end + def refresh_user_cache + return unless user + + Users::KeysCountService.new(user).refresh_cache + end + def post_destroy_hook SystemHooksService.new.execute_hooks_for(self, :destroy) end diff --git a/app/models/label.rb b/app/models/label.rb index 899028a01a0..b5bfa6ea2dd 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -165,12 +165,12 @@ class Label < ActiveRecord::Base # # Returns a String # - def to_reference(from_project = nil, target_project: nil, format: :id, full: false) + def to_reference(from = nil, target_project: nil, format: :id, full: false) format_reference = label_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" - if from_project - "#{from_project.to_reference(target_project, full: full)}#{reference}" + if from + "#{from.to_reference(target_project, full: full)}#{reference}" else reference end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 0056b0f80f4..38b8c41024a 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -8,16 +8,8 @@ class LfsObject < ActiveRecord::Base mount_uploader :file, LfsObjectUploader - def storage_project(project) - if project && project.forked? - storage_project(project.forked_from_project) - else - project - end - end - def project_allowed_access?(project) - projects.exists?(storage_project(project).id) + projects.exists?(project.lfs_storage_project.id) end def self.destroy_unreferenced diff --git a/app/models/member.rb b/app/models/member.rb index cbbd58f2eaf..2fe5fda985f 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,4 +1,5 @@ class Member < ActiveRecord::Base + include AfterCommitQueue include Sortable include Importable include Expirable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 82d0ae90d77..422f138c4ea 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -5,6 +5,9 @@ class MergeRequest < ActiveRecord::Base include Referable include IgnorableColumn include TimeTrackable + include ManualInverseAssociation + include EachBatch + include ThrottledTouch ignore_column :locked_at, :ref_fetched @@ -14,9 +17,28 @@ class MergeRequest < ActiveRecord::Base belongs_to :merge_user, class_name: "User" has_many :merge_request_diffs + has_one :merge_request_diff, -> { order('merge_request_diffs.id DESC') }, inverse_of: :merge_request + belongs_to :latest_merge_request_diff, class_name: 'MergeRequestDiff' + manual_inverse_association :latest_merge_request_diff, :merge_request + + # This is the same as latest_merge_request_diff unless: + # 1. There are arguments - in which case we might be trying to force-reload. + # 2. This association is already loaded. + # 3. The latest diff does not exist. + # + # The second one in particular is important - MergeRequestDiff#merge_request + # is the inverse of MergeRequest#merge_request_diff, which means it may not be + # the latest diff, because we could have loaded any diff from this particular + # MR. If we haven't already loaded a diff, then it's fine to load the latest. + def merge_request_diff(*args) + fallback = latest_merge_request_diff if args.empty? && !association(:merge_request_diff).loaded? + + fallback || super + end + belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -31,7 +53,6 @@ class MergeRequest < ActiveRecord::Base after_create :ensure_merge_request_diff, unless: :importing? after_update :reload_diff_if_branch_changed - after_commit :update_project_counter_caches, on: :destroy # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests @@ -125,6 +146,13 @@ class MergeRequest < ActiveRecord::Base '!' end + # Use this method whenever you need to make sure the head_pipeline is synced with the + # branch head commit, for example checking if a merge request can be merged. + # For more information check: https://gitlab.com/gitlab-org/gitlab-ce/issues/40004 + def actual_head_pipeline + head_pipeline&.sha == diff_head_sha ? head_pipeline : nil + end + # Pattern used to extract `!123` merge request references from text # # This pattern supports cross-project references. @@ -167,6 +195,22 @@ class MergeRequest < ActiveRecord::Base where("merge_requests.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end + # This is used after project import, to reset the IDs to the correct + # values. It is not intended to be called without having already scoped the + # relation. + def self.set_latest_merge_request_diff_ids! + update = ' + latest_merge_request_diff_id = ( + SELECT MAX(id) + FROM merge_request_diffs + WHERE merge_requests.id = merge_request_diffs.merge_request_id + )'.squish + + self.each_batch do |batch| + batch.update_all(update) + end + end + WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze def self.work_in_progress?(title) @@ -181,6 +225,12 @@ class MergeRequest < ActiveRecord::Base work_in_progress?(title) ? title : "WIP: #{title}" end + # Verifies if title has changed not taking into account WIP prefix + # for merge requests. + def wipless_title_changed(old_title) + self.class.wipless_title(old_title) != self.wipless_title + end + def hook_attrs Gitlab::HookData::MergeRequestBuilder.new(self).build end @@ -241,9 +291,9 @@ class MergeRequest < ActiveRecord::Base if persisted? merge_request_diff.commit_shas elsif compare_commits - compare_commits.reverse.map(&:sha) + compare_commits.to_a.reverse.map(&:sha) else - [] + Array(diff_head_sha) end end @@ -322,16 +372,28 @@ class MergeRequest < ActiveRecord::Base # We use these attributes to force these to the intended values. attr_writer :target_branch_sha, :source_branch_sha + def source_branch_ref + return @source_branch_sha if @source_branch_sha + return unless source_branch + + Gitlab::Git::BRANCH_REF_PREFIX + source_branch + end + + def target_branch_ref + return @target_branch_sha if @target_branch_sha + return unless target_branch + + Gitlab::Git::BRANCH_REF_PREFIX + target_branch + end + def source_branch_head return unless source_project - source_branch_ref = @source_branch_sha || source_branch source_project.repository.commit(source_branch_ref) if source_branch_ref end def target_branch_head - target_branch_ref = @target_branch_sha || target_branch - target_project.repository.commit(target_branch_ref) if target_branch_ref + target_project.repository.commit(target_branch_ref) end def branch_merge_base_commit @@ -440,7 +502,7 @@ class MergeRequest < ActiveRecord::Base def merge_request_diff_for(diff_refs_or_sha) @merge_request_diffs_by_diff_refs_or_sha ||= Hash.new do |h, diff_refs_or_sha| - diffs = merge_request_diffs.viewable.select_without_diff + diffs = merge_request_diffs.viewable h[diff_refs_or_sha] = if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) diffs.find_by_diff_refs(diff_refs_or_sha) @@ -576,7 +638,7 @@ class MergeRequest < ActiveRecord::Base commit_notes = Note .except(:order) .where(project_id: [source_project_id, target_project_id]) - .where(noteable_type: 'Commit', commit_id: commit_ids) + .for_commit_id(commit_ids) # We're using a UNION ALL here since this results in better performance # compared to using OR statements. We're using UNION ALL since the queries @@ -587,6 +649,7 @@ class MergeRequest < ActiveRecord::Base .to_sql Note.from("(#{union}) #{Note.table_name}") + .includes(:noteable) end alias_method :discussion_notes, :related_notes @@ -768,8 +831,9 @@ class MergeRequest < ActiveRecord::Base def mergeable_ci_state? return true unless project.only_allow_merge_if_pipeline_succeeds? + return true unless head_pipeline - !head_pipeline || head_pipeline.success? || head_pipeline.skipped? + actual_head_pipeline&.success? || actual_head_pipeline&.skipped? end def environments_for(current_user) @@ -845,7 +909,8 @@ class MergeRequest < ActiveRecord::Base def compute_diverged_commits_count return 0 unless source_branch_sha && target_branch_sha - Gitlab::Git::Commit.between(target_project.repository.raw_repository, source_branch_sha, target_branch_sha).size + target_project.repository + .count_commits_between(source_branch_sha, target_branch_sha) end private :compute_diverged_commits_count @@ -861,18 +926,26 @@ class MergeRequest < ActiveRecord::Base .order(id: :desc) end + def all_commits + # MySQL doesn't support LIMIT in a subquery. + diffs_relation = if Gitlab::Database.postgresql? + merge_request_diffs.recent + else + merge_request_diffs + end + + MergeRequestDiffCommit + .where(merge_request_diff: diffs_relation) + .limit(10_000) + end + # Note that this could also return SHA from now dangling commits # def all_commit_shas - if persisted? - column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).limit(10_000).pluck('sha') - serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas) + @all_commit_shas ||= begin + return commit_shas unless persisted? - (column_shas + serialised_shas).uniq - elsif compare_commits - compare_commits.to_a.reverse.map(&:id) - else - [diff_head_sha] + all_commits.pluck(:sha).uniq end end @@ -940,16 +1013,12 @@ class MergeRequest < ActiveRecord::Base return true if autocomplete_precheck return false unless mergeable?(skip_ci_check: true) - return false if head_pipeline && !(head_pipeline.success? || head_pipeline.active?) + return false if actual_head_pipeline && !(actual_head_pipeline.success? || actual_head_pipeline.active?) return false if last_diff_sha != diff_head_sha true end - def update_project_counter_caches? - state_changed? - end - def update_project_counter_caches Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 1eda0f9cbbd..c37aa0a594b 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -1,21 +1,21 @@ class MergeRequestDiff < ActiveRecord::Base include Sortable include Importable - include Gitlab::EncodingHelper + include ManualInverseAssociation + include IgnorableColumn - # Prevent store of diff if commits amount more then 500 + # Don't display more than 100 commits at once COMMITS_SAFE_SIZE = 100 - # Valid types of serialized diffs allowed by Gitlab::Git::Diff - VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta].freeze + ignore_column :st_commits, + :st_diffs belongs_to :merge_request + manual_inverse_association :merge_request, :merge_request_diff + has_many :merge_request_diff_files, -> { order(:merge_request_diff_id, :relative_order) } has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) } - serialize :st_commits # rubocop:disable Cop/ActiveRecordSerialize - serialize :st_diffs # rubocop:disable Cop/ActiveRecordSerialize - state_machine :state, initial: :empty do state :collected state :overflow @@ -29,6 +29,8 @@ class MergeRequestDiff < ActiveRecord::Base scope :viewable, -> { without_state(:empty) } + scope :recent, -> { order(id: :desc).limit(100) } + # All diff information is collected from repository after object is created. # It allows you to override variables like head_commit_sha before getting diff. after_create :save_git_content, unless: :importing? @@ -37,14 +39,6 @@ class MergeRequestDiff < ActiveRecord::Base find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) end - def self.select_without_diff - select(column_names - ['st_diffs']) - end - - def st_commits - super || [] - end - # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content @@ -126,11 +120,7 @@ class MergeRequestDiff < ActiveRecord::Base end def commit_shas - if st_commits.present? - st_commits.map { |commit| commit[:id] } - else - merge_request_diff_commits.map(&:sha) - end + merge_request_diff_commits.map(&:sha) end def diff_refs=(new_diff_refs) @@ -194,7 +184,7 @@ class MergeRequestDiff < ActiveRecord::Base end def latest? - self == merge_request.merge_request_diff + self.id == merge_request.latest_merge_request_diff_id end def compare_with(sha) @@ -205,34 +195,11 @@ class MergeRequestDiff < ActiveRecord::Base end def commits_count - if st_commits.present? - st_commits.size - else - merge_request_diff_commits.size - end - end - - def utf8_st_diffs - return [] if st_diffs.blank? - - st_diffs.map do |diff| - diff.each do |k, v| - diff[k] = encode_utf8(v) if v.respond_to?(:encoding) - end - end + merge_request_diff_commits.size end private - # Old GitLab implementations may have generated diffs as ["--broken-diff"]. - # Avoid an error 500 by ignoring bad elements. See: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776 - def valid_raw_diff?(raw) - return false unless raw.respond_to?(:each) - - raw.any? { |element| VALID_CLASSES.include?(element.class) } - end - def create_merge_request_diff_files(diffs) rows = diffs.map.with_index do |diff, index| diff_hash = diff.to_hash.merge( @@ -256,9 +223,7 @@ class MergeRequestDiff < ActiveRecord::Base end def load_diffs(options) - return Gitlab::Git::DiffCollection.new([]) unless diffs_from_database - - raw = diffs_from_database + raw = merge_request_diff_files.map(&:to_hash) if paths = options[:paths] raw = raw.select do |diff| @@ -269,23 +234,11 @@ class MergeRequestDiff < ActiveRecord::Base Gitlab::Git::DiffCollection.new(raw, options) end - def diffs_from_database - return @diffs_from_database if defined?(@diffs_from_database) - - @diffs_from_database = - if st_diffs.present? - if valid_raw_diff?(st_diffs) - st_diffs - end - elsif merge_request_diff_files.present? - merge_request_diff_files.map(&:to_hash) - end - end - def load_commits - commits = st_commits.presence || merge_request_diff_commits + commits = merge_request_diff_commits.map { |commit| Commit.from_hash(commit.to_hash, project) } - commits.map { |commit| Commit.from_hash(commit.to_hash, project) } + CommitCollection + .new(merge_request.source_project, commits, merge_request.source_branch) end def save_diffs diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 47e6b785c39..77c19380e66 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -13,6 +13,7 @@ class Milestone < ActiveRecord::Base include Referable include StripAttribute include Milestoneish + include Gitlab::SQL::Pattern cache_markdown_field :title, pipeline: :single_line cache_markdown_field :description @@ -73,10 +74,7 @@ class Milestone < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:title].matches(pattern).or(t[:description].matches(pattern))) + fuzzy_search(query, [:title, :description]) end def filter_by_state(milestones, state) @@ -86,6 +84,13 @@ class Milestone < ActiveRecord::Base else milestones.active end end + + def predefined?(milestone) + milestone == Any || + milestone == None || + milestone == Upcoming || + milestone == Started + end end def self.reference_prefix @@ -162,18 +167,18 @@ class Milestone < ActiveRecord::Base # Milestone.first.to_reference(cross_namespace_project) # => "gitlab-org/gitlab-ce%1" # Milestone.first.to_reference(same_namespace_project) # => "gitlab-ce%1" # - def to_reference(from_project = nil, format: :name, full: false) + def to_reference(from = nil, format: :name, full: false) format_reference = milestone_format_reference(format) reference = "#{self.class.reference_prefix}#{format_reference}" if project - "#{project.to_reference(from_project, full: full)}#{reference}" + "#{project.to_reference(from, full: full)}#{reference}" else reference end end - def reference_link_text(from_project = nil) + def reference_link_text(from = nil) self.title end @@ -256,7 +261,7 @@ class Milestone < ActiveRecord::Base def start_date_should_be_less_than_due_date if due_date <= start_date - errors.add(:start_date, "Can't be greater than due date") + errors.add(:due_date, "must be greater than start date") end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 4d401e7ba18..901dbf2ba69 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base include Routable include AfterCommitQueue include Storage::LegacyNamespace + include Gitlab::SQL::Pattern # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of @@ -86,10 +87,7 @@ class Namespace < ActiveRecord::Base # # Returns an ActiveRecord::Relation def search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:name].matches(pattern).or(t[:path].matches(pattern))) + fuzzy_search(query, [:name, :path]) end def clean_path(path) @@ -141,7 +139,17 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) return nil unless project.fork_network - project.fork_network.find_forks_in(projects).first + if RequestStore.active? + forks_in_namespace = RequestStore.fetch("namespaces:#{id}:forked_projects") do + Hash.new do |found_forks, project| + found_forks[project] = project.fork_network.find_forks_in(projects).first + end + end + + forks_in_namespace[project] + else + project.fork_network.find_forks_in(projects).first + end end def lfs_enabled? diff --git a/app/models/note.rb b/app/models/note.rb index f9676361072..c4c2ab8e67d 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -14,6 +14,8 @@ class Note < ActiveRecord::Base include ResolvableNote include IgnorableColumn include Editable + include Gitlab::SQL::Pattern + include ThrottledTouch module SpecialRole FIRST_TIME_CONTRIBUTOR = :first_time_contributor @@ -54,7 +56,7 @@ class Note < ActiveRecord::Base participant :author belongs_to :project - belongs_to :noteable, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :author, class_name: "User" belongs_to :updated_by, class_name: "User" belongs_to :last_edited_by, class_name: 'User' @@ -110,12 +112,14 @@ class Note < ActiveRecord::Base includes(:author, :noteable, :updated_by, project: [:project_members, { group: [:group_members] }]) end + scope :with_metadata, -> { includes(:system_note_metadata) } after_initialize :ensure_discussion_id before_validation :nullify_blank_type, :nullify_blank_line_code before_validation :set_discussion_id, on: :create after_save :keep_around_commit, if: :for_project_noteable? after_save :expire_etag_cache + after_save :touch_noteable after_destroy :expire_etag_cache class << self @@ -166,10 +170,20 @@ class Note < ActiveRecord::Base def has_special_role?(role, note) note.special_role == role end + + def search(query) + fuzzy_search(query, [:note]) + end end def cross_reference? - system? && matches_cross_reference_regex? + return unless system? + + if force_cross_reference_regex_check? + matches_cross_reference_regex? + else + SystemNoteService.cross_reference?(note) + end end def diff_note? @@ -216,16 +230,18 @@ class Note < ActiveRecord::Base for_personal_snippet? end + def commit + @commit ||= project.commit(commit_id) if commit_id.present? + end + # override to return commits, which are not active record def noteable - if for_commit? - @commit ||= project.commit(commit_id) - else - super - end - # Temp fix to prevent app crash - # if note commit id doesn't exist + return commit if for_commit? + + super rescue + # Temp fix to prevent app crash + # if note commit id doesn't exist nil end @@ -355,6 +371,42 @@ class Note < ActiveRecord::Base Gitlab::EtagCaching::Store.new.touch(key) end + def touch(*args) + # We're not using an explicit transaction here because this would in all + # cases result in all future queries going to the primary, even if no writes + # are performed. + # + # We touch the noteable first so its SELECT query can run before our writes, + # ensuring it runs on a secondary (if no prior write took place). + touch_noteable + super + end + + # By default Rails will issue an "SELECT *" for the relation, which is + # overkill for just updating the timestamps. To work around this we manually + # touch the data so we can SELECT only the columns we need. + def touch_noteable + # Commits are not stored in the DB so we can't touch them. + return if for_commit? + + assoc = association(:noteable) + + noteable_object = + if assoc.loaded? + noteable + else + # If the object is not loaded (e.g. when notes are loaded async) we + # _only_ want the data we actually need. + assoc.scope.select(:id, :updated_at).take + end + + noteable_object&.touch + end + + def banzai_render_context(field) + super.merge(noteable: noteable) + end + private def keep_around_commit @@ -382,4 +434,10 @@ class Note < ActiveRecord::Base def set_discussion_id self.discussion_id ||= discussion_class.discussion_id(self) end + + def force_cross_reference_regex_check? + return unless system? + + SystemNoteMetadata::TYPES_WITH_CROSS_REFERENCES.include?(system_note_metadata&.action) + end end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 2e824cda525..8de42ff9d2e 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -65,12 +65,18 @@ class PagesDomain < ActiveRecord::Base def expired? return false unless x509 + current = Time.new current < x509.not_before || x509.not_after < current end + def expiration + x509&.not_after + end + def subject return unless x509 + x509.subject.to_s end @@ -98,6 +104,7 @@ class PagesDomain < ActiveRecord::Base def validate_pages_domain return unless domain + if domain.downcase.ends_with?(Settings.pages.host.downcase) self.errors.add(:domain, "*.#{Settings.pages.host} is restricted") end @@ -105,6 +112,7 @@ class PagesDomain < ActiveRecord::Base def x509 return unless certificate + @x509 ||= OpenSSL::X509::Certificate.new(certificate) rescue OpenSSL::X509::CertificateError nil @@ -112,6 +120,7 @@ class PagesDomain < ActiveRecord::Base def pkey return unless key + @pkey ||= OpenSSL::PKey::RSA.new(key) rescue OpenSSL::PKey::PKeyError, OpenSSL::Cipher::CipherError nil diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index cfcb03138b7..063dc521324 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -3,6 +3,8 @@ class PersonalAccessToken < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :token + REDIS_EXPIRY_TIME = 3.minutes + serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize belongs_to :user @@ -27,6 +29,21 @@ class PersonalAccessToken < ActiveRecord::Base !revoked? && !expired? end + def self.redis_getdel(user_id) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key(user_id)) + redis.del(redis_shared_state_key(user_id)) + token + end + end + + def self.redis_store!(user_id, token) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(user_id), token, ex: REDIS_EXPIRY_TIME) + token + end + end + protected def validate_scopes @@ -38,4 +55,8 @@ class PersonalAccessToken < ActiveRecord::Base def set_default_scopes self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty? end + + def self.redis_shared_state_key(user_id) + "gitlab:personal_access_token:#{user_id}" + end end diff --git a/app/models/project.rb b/app/models/project.rb index 53df29dab02..6ae15a0a50f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -18,6 +18,7 @@ class Project < ActiveRecord::Base include SelectForProjectAuthorization include Routable include GroupDescendant + include Gitlab::SQL::Pattern extend Gitlab::ConfigHelper extend Gitlab::CurrentSettings @@ -188,7 +189,6 @@ class Project < ActiveRecord::Base has_one :statistics, class_name: 'ProjectStatistics' has_one :cluster_project, class_name: 'Clusters::Project' - has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster' has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, @@ -227,14 +227,13 @@ class Project < ActiveRecord::Base delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team - delegate :empty_repo?, to: :repository # Validations validates :creator, presence: true, on: :create validates :description, length: { maximum: 2000 }, allow_blank: true validates :ci_config_path, - format: { without: /\.{2}/, - message: 'cannot include directory traversal.' }, + format: { without: /(\.{2}|\A\/)/, + message: 'cannot include leading slash or directory traversal.' }, length: { maximum: 255 }, allow_blank: true validates :name, @@ -272,8 +271,9 @@ class Project < ActiveRecord::Base scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } - scope :with_hashed_storage, -> { where('storage_version >= 1') } - scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) } + scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) } + scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) } + scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) } scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) } scope :sorted_by_stars, -> { reorder('projects.star_count DESC') } @@ -365,6 +365,7 @@ class Project < ActiveRecord::Base scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } scope :excluding_project, ->(project) { where.not(id: project) } + scope :import_started, -> { where(import_status: 'started') } state_machine :import_status, initial: :none do event :import_schedule do @@ -423,32 +424,11 @@ class Project < ActiveRecord::Base # # query - The search query as a String. def search(query) - ptable = arel_table - ntable = Namespace.arel_table - pattern = "%#{query}%" - - # unscoping unnecessary conditions that'll be applied - # when executing `where("projects.id IN (#{union.to_sql})")` - projects = unscoped.select(:id).where( - ptable[:path].matches(pattern) - .or(ptable[:name].matches(pattern)) - .or(ptable[:description].matches(pattern)) - ) - - namespaces = unscoped.select(:id) - .joins(:namespace) - .where(ntable[:name].matches(pattern)) - - union = Gitlab::SQL::Union.new([projects, namespaces]) - - where("projects.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + fuzzy_search(query, [:path, :name, :description]) end def search_by_title(query) - pattern = "%#{query}%" - table = Project.arel_table - - non_archived.where(table[:name].matches(pattern)) + non_archived.fuzzy_search(query, [:name]) end def visibility_levels @@ -518,6 +498,10 @@ class Project < ActiveRecord::Base auto_devops&.enabled.nil? && !current_application_settings.auto_devops_enabled? end + def empty_repo? + repository.empty? + end + def repository_storage_path Gitlab.config.repositories.storages[repository_storage].try(:[], 'path') end @@ -580,8 +564,7 @@ class Project < ActiveRecord::Base if forked? RepositoryForkWorker.perform_async(id, forked_from_project.repository_storage_path, - forked_from_project.full_path, - self.namespace.full_path) + forked_from_project.disk_path) else RepositoryImportWorker.perform_async(self.id) end @@ -617,7 +600,7 @@ class Project < ActiveRecord::Base def ci_config_path=(value) # Strip all leading slashes so that //foo -> foo - super(value&.sub(%r{\A/+}, '')&.delete("\0")) + super(value&.delete("\0")) end def import_url=(value) @@ -703,10 +686,6 @@ class Project < ActiveRecord::Base import_type == 'gitea' end - def github_import? - import_type == 'github' - end - def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit @@ -763,10 +742,10 @@ class Project < ActiveRecord::Base end end - def to_human_reference(from_project = nil) - if cross_namespace_reference?(from_project) + def to_human_reference(from = nil) + if cross_namespace_reference?(from) name_with_namespace - elsif cross_project_reference?(from_project) + elsif cross_project_reference?(from) name end end @@ -775,13 +754,14 @@ class Project < ActiveRecord::Base Gitlab::Routing.url_helpers.project_url(self) end - def new_issue_address(author) + def new_issuable_address(author, address_type) return unless Gitlab::IncomingEmail.supports_issue_creation? && author author.ensure_incoming_email_token! + suffix = address_type == 'merge_request' ? '+merge-request' : '' Gitlab::IncomingEmail.reply_address( - "#{full_path}+#{author.incoming_email_token}") + "#{full_path}#{suffix}+#{author.incoming_email_token}") end def build_commit_note(commit) @@ -919,12 +899,10 @@ class Project < ActiveRecord::Base @ci_service ||= ci_services.reorder(nil).find_by(active: true) end - def deployment_services - services.where(category: :deployment) - end - - def deployment_service - @deployment_service ||= deployment_services.reorder(nil).find_by(active: true) + # TODO: This will be extended for multiple enviroment clusters + def deployment_platform + @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes + @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true) end def monitoring_services @@ -1046,6 +1024,18 @@ class Project < ActiveRecord::Base forked_from_project || fork_network&.root_project end + def lfs_storage_project + @lfs_storage_project ||= begin + result = self + + # TODO: Make this go to the fork_network root immeadiatly + # dependant on the discussion in: https://gitlab.com/gitlab-org/gitlab-ce/issues/39769 + result = result.fork_source while result&.forked? + + result || self + end + end + def personal? !group end @@ -1126,7 +1116,11 @@ class Project < ActiveRecord::Base end def project_member(user) - project_members.find_by(user_id: user) + if project_members.loaded? + project_members.find { |member| member.user_id == user.id } + else + project_members.find_by(user_id: user) + end end def default_branch @@ -1190,6 +1184,10 @@ class Project < ActiveRecord::Base !!repository.exists? end + def wiki_repository_exists? + wiki.repository_exists? + end + # update visibility_level of forks def update_forks_visibility_level return unless visibility_level < visibility_level_was @@ -1433,6 +1431,31 @@ class Project < ActiveRecord::Base reload_repository! end + def after_import + repository.after_import + import_finish + remove_import_jid + update_project_counter_caches + end + + def update_project_counter_caches + classes = [ + Projects::OpenIssuesCountService, + Projects::OpenMergeRequestsCountService + ] + + classes.each do |klass| + klass.new(self).refresh_cache + end + end + + def remove_import_jid + return unless import_jid + + Gitlab::SidekiqStatus.unset(import_jid) + update_column(:import_jid, nil) + end + def running_or_pending_build_count(force: false) Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do builds.running_or_pending.count(:all) @@ -1528,9 +1551,9 @@ class Project < ActiveRecord::Base end def deployment_variables - return [] unless deployment_service + return [] unless deployment_platform - deployment_service.predefined_variables + deployment_platform.predefined_variables end def auto_devops_variables @@ -1690,6 +1713,17 @@ class Project < ActiveRecord::Base Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki)) end + # Refreshes the expiration time of the associated import job ID. + # + # This method can be used by asynchronous importers to refresh the status, + # preventing the StuckImportJobsWorker from marking the import as failed. + def refresh_import_jid_expiration + return unless import_jid + + Gitlab::SidekiqStatus + .set(import_jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + end + private def storage diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 976d85246a8..768f0a7472e 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -51,8 +51,10 @@ class HipchatService < Service def execute(data) return unless supported_events.include?(data[:object_kind]) + message = create_message(data) return unless message.present? + gate[room].send('GitLab', message, message_options(data)) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index b487378edd2..1c065e1ddbd 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -176,6 +176,7 @@ class JiraService < IssueTrackerService def test_settings return unless client_url.present? + # Test settings by getting the project jira_request { client.ServerInfo.all.attrs } end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 5080acffb3c..b82567ce2b3 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -1,3 +1,8 @@ +## +# NOTE: +# We'll move this class to Clusters::Platforms::Kubernetes, which contains exactly the same logic. +# After we've migrated data, we'll remove KubernetesService. This would happen in a few months. +# If you're modyfiyng this class, please note that you should update the same change in Clusters::Platforms::Kubernetes. class KubernetesService < DeploymentService include Gitlab::CurrentSettings include Gitlab::Kubernetes @@ -182,6 +187,7 @@ class KubernetesService < DeploymentService kubeclient.get_pods(namespace: actual_namespace).as_json rescue KubeException => err raise err unless err.error_code == 404 + [] end diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 217f753f05f..fa7b3f2bcaf 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -25,7 +25,7 @@ class PrometheusService < MonitoringService end def description - 'Prometheus monitoring' + s_('PrometheusService|Prometheus monitoring') end def self.to_param @@ -38,8 +38,8 @@ class PrometheusService < MonitoringService type: 'text', name: 'api_url', title: 'API URL', - placeholder: 'Prometheus API Base URL, like http://prometheus.example.com/', - help: 'By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.', + placeholder: s_('PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/'), + help: s_('PrometheusService|By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.'), required: true } ] diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 715b215d1db..17b9d2cf7b4 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -35,7 +35,9 @@ class ProjectStatistics < ActiveRecord::Base end def update_build_artifacts_size - self.build_artifacts_size = project.builds.sum(:artifacts_size) + self.build_artifacts_size = + project.builds.sum(:artifacts_size) + + Ci::JobArtifact.artifacts_size_for(self) end def update_storage_size diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 1d35426050e..c679758973a 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -1,4 +1,6 @@ class ProjectTeam + include BulkMemberAccessLoad + attr_accessor :project def initialize(project) @@ -157,39 +159,16 @@ class ProjectTeam # # Returns a Hash mapping user ID -> maximum access level. def max_member_access_for_user_ids(user_ids) - user_ids = user_ids.uniq - key = "max_member_access:#{project.id}" - - access = {} - - if RequestStore.active? - RequestStore.store[key] ||= {} - access = RequestStore.store[key] + max_member_access_for_resource_ids(User, user_ids, project.id) do |user_ids| + project.project_authorizations + .where(user: user_ids) + .group(:user_id) + .maximum(:access_level) end - - # Look up only the IDs we need - user_ids = user_ids - access.keys - - return access if user_ids.empty? - - users_access = project.project_authorizations - .where(user: user_ids) - .group(:user_id) - .maximum(:access_level) - - access.merge!(users_access) - - missing_user_ids = user_ids - users_access.keys - - missing_user_ids.each do |user_id| - access[user_id] = Gitlab::Access::NO_ACCESS - end - - access end def max_member_access(user_id) - max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS + max_member_access_for_user_ids([user_id])[user_id] end private diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 43de6809178..a0af749a93f 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -21,7 +21,7 @@ class ProjectWiki end delegate :empty?, to: :pages - delegate :repository_storage_path, to: :project + delegate :repository_storage_path, :hashed_storage?, to: :project def path @project.path + '.wiki' @@ -76,8 +76,8 @@ class ProjectWiki # Returns an Array of Gitlab WikiPage instances or an # empty Array if this Wiki has no pages. - def pages - wiki.pages.map { |page| WikiPage.new(self, page, true) } + def pages(limit: nil) + wiki.pages(limit: limit).map { |page| WikiPage.new(self, page, true) } end # Finds a page within the repository based on a tile diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 89bfc5f9a9c..d28fed11ca8 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -10,7 +10,9 @@ class ProtectedBranch < ActiveRecord::Base def self.protected?(project, ref_name) return true if project.empty_repo? && default_branch_protected? - self.matching(ref_name, protected_refs: project.protected_branches).present? + refs = project.protected_branches.select(:name) + + self.matching(ref_name, protected_refs: refs).present? end def self.default_branch_protected? diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb index f38109c0e52..42a9bcf7723 100644 --- a/app/models/protected_tag.rb +++ b/app/models/protected_tag.rb @@ -5,6 +5,8 @@ class ProtectedTag < ActiveRecord::Base protected_ref_access_levels :create def self.protected?(project, ref_name) - self.matching(ref_name, protected_refs: project.protected_tags).present? + refs = project.protected_tags.select(:name) + + self.matching(ref_name, protected_refs: refs).present? end end diff --git a/app/models/protected_tag/create_access_level.rb b/app/models/protected_tag/create_access_level.rb index c7e1319719d..6b6ab3d8279 100644 --- a/app/models/protected_tag/create_access_level.rb +++ b/app/models/protected_tag/create_access_level.rb @@ -1,18 +1,6 @@ class ProtectedTag::CreateAccessLevel < ActiveRecord::Base include ProtectedTagAccess - validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER, - Gitlab::Access::DEVELOPER, - Gitlab::Access::NO_ACCESS] } - - def self.human_access_levels - { - Gitlab::Access::MASTER => "Masters", - Gitlab::Access::DEVELOPER => "Developers + Masters", - Gitlab::Access::NO_ACCESS => "No one" - }.with_indifferent_access - end - def check_access(user) return false if access_level == Gitlab::Access::NO_ACCESS diff --git a/app/models/repository.rb b/app/models/repository.rb index eb7766d040c..751306188a0 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -37,7 +37,7 @@ class Repository issue_template_names merge_request_template_names).freeze # Methods that use cache_method but only memoize the value - MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze + MEMOIZED_CACHED_METHODS = %i(license).freeze # Certain method caches should be refreshed when certain types of files are # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to @@ -132,7 +132,8 @@ class Repository commits = Gitlab::Git::Commit.where(options) commits = Commit.decorate(commits, @project) if commits.present? - commits + + CommitCollection.new(project, commits, ref) end def commits_between(from, to) @@ -148,11 +149,14 @@ class Repository end raw_repository.gitaly_migrate(:commits_by_message) do |is_enabled| - if is_enabled - find_commits_by_message_by_gitaly(query, ref, path, limit, offset) - else - find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) - end + commits = + if is_enabled + find_commits_by_message_by_gitaly(query, ref, path, limit, offset) + else + find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) + end + + CommitCollection.new(project, commits, ref) end end @@ -213,11 +217,7 @@ class Repository def branch_exists?(branch_name) return false unless raw_repository - @branch_exists_memo ||= Hash.new do |hash, key| - hash[key] = raw_repository.branch_exists?(key) - end - - @branch_exists_memo[branch_name] + branch_names.include?(branch_name) end def ref_exists?(ref) @@ -242,6 +242,7 @@ class Repository Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" rescue Rugged::OSError => ex raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" end end @@ -255,7 +256,7 @@ class Repository end def diverging_commit_counts(branch) - root_ref_hash = raw_repository.rev_parse_target(root_ref).oid + root_ref_hash = raw_repository.commit(root_ref).id cache.fetch(:"diverging_commit_counts_#{branch.name}") do # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes @@ -473,6 +474,11 @@ class Repository nil end + # items is an Array like: [[oid, path], [oid1, path1]] + def blobs_at(items) + raw_repository.batch_blobs(items).map { |blob| Blob.decorate(blob, project) } + end + def root_ref if raw_repository raw_repository.root_ref @@ -491,7 +497,11 @@ class Repository end cache_method :exists? - delegate :empty?, to: :raw_repository + def empty? + return true unless exists? + + !has_visible_content? + end cache_method :empty? # The size of this repository in megabytes. @@ -662,6 +672,7 @@ class Repository def next_branch(name, opts = {}) branch_ids = self.branch_names.map do |n| next 1 if n == name + result = n.match(/\A#{name}-([0-9]+)\z/) result[1].to_i if result end.compact @@ -902,19 +913,13 @@ class Repository end end - def merged_to_root_ref?(branch_or_name, pre_loaded_merged_branches = nil) + def merged_to_root_ref?(branch_or_name) branch = Gitlab::Git::Branch.find(self, branch_or_name) if branch @root_ref_sha ||= commit(root_ref).sha same_head = branch.target == @root_ref_sha - merged = - if pre_loaded_merged_branches - pre_loaded_merged_branches.include?(branch.name) - else - ancestor?(branch.target, @root_ref_sha) - end - + merged = ancestor?(branch.target, @root_ref_sha) !same_head && merged else nil @@ -943,13 +948,8 @@ class Repository end end - def empty_repo? - !exists? || !has_visible_content? - end - cache_method :empty_repo?, memoize_only: true - def search_files_by_content(query, ref) - return [] if empty_repo? || query.blank? + return [] if empty? || query.blank? offset = 2 args = %W(grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) @@ -958,13 +958,26 @@ class Repository end def search_files_by_name(query, ref) - return [] if empty_repo? || query.blank? + return [] if empty? || query.blank? args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)}) run_git(args).first.lines.map(&:strip) end + def fetch_as_mirror(url, forced: false, refmap: :all_refs, remote_name: nil) + unless remote_name + remote_name = "tmp-#{SecureRandom.hex}" + tmp_remote_name = true + end + + add_remote(remote_name, url) + set_remote_as_mirror(remote_name, refmap: refmap) + fetch_remote(remote_name, forced: forced) + ensure + remove_remote(remote_name) if tmp_remote_name + end + def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false) gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) end @@ -973,6 +986,10 @@ class Repository raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end + def remote_exists?(name) + raw_repository.remote_exists?(name) + end + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) raw_repository.compare_source_branch(target_branch_name, source_repository.raw_repository, source_branch_name, straight: straight) end @@ -986,10 +1003,6 @@ class Repository raw_repository.ls_files(actual_ref) end - def gitattribute(path, name) - raw_repository.attributes(path)[name] - end - def copy_gitattributes(ref) actual_ref = ref || root_ref begin @@ -1058,6 +1071,14 @@ class Repository blob_data_at(sha, path) end + def fetch_ref(source_repository, source_ref:, target_ref:) + raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) + end + + def repository_storage_path + @project.repository_storage_path + end + private # TODO Generice finder, later split this on finders by Ref or Oid @@ -1123,10 +1144,6 @@ class Repository raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip end - def repository_storage_path - @project.repository_storage_path - end - def initialize_raw_repository Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki)) end diff --git a/app/models/service.rb b/app/models/service.rb index fdd2605e3e3..3c4f1885dd0 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -211,7 +211,7 @@ class Service < ActiveRecord::Base def async_execute(data) return unless supported_events.include?(data[:object_kind]) - Sidekiq::Client.enqueue(ProjectServiceWorker, id, data) + ProjectServiceWorker.perform_async(id, data) end def issue_tracker? diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 9533aa7f555..05a16f11b59 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -9,6 +9,7 @@ class Snippet < ActiveRecord::Base include Mentionable include Spammable include Editable + include Gitlab::SQL::Pattern extend Gitlab::CurrentSettings @@ -75,11 +76,11 @@ class Snippet < ActiveRecord::Base @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) end - def to_reference(from_project = nil, full: false) + def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{id}" if project.present? - "#{project.to_reference(from_project, full: full)}#{reference}" + "#{project.to_reference(from, full: full)}#{reference}" else reference end @@ -135,10 +136,7 @@ class Snippet < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) - t = arel_table - pattern = "%#{query}%" - - where(t[:title].matches(pattern).or(t[:file_name].matches(pattern))) + fuzzy_search(query, [:title, :file_name]) end # Searches for snippets with matching content. @@ -149,10 +147,7 @@ class Snippet < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search_code(query) - table = Snippet.arel_table - pattern = "%#{query}%" - - where(table[:content].matches(pattern)) + fuzzy_search(query, [:content]) end end end diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index f025f40994e..fae1b64961a 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -4,7 +4,6 @@ module Storage delegate :gitlab_shell, :repository_storage_path, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze - STORAGE_VERSION = 1 def initialize(project) @project = project diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 1f9f8d7286b..29035480371 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -1,4 +1,14 @@ class SystemNoteMetadata < ActiveRecord::Base + # These notes's action text might contain a reference that is external. + # We should always force a deep validation upon references that are found + # in this note type. + # Other notes can always be safely shown as all its references are + # in the same project (i.e. with the same permissions) + TYPES_WITH_CROSS_REFERENCES = %w[ + commit cross_reference + close duplicate + ].freeze + ICON_TYPES = %w[ commit description merge confidential visible label assignee cross_reference title time_tracking branch milestone discussion task moved diff --git a/app/models/user.rb b/app/models/user.rb index aa88cda4dc0..af1c36d9c93 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -7,6 +7,7 @@ class User < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::CurrentSettings include Gitlab::SQL::Pattern + include AfterCommitQueue include Avatarable include Referable include Sortable @@ -16,6 +17,7 @@ class User < ActiveRecord::Base include FeatureGate include CreatedAtFilterable include IgnorableColumn + include BulkMemberAccessLoad DEFAULT_NOTIFICATION_LEVEL = :participating @@ -170,6 +172,7 @@ class User < ActiveRecord::Base after_save :ensure_namespace_correct after_update :username_changed_hook, if: :username_changed? after_destroy :post_destroy_hook + after_destroy :remove_key_cache after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') } after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } @@ -267,18 +270,22 @@ class User < ActiveRecord::Base end end + def for_github_id(id) + joins(:identities).merge(Identity.with_extern_uid(:github, id)) + end + # Find a User by their primary email or any associated secondary email def find_by_any_email(email) - sql = 'SELECT * - FROM users - WHERE id IN ( - SELECT id FROM users WHERE email = :email - UNION - SELECT emails.user_id FROM emails WHERE email = :email - ) - LIMIT 1;' + by_any_email(email).take + end + + # Returns a relation containing all the users for the given Email address + def by_any_email(email) + users = where(email: email) + emails = joins(:emails).where(emails: { email: email }) + union = Gitlab::SQL::Union.new([users, emails]) - User.find_by_sql([sql, { email: email }]).first + from("(#{union.to_sql}) #{table_name}") end def filter(filter_name) @@ -308,9 +315,6 @@ class User < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) - table = arel_table - pattern = User.to_pattern(query) - order = <<~SQL CASE WHEN users.name = %{query} THEN 0 @@ -320,11 +324,8 @@ class User < ActiveRecord::Base END SQL - where( - table[:name].matches(pattern) - .or(table[:email].matches(pattern)) - .or(table[:username].matches(pattern)) - ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) + fuzzy_search(query, [:name, :email, :username]) + .reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end # searches user by given pattern @@ -332,16 +333,16 @@ class User < ActiveRecord::Base # This method uses ILIKE on PostgreSQL and LIKE on MySQL. def search_with_secondary_emails(query) - table = arel_table email_table = Email.arel_table - pattern = "%#{query}%" - matched_by_emails_user_ids = email_table.project(email_table[:user_id]).where(email_table[:email].matches(pattern)) + matched_by_emails_user_ids = email_table + .project(email_table[:user_id]) + .where(Email.fuzzy_arel_match(:email, query)) where( - table[:name].matches(pattern) - .or(table[:email].matches(pattern)) - .or(table[:username].matches(pattern)) - .or(table[:id].in(matched_by_emails_user_ids)) + fuzzy_arel_match(:name, query) + .or(fuzzy_arel_match(:email, query)) + .or(fuzzy_arel_match(:username, query)) + .or(arel_table[:id].in(matched_by_emails_user_ids)) ) end @@ -432,7 +433,7 @@ class User < ActiveRecord::Base username end - def to_reference(_from_project = nil, target_project: nil, full: nil) + def to_reference(_from = nil, target_project: nil, full: nil) "#{self.class.reference_prefix}#{username}" end @@ -440,6 +441,10 @@ class User < ActiveRecord::Base skip_confirmation! if bool end + def skip_reconfirmation=(bool) + skip_reconfirmation! if bool + end + def generate_reset_token @reset_token, enc = Devise.token_generator.generate(self.class, :reset_password_token) @@ -484,7 +489,11 @@ class User < ActiveRecord::Base end def two_factor_u2f_enabled? - u2f_registrations.exists? + if u2f_registrations.loaded? + u2f_registrations.any? + else + u2f_registrations.exists? + end end def namespace_uniq @@ -619,21 +628,39 @@ class User < ActiveRecord::Base end def require_ssh_key? - keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh') + count = Users::KeysCountService.new(self).count + + count.zero? && Gitlab::ProtocolAccess.allowed?('ssh') + end + + def require_password_creation_for_web? + allow_password_authentication_for_web? && password_automatically_set? end - def require_password_creation? - password_automatically_set? && allow_password_authentication? + def require_password_creation_for_git? + allow_password_authentication_for_git? && password_automatically_set? end def require_personal_access_token_creation_for_git_auth? - return false if current_application_settings.password_authentication_enabled? || ldap_user? + return false if allow_password_authentication_for_git? || ldap_user? PersonalAccessTokensFinder.new(user: self, impersonation: false, state: 'active').execute.none? end + def require_extra_setup_for_git_auth? + require_password_creation_for_git? || require_personal_access_token_creation_for_git_auth? + end + def allow_password_authentication? - !ldap_user? && current_application_settings.password_authentication_enabled? + allow_password_authentication_for_web? || allow_password_authentication_for_git? + end + + def allow_password_authentication_for_web? + current_application_settings.password_authentication_enabled_for_web? && !ldap_user? + end + + def allow_password_authentication_for_git? + current_application_settings.password_authentication_enabled_for_git? && !ldap_user? end def can_change_username? @@ -878,9 +905,14 @@ class User < ActiveRecord::Base def post_destroy_hook log_info("User \"#{name}\" (#{email}) was removed") + system_hook_service.execute_hooks_for(self, :destroy) end + def remove_key_cache + Users::KeysCountService.new(self).delete_cache + end + def delete_async(deleted_by:, params: {}) block if params[:hard_delete] DeleteUserWorker.perform_async(deleted_by.id, id, params) @@ -916,7 +948,16 @@ class User < ActiveRecord::Base end def manageable_namespaces - @manageable_namespaces ||= [namespace] + owned_groups + masters_groups + @manageable_namespaces ||= [namespace] + manageable_groups + end + + def manageable_groups + union = Gitlab::SQL::Union.new([owned_groups.select(:id), + masters_groups.select(:id)]) + arel_union = Arel::Nodes::SqlLiteral.new(union.to_sql) + owned_and_master_groups = Group.where(Group.arel_table[:id].in(arel_union)) + + Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants end def namespaces @@ -964,7 +1005,11 @@ class User < ActiveRecord::Base end def notification_settings_for(source) - notification_settings.find_or_initialize_by(source: source) + if notification_settings.loaded? + notification_settings.find { |notification| notification.source == source } + else + notification_settings.find_or_initialize_by(source: source) + end end # Lazy load global notification setting @@ -1100,11 +1145,40 @@ class User < ActiveRecord::Base super end + # Determine the maximum access level for a group of projects in bulk. + # + # Returns a Hash mapping project ID -> maximum access level. + def max_member_access_for_project_ids(project_ids) + max_member_access_for_resource_ids(Project, project_ids) do |project_ids| + project_authorizations.where(project: project_ids) + .group(:project_id) + .maximum(:access_level) + end + end + + def max_member_access_for_project(project_id) + max_member_access_for_project_ids([project_id])[project_id] + end + + # Determine the maximum access level for a group of groups in bulk. + # + # Returns a Hash mapping project ID -> maximum access level. + def max_member_access_for_group_ids(group_ids) + max_member_access_for_resource_ids(Group, group_ids) do |group_ids| + group_members.where(source: group_ids).group(:source_id).maximum(:access_level) + end + end + + def max_member_access_for_group(group_id) + max_member_access_for_group_ids([group_id])[group_id] + end + protected # override, from Devise::Validatable def password_required? return false if internal? + super end @@ -1122,6 +1196,7 @@ class User < ActiveRecord::Base # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration def send_devise_notification(notification, *args) return true unless can?(:receive_notifications) + devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 5f710961f95..bdfef677ef3 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -127,19 +127,24 @@ class WikiPage @version ||= @page.version end - # Returns an array of Gitlab Commit instances. - def versions + def versions(options = {}) return [] unless persisted? - wiki.wiki.page_versions(@page.path) + wiki.wiki.page_versions(@page.path, options) end - def commit - versions.first + def count_versions + return [] unless persisted? + + wiki.wiki.count_page_versions(@page.path) + end + + def last_version + @last_version ||= versions(limit: 1).first end def last_commit_sha - commit&.sha + last_version&.sha end # Returns the Date that this latest version was @@ -151,7 +156,7 @@ class WikiPage # Returns boolean True or False if this instance # is an old version of the page. def historical? - @page.historical? && versions.first.sha != version.sha + @page.historical? && last_version.sha != version.sha end # Returns boolean True or False if this instance |