diff options
Diffstat (limited to 'app/models')
43 files changed, 750 insertions, 216 deletions
diff --git a/app/models/appearance.rb b/app/models/appearance.rb index ff15689ecac..76cfe28742a 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -2,9 +2,8 @@ class Appearance < ActiveRecord::Base include CacheMarkdownField cache_markdown_field :description + cache_markdown_field :new_project_guidelines - validates :title, presence: true - validates :description, presence: true validates :logo, file_size: { maximum: 1.megabyte } validates :header_logo, file_size: { maximum: 1.megabyte } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 3117c98c846..253e213af81 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -153,11 +153,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { greater_than_or_equal_to: 0 } - validates :circuitbreaker_backoff_threshold, - :circuitbreaker_failure_count_threshold, - :circuitbreaker_failure_wait_time, + validates :circuitbreaker_failure_count_threshold, :circuitbreaker_failure_reset_time, :circuitbreaker_storage_timeout, + :circuitbreaker_check_interval, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -165,13 +164,6 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 1 } - validates_each :circuitbreaker_backoff_threshold do |record, attr, value| - if value.to_i >= record.circuitbreaker_failure_count_threshold - record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\ - "lower than the failure count threshold")) - end - end - validates :gitaly_timeout_default, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } diff --git a/app/models/blob.rb b/app/models/blob.rb index 29e762724e3..19ad110db58 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -77,9 +77,15 @@ class Blob < SimpleDelegator end def self.lazy(project, commit_id, path) - BatchLoader.for(commit_id: commit_id, path: path).batch do |items, loader| - project.repository.blobs_at(items.map(&:values)).each do |blob| - loader.call({ commit_id: blob.commit_id, path: blob.path }, blob) if blob + BatchLoader.for({ project: project, commit_id: commit_id, path: path }).batch do |items, loader| + items_by_project = items.group_by { |i| i[:project] } + + items_by_project.each do |project, items| + items = items.map { |i| i.values_at(:commit_id, :path) } + + project.repository.blobs_at(items).each do |blob| + loader.call({ project: blob.project, commit_id: blob.commit_id, path: blob.path }, blob) if blob + end end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4ea040dfad5..83fe23606d1 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -1,18 +1,26 @@ module Ci class Build < CommitStatus + prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue include Presentable include Importable + MissingDependenciesError = Class.new(StandardError) + belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' has_many :deployments, as: :deployable + has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @persisted_environment ||= Environment.find_by( @@ -31,15 +39,37 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } - scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) } + scope :with_artifacts, ->() do + where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', + '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id')) + end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } - mount_uploader :artifacts_file, ArtifactUploader - mount_uploader :artifacts_metadata, ArtifactUploader + scope :matches_tag_ids, -> (tag_ids) do + matcher = ::ActsAsTaggableOn::Tagging + .where(taggable_type: CommitStatus) + .where(context: 'tags') + .where('taggable_id = ci_builds.id') + .where.not(tag_id: tag_ids).select('1') + + where("NOT EXISTS (?)", matcher) + end + + scope :with_any_tags, -> do + matcher = ::ActsAsTaggableOn::Tagging + .where(taggable_type: CommitStatus) + .where(context: 'tags') + .where('taggable_id = ci_builds.id').select('1') + + where("EXISTS (?)", matcher) + end + + mount_uploader :legacy_artifacts_file, LegacyArtifactUploader, mount_on: :artifacts_file + mount_uploader :legacy_artifacts_metadata, LegacyArtifactUploader, mount_on: :artifacts_metadata acts_as_taggable @@ -111,6 +141,10 @@ module Ci Ci::Build.retry(build, build.user) end end + + before_transition any => [:running] do |build| + build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') + end end def detailed_status(current_user) @@ -326,14 +360,6 @@ module Ci project.running_or_pending_build_count(force: true) end - def artifacts? - !artifacts_expired? && artifacts_file.exists? - end - - def artifacts_metadata? - artifacts? && artifacts_metadata.exists? - end - def artifacts_metadata_entry(path, **options) metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( artifacts_metadata.path, @@ -386,6 +412,7 @@ module Ci def keep_artifacts! self.update(artifacts_expire_at: nil) + self.job_artifacts.update_all(expire_at: nil) end def coverage_regex @@ -457,6 +484,19 @@ module Ci options[:dependencies]&.empty? end + def validates_dependencies! + dependencies.each do |dependency| + raise MissingDependenciesError unless dependency.valid_dependency? + end + end + + def valid_dependency? + return false if artifacts_expired? + return false if erased? + + true + end + def hide_secrets(trace) return unless trace @@ -473,11 +513,7 @@ module Ci private def update_artifacts_size - self.artifacts_size = if artifacts_file.exists? - artifacts_file.size - else - nil - end + self.artifacts_size = legacy_artifacts_file&.size end def erase_trace! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb new file mode 100644 index 00000000000..84fc6863567 --- /dev/null +++ b/app/models/ci/job_artifact.rb @@ -0,0 +1,36 @@ +module Ci + class JobArtifact < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :project + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + before_save :set_size, if: :file_changed? + + mount_uploader :file, JobArtifactUploader + + enum file_type: { + archive: 1, + metadata: 2 + } + + def self.artifacts_size_for(project) + self.where(project: project).sum(:size) + end + + def set_size + self.size = file.size + end + + def expire_in + expire_at - Time.now if expire_at + end + + def expire_in=(value) + self.expire_at = + if value + ChronicDuration.parse(value)&.seconds&.from_now + end + end + end +end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ebbefc51a4f..eebbf7c4218 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -40,7 +40,6 @@ module Ci validates :status, presence: { unless: :importing? } validate :valid_commit_sha, unless: :importing? - after_initialize :set_config_source, if: :new_record? after_create :keep_around_commits, unless: :importing? enum source: { @@ -365,7 +364,7 @@ module Ci end def has_kubernetes_active? - project.kubernetes_service&.active? + project.deployment_platform&.active? end def has_stage_seeds? diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d39610a8995..dcbb397fb78 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -112,7 +112,7 @@ module Ci def can_pick?(build) return false if self.ref_protected? && !build.protected? - assignable_for?(build.project) && accepting_tags?(build) + assignable_for?(build.project_id) && accepting_tags?(build) end def only_for?(project) @@ -171,8 +171,8 @@ module Ci end end - def assignable_for?(project) - is_shared? || projects.exists?(id: project.id) + def assignable_for?(project_id) + is_shared? || projects.exists?(id: project_id) end def accepting_tags?(build) diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 185d9473aab..55419189282 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -17,8 +17,7 @@ module Clusters # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true - # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration - has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true has_one :application_helm, class_name: 'Clusters::Applications::Helm' has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' @@ -29,15 +28,9 @@ module Clusters validates :name, cluster_name: true validate :restrict_modification, on: :update - # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3 - # We need callback here because `enabled` belongs to Clusters::Cluster - # Callbacks in Clusters::Platforms::Kubernetes will not be called after update - after_save :update_kubernetes_integration! - delegate :status, to: :provider, allow_nil: true delegate :status_reason, to: :provider, allow_nil: true delegate :on_creation?, to: :provider, allow_nil: true - delegate :update_kubernetes_integration!, to: :platform, allow_nil: true delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true delegate :installed?, to: :application_helm, prefix: true, allow_nil: true @@ -62,6 +55,10 @@ module Clusters end end + def created? + status_name == :created + end + def applications [ application_helm || build_application_helm, @@ -77,6 +74,10 @@ module Clusters return platform_kubernetes if kubernetes? end + def managed? + !user? + end + def first_project return @first_project if defined?(@first_project) diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 6dc1ee810d3..9160a169452 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -1,7 +1,12 @@ module Clusters module Platforms class Kubernetes < ActiveRecord::Base + include Gitlab::CurrentSettings + include Gitlab::Kubernetes + include ReactiveCaching + self.table_name = 'cluster_platforms_kubernetes' + self.reactive_cache_key = ->(kubernetes) { [kubernetes.class.model_name.singular, kubernetes.id] } belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster' @@ -29,19 +34,17 @@ module Clusters validates :api_url, url: true, presence: true validates :token, presence: true - # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes - after_destroy :destroy_kubernetes_integration! + validate :prevent_modification, on: :update + + after_save :clear_reactive_cache! alias_attribute :ca_pem, :ca_cert delegate :project, to: :cluster, allow_nil: true delegate :enabled?, to: :cluster, allow_nil: true + delegate :managed?, to: :cluster, allow_nil: true - class << self - def namespace_for_project(project) - "#{project.path}-#{project.id}" - end - end + alias_method :active?, :enabled? def actual_namespace if namespace.present? @@ -51,58 +54,138 @@ module Clusters end end - def default_namespace - self.class.namespace_for_project(project) if project + def predefined_variables + config = YAML.dump(kubeconfig) + + variables = [ + { key: 'KUBE_URL', value: api_url, public: true }, + { key: 'KUBE_TOKEN', value: token, public: false }, + { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true }, + { key: 'KUBECONFIG', value: config, public: false, file: true } + ] + + if ca_pem.present? + variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true } + variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true } + end + + variables end - def kubeclient - @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service? + # Constructs a list of terminals from the reactive cache + # + # Returns nil if the cache is empty, in which case you should try again a + # short time later + def terminals(environment) + with_reactive_cache do |data| + pods = filter_by_label(data[:pods], app: environment.slug) + terminals = pods.flat_map { |pod| terminals_for_pod(api_url, actual_namespace, pod) } + terminals.each { |terminal| add_terminal_auth(terminal, terminal_auth) } + end end - def update_kubernetes_integration! - raise 'Kubernetes service already configured' unless manages_kubernetes_service? + # Caches resources in the namespace so other calls don't need to block on + # network access + def calculate_reactive_cache + return unless enabled? && project && !project.pending_delete? - # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false - cluster.reload + # We may want to cache extra things in the future + { pods: read_pods } + end + + def kubeclient + @kubeclient ||= build_kubeclient! + end + + private - ensure_kubernetes_service&.update!( - active: enabled?, - api_url: api_url, - namespace: namespace, + def kubeconfig + to_kubeconfig( + url: api_url, + namespace: actual_namespace, token: token, - ca_pem: ca_cert - ) + ca_pem: ca_pem) end - def active? - manages_kubernetes_service? + def default_namespace + return unless project + + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') end - private + def build_kubeclient!(api_path: 'api', api_version: 'v1') + raise "Incomplete settings" unless api_url && actual_namespace - def enforce_namespace_to_lower_case - self.namespace = self.namespace&.downcase + unless (username && password) || token + raise "Either username/password or token is required to access API" + end + + ::Kubeclient::Client.new( + join_api_url(api_path), + api_version, + auth_options: kubeclient_auth_options, + ssl_options: kubeclient_ssl_options, + http_proxy_uri: ENV['http_proxy'] + ) end - # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class - def manages_kubernetes_service? - return true unless kubernetes_service&.active? + # Returns a hash of all pods in the namespace + def read_pods + kubeclient = build_kubeclient! + + kubeclient.get_pods(namespace: actual_namespace).as_json + rescue KubeException => err + raise err unless err.error_code == 404 + + [] + end + + def kubeclient_ssl_options + opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER } + + if ca_pem.present? + opts[:cert_store] = OpenSSL::X509::Store.new + opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem)) + end + + opts + end - kubernetes_service.api_url == api_url + def kubeclient_auth_options + { bearer_token: token } end - def destroy_kubernetes_integration! - return unless manages_kubernetes_service? + def join_api_url(api_path) + url = URI.parse(api_url) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [prefix, api_path].join("/") - kubernetes_service&.destroy! + url.to_s end - def kubernetes_service - @kubernetes_service ||= project&.kubernetes_service + def terminal_auth + { + token: token, + ca_pem: ca_pem, + max_session_time: current_application_settings.terminal_max_session_time + } end - def ensure_kubernetes_service - @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service + def enforce_namespace_to_lower_case + self.namespace = self.namespace&.downcase + end + + def prevent_modification + return unless managed? + + if api_url_changed? || token_changed? || ca_pem_changed? + errors.add(:base, "cannot modify managed cluster") + return false + end + + true end end end diff --git a/app/models/commit.rb b/app/models/commit.rb index 6b28d290f99..13c31111134 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -1,3 +1,4 @@ +# coding: utf-8 class Commit extend ActiveModel::Naming extend Gitlab::Cache::RequestCache @@ -25,7 +26,7 @@ class Commit DIFF_HARD_LIMIT_FILES = 1000 DIFF_HARD_LIMIT_LINES = 50000 - MIN_SHA_LENGTH = 7 + MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze def banzai_render_context(field) @@ -51,6 +52,20 @@ class Commit diffs.reduce(0) { |sum, d| sum + Gitlab::Git::Util.count_lines(d.diff) } end + def order_by(collection:, order_by:, sort:) + return collection unless %w[email name commits].include?(order_by) + return collection unless %w[asc desc].include?(sort) + + collection.sort do |a, b| + operands = [a, b].tap { |o| o.reverse! if sort == 'desc' } + + attr1, attr2 = operands.first.public_send(order_by), operands.second.public_send(order_by) # rubocop:disable PublicSend + + # use case insensitive comparison for string values + order_by.in?(%w[email name]) ? attr1.casecmp(attr2) : attr1 <=> attr2 + end + end + # Truncate sha to 8 characters def truncate_sha(sha) sha[0..MIN_SHA_LENGTH] diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index ee21ed8e420..c0263c0b4e2 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -43,7 +43,8 @@ class CommitStatus < ActiveRecord::Base script_failure: 1, api_failure: 2, stuck_or_timeout_failure: 3, - runner_system_failure: 4 + runner_system_failure: 4, + missing_dependency_failure: 5 } ## diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb new file mode 100644 index 00000000000..0460439e9e6 --- /dev/null +++ b/app/models/concerns/artifact_migratable.rb @@ -0,0 +1,45 @@ +# Adapter class to unify the interface between mounted uploaders and the +# Ci::Artifact model +# Meant to be prepended so the interface can stay the same +module ArtifactMigratable + def artifacts_file + job_artifacts_archive&.file || legacy_artifacts_file + end + + def artifacts_metadata + job_artifacts_metadata&.file || legacy_artifacts_metadata + end + + def artifacts? + !artifacts_expired? && artifacts_file.exists? + end + + def artifacts_metadata? + artifacts? && artifacts_metadata.exists? + end + + def artifacts_file_changed? + job_artifacts_archive&.file_changed? || attribute_changed?(:artifacts_file) + end + + def remove_artifacts_file! + if job_artifacts_archive + job_artifacts_archive.destroy + else + remove_legacy_artifacts_file! + end + end + + def remove_artifacts_metadata! + if job_artifacts_metadata + job_artifacts_metadata.destroy + else + remove_legacy_artifacts_metadata! + end + end + + def artifacts_size + read_attribute(:artifacts_size).to_i + + job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i + end +end diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb new file mode 100644 index 00000000000..984c4f53bf7 --- /dev/null +++ b/app/models/concerns/bulk_member_access_load.rb @@ -0,0 +1,46 @@ +# Returns and caches in thread max member access for a resource +# +module BulkMemberAccessLoad + extend ActiveSupport::Concern + + included do + # Determine the maximum access level for a group of resources in bulk. + # + # Returns a Hash mapping resource ID -> maximum access level. + def max_member_access_for_resource_ids(resource_klass, resource_ids, memoization_index = self.id, &block) + raise 'Block is mandatory' unless block_given? + + resource_ids = resource_ids.uniq + key = max_member_access_for_resource_key(resource_klass, memoization_index) + access = {} + + if RequestStore.active? + RequestStore.store[key] ||= {} + access = RequestStore.store[key] + end + + # Look up only the IDs we need + resource_ids = resource_ids - access.keys + + return access if resource_ids.empty? + + resource_access = yield(resource_ids) + + access.merge!(resource_access) + + missing_resource_ids = resource_ids - resource_access.keys + + missing_resource_ids.each do |resource_id| + access[resource_id] = Gitlab::Access::NO_ACCESS + end + + access + end + + private + + def max_member_access_for_resource_key(klass, memoization_index) + "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}" + end + end +end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 98776eab424..90ad644ce34 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -85,8 +85,7 @@ module CacheMarkdownField def cached_html_up_to_date?(markdown_field) html_field = cached_markdown_fields.html_field(markdown_field) - cached = cached_html_for(markdown_field).present? && __send__(markdown_field).present? # rubocop:disable GitlabSecurity/PublicSend - return false unless cached + return false if cached_html_for(markdown_field).nil? && !__send__(markdown_field).nil? # rubocop:disable GitlabSecurity/PublicSend markdown_changed = attribute_changed?(markdown_field) || false html_changed = attribute_changed?(html_field) || false diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb index f5cbb3becad..4b4d519f3df 100644 --- a/app/models/concerns/discussion_on_diff.rb +++ b/app/models/concerns/discussion_on_diff.rb @@ -32,6 +32,10 @@ module DiscussionOnDiff first_note.position.new_path end + def on_merge_request_commit? + for_merge_request? && commit_id.present? + end + # Returns an array of at most 16 highlighted lines above a diff note def truncated_diff_lines(highlight: true) lines = highlight ? highlighted_diff_lines : diff_lines diff --git a/app/models/concerns/throttled_touch.rb b/app/models/concerns/throttled_touch.rb new file mode 100644 index 00000000000..ad0ff0f20d4 --- /dev/null +++ b/app/models/concerns/throttled_touch.rb @@ -0,0 +1,10 @@ +# ThrottledTouch can be used to throttle the number of updates triggered by +# calling "touch" on an ActiveRecord model. +module ThrottledTouch + # The amount of time to wait before "touch" can update a record again. + TOUCH_INTERVAL = 1.minute + + def touch(*args) + super if (Time.zone.now - updated_at) > TOUCH_INTERVAL + end +end diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb index 6eba87da1a1..4a65738214b 100644 --- a/app/models/diff_discussion.rb +++ b/app/models/diff_discussion.rb @@ -24,7 +24,11 @@ class DiffDiscussion < Discussion return unless for_merge_request? return {} if active? - noteable.version_params_for(position.diff_refs) + if on_merge_request_commit? + { commit_id: commit_id } + else + noteable.version_params_for(position.diff_refs) + end end def reply_attributes diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index ae5f138a920..b53d44cda95 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -17,6 +17,7 @@ class DiffNote < Note validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete validate :verify_supported + validate :diff_refs_match_commit, if: :for_commit? before_validation :set_original_position, on: :create before_validation :update_position, on: :create, if: :on_text? @@ -135,6 +136,12 @@ class DiffNote < Note errors.add(:position, "is invalid") end + def diff_refs_match_commit + return if self.original_position.diff_refs == self.commit.diff_refs + + errors.add(:commit_id, 'does not match the diff refs') + end + def keep_around_commits project.repository.keep_around(self.original_position.base_sha) project.repository.keep_around(self.original_position.start_sha) diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 437df923d2d..92482a1a875 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -11,6 +11,7 @@ class Discussion :author, :noteable, + :commit_id, :for_commit?, :for_merge_request?, diff --git a/app/models/environment.rb b/app/models/environment.rb index 21a028e351c..bf69b4c50f0 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -138,11 +138,11 @@ class Environment < ActiveRecord::Base end def has_terminals? - project.deployment_service.present? && available? && last_deployment.present? + project.deployment_platform.present? && available? && last_deployment.present? end def terminals - project.deployment_service.terminals(self) if has_terminals? + project.deployment_platform.terminals(self) if has_terminals? end def has_metrics? diff --git a/app/models/epic.rb b/app/models/epic.rb index 62898a02e2d..286b855de3f 100644 --- a/app/models/epic.rb +++ b/app/models/epic.rb @@ -1,7 +1,11 @@ # Placeholder class for model that is implemented in EE -# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE +# It reserves '&' as a reference prefix, but the table does not exists in CE class Epic < ActiveRecord::Base - # TODO: this will be implemented as part of #3853 - def to_reference + def self.reference_prefix + '&' + end + + def self.reference_prefix_escaped + '&' end end diff --git a/app/models/event.rb b/app/models/event.rb index 0997b056c6a..6053594fab5 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -72,7 +72,7 @@ class Event < ActiveRecord::Base # We're using preload for "push_event_payload" as otherwise the association # is not always available (depending on the query being built). includes(:author, :project, project: :namespace) - .preload(:target, :push_event_payload) + .preload(:push_event_payload, target: :author) end scope :for_milestone_id, ->(milestone_id) { where(target_type: "Milestone", target_id: milestone_id) } diff --git a/app/models/group.rb b/app/models/group.rb index 76262acf50c..fddace03387 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper + include AfterCommitQueue include AccessRequestable include Avatarable include Referable @@ -289,6 +290,18 @@ class Group < Namespace "#{parent.full_path}/#{path_was}" end + def group_member(user) + if group_members.loaded? + group_members.find { |gm| gm.user_id == user.id } + else + group_members.find_by(user_id: user) + end + end + + def hashed_storage?(_feature) + false + end + private def update_two_factor_requirement diff --git a/app/models/issue.rb b/app/models/issue.rb index d6ef58d150b..dc64888b6fc 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -9,6 +9,10 @@ class Issue < ActiveRecord::Base include FasterCacheKeys include RelativePositioning include TimeTrackable + include ThrottledTouch + include IgnorableColumn + + ignore_column :assignee_id, :branch_name DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze diff --git a/app/models/key.rb b/app/models/key.rb index 815fd1de909..a3f8a5d6dc7 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -2,6 +2,7 @@ require 'digest/md5' class Key < ActiveRecord::Base include Gitlab::CurrentSettings + include AfterCommitQueue include Sortable belongs_to :user diff --git a/app/models/member.rb b/app/models/member.rb index cbbd58f2eaf..c47145667b5 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,8 +1,10 @@ class Member < ActiveRecord::Base + include AfterCommitQueue include Sortable include Importable include Expirable include Gitlab::Access + include Presentable attr_accessor :raw_invite_token diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index e232feaeada..c39789b047d 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -7,6 +7,8 @@ class MergeRequest < ActiveRecord::Base include TimeTrackable include ManualInverseAssociation include EachBatch + include ThrottledTouch + include Gitlab::Utils::StrongMemoize ignore_column :locked_at, :ref_fetched @@ -51,6 +53,7 @@ class MergeRequest < ActiveRecord::Base serialize :merge_params, Hash # rubocop:disable Cop/ActiveRecordSerialize after_create :ensure_merge_request_diff, unless: :importing? + after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed # When this attribute is true some MR validation is ignored @@ -82,6 +85,14 @@ class MergeRequest < ActiveRecord::Base transition locked: :opened end + before_transition any => :opened do |merge_request| + merge_request.merge_jid = nil + + merge_request.run_after_commit do + UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id) + end + end + state :opened state :closed state :merged @@ -145,6 +156,13 @@ class MergeRequest < ActiveRecord::Base '!' end + # Use this method whenever you need to make sure the head_pipeline is synced with the + # branch head commit, for example checking if a merge request can be merged. + # For more information check: https://gitlab.com/gitlab-org/gitlab-ce/issues/40004 + def actual_head_pipeline + head_pipeline&.sha == diff_head_sha ? head_pipeline : nil + end + # Pattern used to extract `!123` merge request references from text # # This pattern supports cross-project references. @@ -379,13 +397,17 @@ class MergeRequest < ActiveRecord::Base end def source_branch_head - return unless source_project - - source_project.repository.commit(source_branch_ref) if source_branch_ref + strong_memoize(:source_branch_head) do + if source_project && source_branch_ref + source_project.repository.commit(source_branch_ref) + end + end end def target_branch_head - target_project.repository.commit(target_branch_ref) + strong_memoize(:target_branch_head) do + target_project.repository.commit(target_branch_ref) + end end def branch_merge_base_commit @@ -517,6 +539,13 @@ class MergeRequest < ActiveRecord::Base end end + def clear_memoized_shas + @target_branch_sha = @source_branch_sha = nil + + clear_memoization(:source_branch_head) + clear_memoization(:target_branch_head) + end + def reload_diff_if_branch_changed if (source_branch_changed? || target_branch_changed?) && (source_branch_head && target_branch_head) @@ -641,6 +670,7 @@ class MergeRequest < ActiveRecord::Base .to_sql Note.from("(#{union}) #{Note.table_name}") + .includes(:noteable) end alias_method :discussion_notes, :related_notes @@ -822,8 +852,9 @@ class MergeRequest < ActiveRecord::Base def mergeable_ci_state? return true unless project.only_allow_merge_if_pipeline_succeeds? + return true unless head_pipeline - !head_pipeline || head_pipeline.success? || head_pipeline.skipped? + actual_head_pipeline&.success? || actual_head_pipeline&.skipped? end def environments_for(current_user) @@ -856,11 +887,11 @@ class MergeRequest < ActiveRecord::Base def state_icon_name if merged? - "check" + "git-merge" elsif closed? - "times" + "close" else - "circle-o" + "issue-open-m" end end @@ -899,7 +930,8 @@ class MergeRequest < ActiveRecord::Base def compute_diverged_commits_count return 0 unless source_branch_sha && target_branch_sha - Gitlab::Git::Commit.between(target_project.repository.raw_repository, source_branch_sha, target_branch_sha).size + target_project.repository + .count_commits_between(source_branch_sha, target_branch_sha) end private :compute_diverged_commits_count @@ -915,21 +947,27 @@ class MergeRequest < ActiveRecord::Base .order(id: :desc) end - # Note that this could also return SHA from now dangling commits - # - def all_commit_shas - return commit_shas unless persisted? - - diffs_relation = merge_request_diffs - + def all_commits # MySQL doesn't support LIMIT in a subquery. - diffs_relation = diffs_relation.recent if Gitlab::Database.postgresql? + diffs_relation = if Gitlab::Database.postgresql? + merge_request_diffs.recent + else + merge_request_diffs + end MergeRequestDiffCommit .where(merge_request_diff: diffs_relation) .limit(10_000) - .pluck('sha') - .uniq + end + + # Note that this could also return SHA from now dangling commits + # + def all_commit_shas + @all_commit_shas ||= begin + return commit_shas unless persisted? + + all_commits.pluck(:sha).uniq + end end def merge_commit @@ -996,7 +1034,7 @@ class MergeRequest < ActiveRecord::Base return true if autocomplete_precheck return false unless mergeable?(skip_ci_check: true) - return false if head_pipeline && !(head_pipeline.success? || head_pipeline.active?) + return false if actual_head_pipeline && !(actual_head_pipeline.success? || actual_head_pipeline.active?) return false if last_diff_sha != diff_head_sha true diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index c37aa0a594b..e35de9b97ee 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -104,19 +104,19 @@ class MergeRequestDiff < ActiveRecord::Base def base_commit return unless base_commit_sha - project.commit(base_commit_sha) + project.commit_by(oid: base_commit_sha) end def start_commit return unless start_commit_sha - project.commit(start_commit_sha) + project.commit_by(oid: start_commit_sha) end def head_commit return unless head_commit_sha - project.commit(head_commit_sha) + project.commit_by(oid: head_commit_sha) end def commit_shas diff --git a/app/models/milestone.rb b/app/models/milestone.rb index c06ee8083f0..77c19380e66 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -84,6 +84,13 @@ class Milestone < ActiveRecord::Base else milestones.active end end + + def predefined?(milestone) + milestone == Any || + milestone == None || + milestone == Upcoming || + milestone == Started + end end def self.reference_prefix diff --git a/app/models/namespace.rb b/app/models/namespace.rb index fa76729a702..0ff169d4531 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -40,6 +40,7 @@ class Namespace < ActiveRecord::Base namespace_path: true validate :nesting_level_allowed + validate :allowed_path_by_redirects delegate :name, to: :owner, allow_nil: true, prefix: true @@ -139,7 +140,17 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) return nil unless project.fork_network - project.fork_network.find_forks_in(projects).first + if RequestStore.active? + forks_in_namespace = RequestStore.fetch("namespaces:#{id}:forked_projects") do + Hash.new do |found_forks, project| + found_forks[project] = project.fork_network.find_forks_in(projects).first + end + end + + forks_in_namespace[project] + else + project.fork_network.find_forks_in(projects).first + end end def lfs_enabled? @@ -247,4 +258,14 @@ class Namespace < ActiveRecord::Base Namespace.where(id: descendants.select(:id)) .update_all(share_with_group_lock: true) end + + def allowed_path_by_redirects + return if path.nil? + + errors.add(:path, "#{path} has been taken before. Please use another one") if namespace_previously_created_with_same_path? + end + + def namespace_previously_created_with_same_path? + RedirectRoute.permanent.exists?(path: path) + end end diff --git a/app/models/note.rb b/app/models/note.rb index 340fe087f82..184fbd5f5ae 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -15,6 +15,7 @@ class Note < ActiveRecord::Base include IgnorableColumn include Editable include Gitlab::SQL::Pattern + include ThrottledTouch module SpecialRole FIRST_TIME_CONTRIBUTOR = :first_time_contributor @@ -55,7 +56,7 @@ class Note < ActiveRecord::Base participant :author belongs_to :project - belongs_to :noteable, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :author, class_name: "User" belongs_to :updated_by, class_name: "User" belongs_to :last_edited_by, class_name: 'User' @@ -118,6 +119,7 @@ class Note < ActiveRecord::Base before_validation :set_discussion_id, on: :create after_save :keep_around_commit, if: :for_project_noteable? after_save :expire_etag_cache + after_save :touch_noteable after_destroy :expire_etag_cache class << self @@ -228,16 +230,18 @@ class Note < ActiveRecord::Base for_personal_snippet? end + def commit + @commit ||= project.commit(commit_id) if commit_id.present? + end + # override to return commits, which are not active record def noteable - if for_commit? - @commit ||= project.commit(commit_id) - else - super - end - # Temp fix to prevent app crash - # if note commit id doesn't exist + return commit if for_commit? + + super rescue + # Temp fix to prevent app crash + # if note commit id doesn't exist nil end @@ -356,6 +360,16 @@ class Note < ActiveRecord::Base end end + def references + refs = [noteable] + + if part_of_discussion? + refs += discussion.notes.take_while { |n| n.id < id } + end + + refs + end + def expire_etag_cache return unless noteable&.discussions_rendered_on_frontend? @@ -367,6 +381,45 @@ class Note < ActiveRecord::Base Gitlab::EtagCaching::Store.new.touch(key) end + def touch(*args) + # We're not using an explicit transaction here because this would in all + # cases result in all future queries going to the primary, even if no writes + # are performed. + # + # We touch the noteable first so its SELECT query can run before our writes, + # ensuring it runs on a secondary (if no prior write took place). + touch_noteable + super + end + + # By default Rails will issue an "SELECT *" for the relation, which is + # overkill for just updating the timestamps. To work around this we manually + # touch the data so we can SELECT only the columns we need. + def touch_noteable + # Commits are not stored in the DB so we can't touch them. + return if for_commit? + + assoc = association(:noteable) + + noteable_object = + if assoc.loaded? + noteable + else + # If the object is not loaded (e.g. when notes are loaded async) we + # _only_ want the data we actually need. + assoc.scope.select(:id, :updated_at).take + end + + noteable_object&.touch + + # We return the noteable object so we can re-use it in EE for ElasticSearch. + noteable_object + end + + def banzai_render_context(field) + super.merge(noteable: noteable) + end + private def keep_around_commit diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index cfcb03138b7..063dc521324 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -3,6 +3,8 @@ class PersonalAccessToken < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :token + REDIS_EXPIRY_TIME = 3.minutes + serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize belongs_to :user @@ -27,6 +29,21 @@ class PersonalAccessToken < ActiveRecord::Base !revoked? && !expired? end + def self.redis_getdel(user_id) + Gitlab::Redis::SharedState.with do |redis| + token = redis.get(redis_shared_state_key(user_id)) + redis.del(redis_shared_state_key(user_id)) + token + end + end + + def self.redis_store!(user_id, token) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_shared_state_key(user_id), token, ex: REDIS_EXPIRY_TIME) + token + end + end + protected def validate_scopes @@ -38,4 +55,8 @@ class PersonalAccessToken < ActiveRecord::Base def set_default_scopes self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty? end + + def self.redis_shared_state_key(user_id) + "gitlab:personal_access_token:#{user_id}" + end end diff --git a/app/models/project.rb b/app/models/project.rb index 5a3f591c2e7..5183a216c53 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -189,7 +189,6 @@ class Project < ActiveRecord::Base has_one :statistics, class_name: 'ProjectStatistics' has_one :cluster_project, class_name: 'Clusters::Project' - has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster' has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, @@ -228,14 +227,13 @@ class Project < ActiveRecord::Base delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team - delegate :empty_repo?, to: :repository # Validations validates :creator, presence: true, on: :create validates :description, length: { maximum: 2000 }, allow_blank: true validates :ci_config_path, - format: { without: /\.{2}/, - message: 'cannot include directory traversal.' }, + format: { without: /(\.{2}|\A\/)/, + message: 'cannot include leading slash or directory traversal.' }, length: { maximum: 255 }, allow_blank: true validates :name, @@ -500,6 +498,10 @@ class Project < ActiveRecord::Base auto_devops&.enabled.nil? && !current_application_settings.auto_devops_enabled? end + def empty_repo? + repository.empty? + end + def repository_storage_path Gitlab.config.repositories.storages[repository_storage].try(:[], 'path') end @@ -562,8 +564,7 @@ class Project < ActiveRecord::Base if forked? RepositoryForkWorker.perform_async(id, forked_from_project.repository_storage_path, - forked_from_project.full_path, - self.namespace.full_path) + forked_from_project.disk_path) else RepositoryImportWorker.perform_async(self.id) end @@ -599,7 +600,7 @@ class Project < ActiveRecord::Base def ci_config_path=(value) # Strip all leading slashes so that //foo -> foo - super(value&.sub(%r{\A/+}, '')&.delete("\0")) + super(value&.delete("\0")) end def import_url=(value) @@ -658,7 +659,8 @@ class Project < ActiveRecord::Base end def import_started? - import? && import_status == 'started' + # import? does SQL work so only run it if it looks like there's an import running + import_status == 'started' && import? end def import_scheduled? @@ -753,13 +755,14 @@ class Project < ActiveRecord::Base Gitlab::Routing.url_helpers.project_url(self) end - def new_issue_address(author) + def new_issuable_address(author, address_type) return unless Gitlab::IncomingEmail.supports_issue_creation? && author author.ensure_incoming_email_token! + suffix = address_type == 'merge_request' ? '+merge-request' : '' Gitlab::IncomingEmail.reply_address( - "#{full_path}+#{author.incoming_email_token}") + "#{full_path}#{suffix}+#{author.incoming_email_token}") end def build_commit_note(commit) @@ -897,12 +900,10 @@ class Project < ActiveRecord::Base @ci_service ||= ci_services.reorder(nil).find_by(active: true) end - def deployment_services - services.where(category: :deployment) - end - - def deployment_service - @deployment_service ||= deployment_services.reorder(nil).find_by(active: true) + # TODO: This will be extended for multiple enviroment clusters + def deployment_platform + @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes + @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true) end def monitoring_services @@ -1116,7 +1117,11 @@ class Project < ActiveRecord::Base end def project_member(user) - project_members.find_by(user_id: user) + if project_members.loaded? + project_members.find { |member| member.user_id == user.id } + else + project_members.find_by(user_id: user) + end end def default_branch @@ -1143,7 +1148,7 @@ class Project < ActiveRecord::Base def change_head(branch) if repository.branch_exists?(branch) repository.before_change_head - repository.write_ref('HEAD', "refs/heads/#{branch}") + repository.write_ref('HEAD', "refs/heads/#{branch}", force: true) repository.copy_gitattributes(branch) repository.after_change_head reload_default_branch @@ -1547,9 +1552,9 @@ class Project < ActiveRecord::Base end def deployment_variables - return [] unless deployment_service + return [] unless deployment_platform - deployment_service.predefined_variables + deployment_platform.predefined_variables end def auto_devops_variables diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index bc62972dbb0..b82567ce2b3 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -1,3 +1,8 @@ +## +# NOTE: +# We'll move this class to Clusters::Platforms::Kubernetes, which contains exactly the same logic. +# After we've migrated data, we'll remove KubernetesService. This would happen in a few months. +# If you're modyfiyng this class, please note that you should update the same change in Clusters::Platforms::Kubernetes. class KubernetesService < DeploymentService include Gitlab::CurrentSettings include Gitlab::Kubernetes diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 715b215d1db..17b9d2cf7b4 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -35,7 +35,9 @@ class ProjectStatistics < ActiveRecord::Base end def update_build_artifacts_size - self.build_artifacts_size = project.builds.sum(:artifacts_size) + self.build_artifacts_size = + project.builds.sum(:artifacts_size) + + Ci::JobArtifact.artifacts_size_for(self) end def update_storage_size diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 1d35426050e..c679758973a 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -1,4 +1,6 @@ class ProjectTeam + include BulkMemberAccessLoad + attr_accessor :project def initialize(project) @@ -157,39 +159,16 @@ class ProjectTeam # # Returns a Hash mapping user ID -> maximum access level. def max_member_access_for_user_ids(user_ids) - user_ids = user_ids.uniq - key = "max_member_access:#{project.id}" - - access = {} - - if RequestStore.active? - RequestStore.store[key] ||= {} - access = RequestStore.store[key] + max_member_access_for_resource_ids(User, user_ids, project.id) do |user_ids| + project.project_authorizations + .where(user: user_ids) + .group(:user_id) + .maximum(:access_level) end - - # Look up only the IDs we need - user_ids = user_ids - access.keys - - return access if user_ids.empty? - - users_access = project.project_authorizations - .where(user: user_ids) - .group(:user_id) - .maximum(:access_level) - - access.merge!(users_access) - - missing_user_ids = user_ids - users_access.keys - - missing_user_ids.each do |user_id| - access[user_id] = Gitlab::Access::NO_ACCESS - end - - access end def max_member_access(user_id) - max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS + max_member_access_for_user_ids([user_id])[user_id] end private diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 89bfc5f9a9c..d28fed11ca8 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -10,7 +10,9 @@ class ProtectedBranch < ActiveRecord::Base def self.protected?(project, ref_name) return true if project.empty_repo? && default_branch_protected? - self.matching(ref_name, protected_refs: project.protected_branches).present? + refs = project.protected_branches.select(:name) + + self.matching(ref_name, protected_refs: refs).present? end def self.default_branch_protected? diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb index f38109c0e52..42a9bcf7723 100644 --- a/app/models/protected_tag.rb +++ b/app/models/protected_tag.rb @@ -5,6 +5,8 @@ class ProtectedTag < ActiveRecord::Base protected_ref_access_levels :create def self.protected?(project, ref_name) - self.matching(ref_name, protected_refs: project.protected_tags).present? + refs = project.protected_tags.select(:name) + + self.matching(ref_name, protected_refs: refs).present? end end diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb index 31de204d824..20532527346 100644 --- a/app/models/redirect_route.rb +++ b/app/models/redirect_route.rb @@ -17,4 +17,32 @@ class RedirectRoute < ActiveRecord::Base where(wheres, path, "#{sanitize_sql_like(path)}/%") end + + scope :permanent, -> do + if column_permanent_exists? + where(permanent: true) + else + none + end + end + + scope :temporary, -> do + if column_permanent_exists? + where(permanent: [false, nil]) + else + all + end + end + + default_value_for :permanent, false + + def permanent=(value) + if self.class.column_permanent_exists? + super + end + end + + def self.column_permanent_exists? + ActiveRecord::Base.connection.column_exists?(:redirect_routes, :permanent) + end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 165dafd83fd..552a354d1ce 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -19,6 +19,7 @@ class Repository attr_accessor :full_path, :disk_path, :project, :is_wiki delegate :ref_name_for_sha, to: :raw_repository + delegate :write_ref, to: :raw_repository CreateTreeError = Class.new(StandardError) @@ -37,7 +38,7 @@ class Repository issue_template_names merge_request_template_names).freeze # Methods that use cache_method but only memoize the value - MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze + MEMOIZED_CACHED_METHODS = %i(license).freeze # Certain method caches should be refreshed when certain types of files are # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to @@ -237,11 +238,10 @@ class Repository # This will still fail if the file is corrupted (e.g. 0 bytes) begin - write_ref(keep_around_ref_name(sha), sha) - rescue Rugged::ReferenceError => ex - Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" - rescue Rugged::OSError => ex - raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ + write_ref(keep_around_ref_name(sha), sha, force: true) + rescue Gitlab::Git::Repository::GitError => ex + # Necessary because https://gitlab.com/gitlab-org/gitlab-ce/issues/20156 + return true if ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}" end @@ -251,12 +251,8 @@ class Repository ref_exists?(keep_around_ref_name(sha)) end - def write_ref(ref_path, sha) - rugged.references.create(ref_path, sha, force: true) - end - def diverging_commit_counts(branch) - root_ref_hash = raw_repository.rev_parse_target(root_ref).oid + root_ref_hash = raw_repository.commit(root_ref).id cache.fetch(:"diverging_commit_counts_#{branch.name}") do # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes @@ -497,7 +493,11 @@ class Repository end cache_method :exists? - delegate :empty?, to: :raw_repository + def empty? + return true unless exists? + + !has_visible_content? + end cache_method :empty? # The size of this repository in megabytes. @@ -686,7 +686,9 @@ class Repository def tags_sorted_by(value) case value - when 'name' + when 'name_asc' + VersionSorter.sort(tags) { |tag| tag.name } + when 'name_desc' VersionSorter.rsort(tags) { |tag| tag.name } when 'updated_desc' tags_sorted_by_committed_date.reverse @@ -697,10 +699,14 @@ class Repository end end - def contributors + # Params: + # + # order_by: name|email|commits + # sort: asc|desc default: 'asc' + def contributors(order_by: nil, sort: 'asc') commits = self.commits(nil, limit: 2000, offset: 0, skip_merges: true) - commits.group_by(&:author_email).map do |email, commits| + commits = commits.group_by(&:author_email).map do |email, commits| contributor = Gitlab::Contributor.new contributor.email = email @@ -714,6 +720,7 @@ class Repository contributor end + Commit.order_by(collection: commits, order_by: order_by, sort: sort) end def refs_contains_sha(ref_type, sha) @@ -927,7 +934,7 @@ class Repository def merge_base(first_commit_id, second_commit_id) first_commit_id = commit(first_commit_id).try(:id) || first_commit_id second_commit_id = commit(second_commit_id).try(:id) || second_commit_id - rugged.merge_base(first_commit_id, second_commit_id) + raw_repository.merge_base(first_commit_id, second_commit_id) rescue Rugged::ReferenceError nil end @@ -944,13 +951,8 @@ class Repository end end - def empty_repo? - !exists? || !has_visible_content? - end - cache_method :empty_repo?, memoize_only: true - def search_files_by_content(query, ref) - return [] if empty_repo? || query.blank? + return [] if empty? || query.blank? offset = 2 args = %W(grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref}) @@ -959,7 +961,7 @@ class Repository end def search_files_by_name(query, ref) - return [] if empty_repo? || query.blank? + return [] if empty? || query.blank? args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)}) @@ -972,8 +974,7 @@ class Repository tmp_remote_name = true end - add_remote(remote_name, url) - set_remote_as_mirror(remote_name, refmap: refmap) + add_remote(remote_name, url, mirror_refmap: refmap) fetch_remote(remote_name, forced: forced) ensure remove_remote(remote_name) if tmp_remote_name @@ -996,7 +997,7 @@ class Repository end def create_ref(ref, ref_path) - raw_repository.write_ref(ref_path, ref) + write_ref(ref_path, ref) end def ls_files(ref) diff --git a/app/models/route.rb b/app/models/route.rb index 97e8a6ad9e9..7ba3ec06041 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -8,6 +8,8 @@ class Route < ActiveRecord::Base presence: true, uniqueness: { case_sensitive: false } + validate :ensure_permanent_paths + after_create :delete_conflicting_redirects after_update :delete_conflicting_redirects, if: :path_changed? after_update :create_redirect_for_old_path @@ -40,7 +42,7 @@ class Route < ActiveRecord::Base # We are not calling route.delete_conflicting_redirects here, in hopes # of avoiding deadlocks. The parent (self, in this method) already # called it, which deletes conflicts for all descendants. - route.create_redirect(old_path) if attributes[:path] + route.create_redirect(old_path, permanent: permanent_redirect?) if attributes[:path] end end end @@ -50,16 +52,30 @@ class Route < ActiveRecord::Base end def conflicting_redirects - RedirectRoute.matching_path_and_descendants(path) + RedirectRoute.temporary.matching_path_and_descendants(path) end - def create_redirect(path) - RedirectRoute.create(source: source, path: path) + def create_redirect(path, permanent: false) + RedirectRoute.create(source: source, path: path, permanent: permanent) end private def create_redirect_for_old_path - create_redirect(path_was) if path_changed? + create_redirect(path_was, permanent: permanent_redirect?) if path_changed? + end + + def permanent_redirect? + source_type != "Project" + end + + def ensure_permanent_paths + return if path.nil? + + errors.add(:path, "#{path} has been taken before. Please use another one") if conflicting_redirect_exists? + end + + def conflicting_redirect_exists? + RedirectRoute.permanent.matching_path_and_descendants(path).exists? end end diff --git a/app/models/service.rb b/app/models/service.rb index fdd2605e3e3..3c4f1885dd0 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -211,7 +211,7 @@ class Service < ActiveRecord::Base def async_execute(data) return unless supported_events.include?(data[:object_kind]) - Sidekiq::Client.enqueue(ProjectServiceWorker, id, data) + ProjectServiceWorker.perform_async(id, data) end def issue_tracker? diff --git a/app/models/user.rb b/app/models/user.rb index 14941fd7f98..92b461ce3ed 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -7,6 +7,7 @@ class User < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::CurrentSettings include Gitlab::SQL::Pattern + include AfterCommitQueue include Avatarable include Referable include Sortable @@ -16,6 +17,7 @@ class User < ActiveRecord::Base include FeatureGate include CreatedAtFilterable include IgnorableColumn + include BulkMemberAccessLoad DEFAULT_NOTIFICATION_LEVEL = :participating @@ -313,6 +315,8 @@ class User < ActiveRecord::Base # # Returns an ActiveRecord::Relation. def search(query) + query = query.downcase + order = <<~SQL CASE WHEN users.name = %{query} THEN 0 @@ -322,8 +326,11 @@ class User < ActiveRecord::Base END SQL - fuzzy_search(query, [:name, :email, :username]) - .reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) + where( + fuzzy_arel_match(:name, query) + .or(fuzzy_arel_match(:username, query)) + .or(arel_table[:email].eq(query)) + ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end # searches user by given pattern @@ -331,15 +338,17 @@ class User < ActiveRecord::Base # This method uses ILIKE on PostgreSQL and LIKE on MySQL. def search_with_secondary_emails(query) + query = query.downcase + email_table = Email.arel_table matched_by_emails_user_ids = email_table .project(email_table[:user_id]) - .where(Email.fuzzy_arel_match(:email, query)) + .where(email_table[:email].eq(query)) where( fuzzy_arel_match(:name, query) - .or(fuzzy_arel_match(:email, query)) .or(fuzzy_arel_match(:username, query)) + .or(arel_table[:email].eq(query)) .or(arel_table[:id].in(matched_by_emails_user_ids)) ) end @@ -487,7 +496,11 @@ class User < ActiveRecord::Base end def two_factor_u2f_enabled? - u2f_registrations.exists? + if u2f_registrations.loaded? + u2f_registrations.any? + else + u2f_registrations.exists? + end end def namespace_uniq @@ -899,6 +912,7 @@ class User < ActiveRecord::Base def post_destroy_hook log_info("User \"#{name}\" (#{email}) was removed") + system_hook_service.execute_hooks_for(self, :destroy) end @@ -998,7 +1012,11 @@ class User < ActiveRecord::Base end def notification_settings_for(source) - notification_settings.find_or_initialize_by(source: source) + if notification_settings.loaded? + notification_settings.find { |notification| notification.source == source } + else + notification_settings.find_or_initialize_by(source: source) + end end # Lazy load global notification setting @@ -1043,13 +1061,13 @@ class User < ActiveRecord::Base end def todos_done_count(force: false) - Rails.cache.fetch(['users', id, 'todos_done_count'], force: force) do + Rails.cache.fetch(['users', id, 'todos_done_count'], force: force, expires_in: 20.minutes) do TodosFinder.new(self, state: :done).execute.count end end def todos_pending_count(force: false) - Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force) do + Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force, expires_in: 20.minutes) do TodosFinder.new(self, state: :pending).execute.count end end @@ -1134,6 +1152,34 @@ class User < ActiveRecord::Base super end + # Determine the maximum access level for a group of projects in bulk. + # + # Returns a Hash mapping project ID -> maximum access level. + def max_member_access_for_project_ids(project_ids) + max_member_access_for_resource_ids(Project, project_ids) do |project_ids| + project_authorizations.where(project: project_ids) + .group(:project_id) + .maximum(:access_level) + end + end + + def max_member_access_for_project(project_id) + max_member_access_for_project_ids([project_id])[project_id] + end + + # Determine the maximum access level for a group of groups in bulk. + # + # Returns a Hash mapping project ID -> maximum access level. + def max_member_access_for_group_ids(group_ids) + max_member_access_for_resource_ids(Group, group_ids) do |group_ids| + group_members.where(source: group_ids).group(:source_id).maximum(:access_level) + end + end + + def max_member_access_for_group(group_id) + max_member_access_for_group_ids([group_id])[group_id] + end + protected # override, from Devise::Validatable |