Merge commit '7fabc892f251740dbd9a4755baede662e6854870' into object-storage-ee-to-ce-backport

68 files changed, 1256 insertions, 500 deletions

diff --git a/app/models/appearance.rb b/app/models/appearance.rb
index ff15689ecac..76cfe28742a 100644
--- a/app/models/appearance.rb
+++ b/app/models/appearance.rb
@@ -2,9 +2,8 @@ class Appearance < ActiveRecord::Base
   include CacheMarkdownField
 
   cache_markdown_field :description
+  cache_markdown_field :new_project_guidelines
 
-  validates :title,       presence: true
-  validates :description, presence: true
   validates :logo,        file_size: { maximum: 1.megabyte }
   validates :header_logo, file_size: { maximum: 1.megabyte }
 
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 5e16badabec..253e213af81 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -153,11 +153,10 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { greater_than_or_equal_to: 0 }
 
-  validates :circuitbreaker_backoff_threshold,
-            :circuitbreaker_failure_count_threshold,
-            :circuitbreaker_failure_wait_time,
+  validates :circuitbreaker_failure_count_threshold,
             :circuitbreaker_failure_reset_time,
             :circuitbreaker_storage_timeout,
+            :circuitbreaker_check_interval,
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
@@ -165,12 +164,26 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 1 }
 
-  validates_each :circuitbreaker_backoff_threshold do |record, attr, value|
-    if value.to_i >= record.circuitbreaker_failure_count_threshold
-      record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\
-                                "lower than the failure count threshold"))
-    end
-  end
+  validates :gitaly_timeout_default,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+
+  validates :gitaly_timeout_medium,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+  validates :gitaly_timeout_medium,
+            numericality: { less_than_or_equal_to: :gitaly_timeout_default },
+            if: :gitaly_timeout_default
+  validates :gitaly_timeout_medium,
+            numericality: { greater_than_or_equal_to: :gitaly_timeout_fast },
+            if: :gitaly_timeout_fast
+
+  validates :gitaly_timeout_fast,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+  validates :gitaly_timeout_fast,
+            numericality: { less_than_or_equal_to: :gitaly_timeout_default },
+            if: :gitaly_timeout_default
 
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
@@ -276,7 +289,8 @@ class ApplicationSetting < ActiveRecord::Base
       koding_url: nil,
       max_artifacts_size: Settings.artifacts['max_size'],
       max_attachment_size: Settings.gitlab['max_attachment_size'],
-      password_authentication_enabled: Settings.gitlab['password_authentication_enabled'],
+      password_authentication_enabled_for_web: Settings.gitlab['signin_enabled'],
+      password_authentication_enabled_for_git: true,
       performance_bar_allowed_group_id: nil,
       rsa_key_restriction: 0,
       plantuml_enabled: false,
@@ -295,10 +309,22 @@ class ApplicationSetting < ActiveRecord::Base
       sign_in_text: nil,
       signup_enabled: Settings.gitlab['signup_enabled'],
       terminal_max_session_time: 0,
+      throttle_unauthenticated_enabled: false,
+      throttle_unauthenticated_requests_per_period: 3600,
+      throttle_unauthenticated_period_in_seconds: 3600,
+      throttle_authenticated_web_enabled: false,
+      throttle_authenticated_web_requests_per_period: 7200,
+      throttle_authenticated_web_period_in_seconds: 3600,
+      throttle_authenticated_api_enabled: false,
+      throttle_authenticated_api_requests_per_period: 7200,
+      throttle_authenticated_api_period_in_seconds: 3600,
       two_factor_grace_period: 48,
       user_default_external: false,
       polling_interval_multiplier: 1,
-      usage_ping_enabled: Settings.gitlab['usage_ping_enabled']
+      usage_ping_enabled: Settings.gitlab['usage_ping_enabled'],
+      gitaly_timeout_fast: 10,
+      gitaly_timeout_medium: 30,
+      gitaly_timeout_default: 55
     }
   end
 
@@ -465,6 +491,14 @@ class ApplicationSetting < ActiveRecord::Base
     has_attribute?(attr_name) ? public_send(attr_name) : FORBIDDEN_KEY_VALUE # rubocop:disable GitlabSecurity/PublicSend
   end
 
+  def allow_signup?
+    signup_enabled? && password_authentication_enabled_for_web?
+  end
+
+  def password_authentication_enabled?
+    password_authentication_enabled_for_web? || password_authentication_enabled_for_git?
+  end
+
   private
 
   def ensure_uuid!
diff --git a/app/models/blob.rb b/app/models/blob.rb
index ad0bc2e2ead..29e762724e3 100644
--- a/app/models/blob.rb
+++ b/app/models/blob.rb
@@ -76,12 +76,24 @@ class Blob < SimpleDelegator
     new(blob, project)
   end
 
+  def self.lazy(project, commit_id, path)
+    BatchLoader.for(commit_id: commit_id, path: path).batch do |items, loader|
+      project.repository.blobs_at(items.map(&:values)).each do |blob|
+        loader.call({ commit_id: blob.commit_id, path: blob.path }, blob) if blob
+      end
+    end
+  end
+
   def initialize(blob, project = nil)
     @project = project
 
     super(blob)
   end
 
+  def inspect
+    "#<#{self.class.name} oid:#{id[0..8]} commit:#{commit_id[0..8]} path:#{path}>"
+  end
+
   # Returns the data of the blob.
   #
   # If the blob is a text based blob the content is converted to UTF-8 and any
@@ -95,7 +107,10 @@ class Blob < SimpleDelegator
   end
 
   def load_all_data!
-    super(project.repository) if project
+    # Endpoint needed: gitlab-org/gitaly#756
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      super(project.repository) if project
+    end
   end
 
   def no_highlighting?
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index ebeab87d8fc..ddd075e1fcb 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -1,18 +1,26 @@
 module Ci
   class Build < CommitStatus
+    prepend ArtifactMigratable
     include TokenAuthenticatable
     include AfterCommitQueue
     include Presentable
     include Importable
 
+    MissingDependenciesError = Class.new(StandardError)
+
     belongs_to :runner
     belongs_to :trigger_request
     belongs_to :erased_by, class_name: 'User'
 
     has_many :deployments, as: :deployable
+
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
 
+    has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+    has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
       @persisted_environment ||= Environment.find_by(
@@ -31,7 +39,10 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
-    scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) }
+    scope :with_artifacts, ->() do
+      where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
+        '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id'))
+    end
     scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
     scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, ArtifactUploader::LOCAL_STORE]) }
@@ -39,8 +50,27 @@ module Ci
     scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
     scope :ref_protected, -> { where(protected: true) }
 
-    mount_uploader :artifacts_file, ArtifactUploader
-    mount_uploader :artifacts_metadata, ArtifactUploader
+    scope :matches_tag_ids, -> (tag_ids) do
+      matcher = ::ActsAsTaggableOn::Tagging
+        .where(taggable_type: CommitStatus)
+        .where(context: 'tags')
+        .where('taggable_id = ci_builds.id')
+        .where.not(tag_id: tag_ids).select('1')
+
+      where("NOT EXISTS (?)", matcher)
+    end
+
+    scope :with_any_tags, -> do
+      matcher = ::ActsAsTaggableOn::Tagging
+        .where(taggable_type: CommitStatus)
+        .where(context: 'tags')
+        .where('taggable_id = ci_builds.id').select('1')
+
+      where("EXISTS (?)", matcher)
+    end
+
+    mount_uploader :legacy_artifacts_file, LegacyArtifactUploader, mount_on: :artifacts_file
+    mount_uploader :legacy_artifacts_metadata, LegacyArtifactUploader, mount_on: :artifacts_metadata
 
     acts_as_taggable
 
@@ -105,12 +135,17 @@ module Ci
       end
 
       before_transition any => [:failed] do |build|
+        next unless build.project
         next if build.retries_max.zero?
 
         if build.retries_count < build.retries_max
           Ci::Build.retry(build, build.user)
         end
       end
+
+      before_transition any => [:running] do |build|
+        build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies')
+      end
     end
 
     def detailed_status(current_user)
@@ -193,6 +228,10 @@ module Ci
       project.build_timeout
     end
 
+    def triggered_by?(current_user)
+      user == current_user
+    end
+
     # A slugified version of the build ref, suitable for inclusion in URLs and
     # domain names. Rules:
     #
@@ -240,7 +279,7 @@ module Ci
 
       @merge_request ||=
         begin
-          merge_requests = MergeRequest.includes(:merge_request_diff)
+          merge_requests = MergeRequest.includes(:latest_merge_request_diff)
             .where(source_branch: ref,
                    source_project: pipeline.project)
             .reorder(iid: :desc)
@@ -314,6 +353,7 @@ module Ci
 
     def execute_hooks
       return unless project
+
       build_data = Gitlab::DataBuilder::Build.build(self)
       project.execute_hooks(build_data.dup, :job_hooks)
       project.execute_services(build_data.dup, :job_hooks)
@@ -391,6 +431,7 @@ module Ci
 
     def keep_artifacts!
       self.update(artifacts_expire_at: nil)
+      self.job_artifacts.update_all(expire_at: nil)
     end
 
     def coverage_regex
@@ -462,6 +503,20 @@ module Ci
       options[:dependencies]&.empty?
     end
 
+    def validates_dependencies!
+      dependencies.each do |dependency|
+        raise MissingDependenciesError unless dependency.valid_dependency?
+      end
+    end
+
+    def valid_dependency?
+      return false unless complete?
+      return false if artifacts_expired?
+      return false if erased?
+
+      true
+    end
+
     def hide_secrets(trace)
       return unless trace
 
@@ -478,11 +533,7 @@ module Ci
     private
 
     def update_artifacts_size
-      self.artifacts_size = if artifacts_file.exists?
-                              artifacts_file.size
-                            else
-                              nil
-                            end
+      self.artifacts_size = legacy_artifacts_file&.size
     end
 
     def erase_trace!
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
new file mode 100644
index 00000000000..84fc6863567
--- /dev/null
+++ b/app/models/ci/job_artifact.rb
@@ -0,0 +1,36 @@
+module Ci
+  class JobArtifact < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :project
+    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
+
+    before_save :set_size, if: :file_changed?
+
+    mount_uploader :file, JobArtifactUploader
+
+    enum file_type: {
+      archive: 1,
+      metadata: 2
+    }
+
+    def self.artifacts_size_for(project)
+      self.where(project: project).sum(:size)
+    end
+
+    def set_size
+      self.size = file.size
+    end
+
+    def expire_in
+      expire_at - Time.now if expire_at
+    end
+
+    def expire_in=(value)
+      self.expire_at =
+        if value
+          ChronicDuration.parse(value)&.seconds&.from_now
+        end
+    end
+  end
+end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index ca65e81f27a..eebbf7c4218 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -40,7 +40,6 @@ module Ci
     validates :status, presence: { unless: :importing? }
     validate :valid_commit_sha, unless: :importing?
 
-    after_initialize :set_config_source, if: :new_record?
     after_create :keep_around_commits, unless: :importing?
 
     enum source: {
@@ -66,8 +65,8 @@ module Ci
 
     state_machine :status, initial: :created do
       event :enqueue do
-        transition created: :pending
-        transition [:success, :failed, :canceled, :skipped] => :running
+        transition [:created, :skipped] => :pending
+        transition [:success, :failed, :canceled] => :running
       end
 
       event :run do
@@ -149,34 +148,70 @@ module Ci
       end
     end
 
-    # ref can't be HEAD or SHA, can only be branch/tag name
-    scope :latest, ->(ref = nil) do
-      max_id = unscope(:select)
-        .select("max(#{quoted_table_name}.id)")
-        .group(:ref, :sha)
+    scope :internal, -> { where(source: internal_sources) }
 
-      if ref
-        where(ref: ref, id: max_id.where(ref: ref))
-      else
-        where(id: max_id)
-      end
+    # Returns the pipelines in descending order (= newest first), optionally
+    # limited to a number of references.
+    #
+    # ref - The name (or names) of the branch(es)/tag(s) to limit the list of
+    #       pipelines to.
+    def self.newest_first(ref = nil)
+      relation = order(id: :desc)
+
+      ref ? relation.where(ref: ref) : relation
     end
-    scope :internal, -> { where(source: internal_sources) }
 
     def self.latest_status(ref = nil)
-      latest(ref).status
+      newest_first(ref).pluck(:status).first
     end
 
     def self.latest_successful_for(ref)
-      success.latest(ref).order(id: :desc).first
+      newest_first(ref).success.take
     end
 
     def self.latest_successful_for_refs(refs)
-      success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash|
+      relation = newest_first(refs).success
+
+      relation.each_with_object({}) do |pipeline, hash|
         hash[pipeline.ref] ||= pipeline
       end
     end
 
+    # Returns a Hash containing the latest pipeline status for every given
+    # commit.
+    #
+    # The keys of this Hash are the commit SHAs, the values the statuses.
+    #
+    # commits - The list of commit SHAs to get the status for.
+    # ref - The ref to scope the data to (e.g. "master"). If the ref is not
+    #       given we simply get the latest status for the commits, regardless
+    #       of what refs their pipelines belong to.
+    def self.latest_status_per_commit(commits, ref = nil)
+      p1 = arel_table
+      p2 = arel_table.alias
+
+      # This LEFT JOIN will filter out all but the newest row for every
+      # combination of (project_id, sha) or (project_id, sha, ref) if a ref is
+      # given.
+      cond = p1[:sha].eq(p2[:sha])
+        .and(p1[:project_id].eq(p2[:project_id]))
+        .and(p1[:id].lt(p2[:id]))
+
+      cond = cond.and(p1[:ref].eq(p2[:ref])) if ref
+      join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond)
+
+      relation = select(:sha, :status)
+        .where(sha: commits)
+        .where(p2[:id].eq(nil))
+        .joins(join.join_sources)
+
+      relation = relation.where(ref: ref) if ref
+
+      relation.each_with_object({}) do |row, hash|
+        hash[row[:sha]] = row[:status]
+      end
+    end
+
     def self.truncate_sha(sha)
       sha[0...8]
     end
@@ -300,8 +335,10 @@ module Ci
 
     def latest?
       return false unless ref
+
       commit = project.commit(ref)
       return false unless commit
+
       commit.sha == sha
     end
 
@@ -327,7 +364,7 @@ module Ci
     end
 
     def has_kubernetes_active?
-      project.kubernetes_service&.active?
+      project.deployment_platform&.active?
     end
 
     def has_stage_seeds?
@@ -409,7 +446,7 @@ module Ci
     end
 
     def notes
-      Note.for_commit_id(sha)
+      project.notes.for_commit_id(sha)
     end
 
     def process!
@@ -469,7 +506,10 @@ module Ci
     end
 
     def latest_builds_with_artifacts
-      @latest_builds_with_artifacts ||= builds.latest.with_artifacts
+      # We purposely cast the builds to an Array here. Because we always use the
+      # rows if there are more than 0 this prevents us from having to run two
+      # queries: one to get the count and one to get the rows.
+      @latest_builds_with_artifacts ||= builds.latest.with_artifacts.to_a
     end
 
     private
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index c6509f89117..dcbb397fb78 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -1,6 +1,7 @@
 module Ci
   class Runner < ActiveRecord::Base
     extend Gitlab::Ci::Model
+    include Gitlab::SQL::Pattern
 
     RUNNER_QUEUE_EXPIRY_TIME = 60.minutes
     ONLINE_CONTACT_TIMEOUT = 1.hour
@@ -59,10 +60,7 @@ module Ci
     #
     # Returns an ActiveRecord::Relation.
     def self.search(query)
-      t = arel_table
-      pattern = "%#{query}%"
-
-      where(t[:token].matches(pattern).or(t[:description].matches(pattern)))
+      fuzzy_search(query, [:token, :description])
     end
 
     def self.contact_time_deadline
@@ -114,7 +112,7 @@ module Ci
     def can_pick?(build)
       return false if self.ref_protected? && !build.protected?
 
-      assignable_for?(build.project) && accepting_tags?(build)
+      assignable_for?(build.project_id) && accepting_tags?(build)
     end
 
     def only_for?(project)
@@ -173,8 +171,8 @@ module Ci
       end
     end
 
-    def assignable_for?(project)
-      is_shared? || projects.exists?(id: project.id)
+    def assignable_for?(project_id)
+      is_shared? || projects.exists?(id: project_id)
     end
 
     def accepting_tags?(build)
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index 185d9473aab..55419189282 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -17,8 +17,7 @@ module Clusters
     # we force autosave to happen when we save `Cluster` model
     has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
 
-    # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration
-    has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+    has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true
 
     has_one :application_helm, class_name: 'Clusters::Applications::Helm'
     has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
@@ -29,15 +28,9 @@ module Clusters
     validates :name, cluster_name: true
     validate :restrict_modification, on: :update
 
-    # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3
-    # We need callback here because `enabled` belongs to Clusters::Cluster
-    # Callbacks in Clusters::Platforms::Kubernetes will not be called after update
-    after_save :update_kubernetes_integration!
-
     delegate :status, to: :provider, allow_nil: true
     delegate :status_reason, to: :provider, allow_nil: true
     delegate :on_creation?, to: :provider, allow_nil: true
-    delegate :update_kubernetes_integration!, to: :platform, allow_nil: true
 
     delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true
     delegate :installed?, to: :application_helm, prefix: true, allow_nil: true
@@ -62,6 +55,10 @@ module Clusters
       end
     end
 
+    def created?
+      status_name == :created
+    end
+
     def applications
       [
         application_helm || build_application_helm,
@@ -77,6 +74,10 @@ module Clusters
       return platform_kubernetes if kubernetes?
     end
 
+    def managed?
+      !user?
+    end
+
     def first_project
       return @first_project if defined?(@first_project)
 
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
index 6dc1ee810d3..9160a169452 100644
--- a/app/models/clusters/platforms/kubernetes.rb
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -1,7 +1,12 @@
 module Clusters
   module Platforms
     class Kubernetes < ActiveRecord::Base
+      include Gitlab::CurrentSettings
+      include Gitlab::Kubernetes
+      include ReactiveCaching
+
       self.table_name = 'cluster_platforms_kubernetes'
+      self.reactive_cache_key = ->(kubernetes) { [kubernetes.class.model_name.singular, kubernetes.id] }
 
       belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster'
 
@@ -29,19 +34,17 @@ module Clusters
       validates :api_url, url: true, presence: true
       validates :token, presence: true
 
-      # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes
-      after_destroy :destroy_kubernetes_integration!
+      validate :prevent_modification, on: :update
+
+      after_save :clear_reactive_cache!
 
       alias_attribute :ca_pem, :ca_cert
 
       delegate :project, to: :cluster, allow_nil: true
       delegate :enabled?, to: :cluster, allow_nil: true
+      delegate :managed?, to: :cluster, allow_nil: true
 
-      class << self
-        def namespace_for_project(project)
-          "#{project.path}-#{project.id}"
-        end
-      end
+      alias_method :active?, :enabled?
 
       def actual_namespace
         if namespace.present?
@@ -51,58 +54,138 @@ module Clusters
         end
       end
 
-      def default_namespace
-        self.class.namespace_for_project(project) if project
+      def predefined_variables
+        config = YAML.dump(kubeconfig)
+
+        variables = [
+          { key: 'KUBE_URL', value: api_url, public: true },
+          { key: 'KUBE_TOKEN', value: token, public: false },
+          { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true },
+          { key: 'KUBECONFIG', value: config, public: false, file: true }
+        ]
+
+        if ca_pem.present?
+          variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true }
+          variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true }
+        end
+
+        variables
       end
 
-      def kubeclient
-        @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service?
+      # Constructs a list of terminals from the reactive cache
+      #
+      # Returns nil if the cache is empty, in which case you should try again a
+      # short time later
+      def terminals(environment)
+        with_reactive_cache do |data|
+          pods = filter_by_label(data[:pods], app: environment.slug)
+          terminals = pods.flat_map { |pod| terminals_for_pod(api_url, actual_namespace, pod) }
+          terminals.each { |terminal| add_terminal_auth(terminal, terminal_auth) }
+        end
       end
 
-      def update_kubernetes_integration!
-        raise 'Kubernetes service already configured' unless manages_kubernetes_service?
+      # Caches resources in the namespace so other calls don't need to block on
+      # network access
+      def calculate_reactive_cache
+        return unless enabled? && project && !project.pending_delete?
 
-        # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false
-        cluster.reload
+        # We may want to cache extra things in the future
+        { pods: read_pods }
+      end
+
+      def kubeclient
+        @kubeclient ||= build_kubeclient!
+      end
+
+      private
 
-        ensure_kubernetes_service&.update!(
-          active: enabled?,
-          api_url: api_url,
-          namespace: namespace,
+      def kubeconfig
+        to_kubeconfig(
+          url: api_url,
+          namespace: actual_namespace,
           token: token,
-          ca_pem: ca_cert
-        )
+          ca_pem: ca_pem)
       end
 
-      def active?
-        manages_kubernetes_service?
+      def default_namespace
+        return unless project
+
+        slug = "#{project.path}-#{project.id}".downcase
+        slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
       end
 
-      private
+      def build_kubeclient!(api_path: 'api', api_version: 'v1')
+        raise "Incomplete settings" unless api_url && actual_namespace
 
-      def enforce_namespace_to_lower_case
-        self.namespace = self.namespace&.downcase
+        unless (username && password) || token
+          raise "Either username/password or token is required to access API"
+        end
+
+        ::Kubeclient::Client.new(
+          join_api_url(api_path),
+          api_version,
+          auth_options: kubeclient_auth_options,
+          ssl_options: kubeclient_ssl_options,
+          http_proxy_uri: ENV['http_proxy']
+        )
       end
 
-      # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class
-      def manages_kubernetes_service?
-        return true unless kubernetes_service&.active?
+      # Returns a hash of all pods in the namespace
+      def read_pods
+        kubeclient = build_kubeclient!
+
+        kubeclient.get_pods(namespace: actual_namespace).as_json
+      rescue KubeException => err
+        raise err unless err.error_code == 404
+
+        []
+      end
+
+      def kubeclient_ssl_options
+        opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
+
+        if ca_pem.present?
+          opts[:cert_store] = OpenSSL::X509::Store.new
+          opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
+        end
+
+        opts
+      end
 
-        kubernetes_service.api_url == api_url
+      def kubeclient_auth_options
+        { bearer_token: token }
       end
 
-      def destroy_kubernetes_integration!
-        return unless manages_kubernetes_service?
+      def join_api_url(api_path)
+        url = URI.parse(api_url)
+        prefix = url.path.sub(%r{/+\z}, '')
+
+        url.path = [prefix, api_path].join("/")
 
-        kubernetes_service&.destroy!
+        url.to_s
       end
 
-      def kubernetes_service
-        @kubernetes_service ||= project&.kubernetes_service
+      def terminal_auth
+        {
+          token: token,
+          ca_pem: ca_pem,
+          max_session_time: current_application_settings.terminal_max_session_time
+        }
       end
 
-      def ensure_kubernetes_service
-        @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service
+      def enforce_namespace_to_lower_case
+        self.namespace = self.namespace&.downcase
+      end
+
+      def prevent_modification
+        return unless managed?
+
+        if api_url_changed? || token_changed? || ca_pem_changed?
+          errors.add(:base, "cannot modify managed cluster")
+          return false
+        end
+
+        true
       end
     end
   end
diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb
index ee2e43ee9dd..7fac32466ab 100644
--- a/app/models/clusters/providers/gcp.rb
+++ b/app/models/clusters/providers/gcp.rb
@@ -56,6 +56,7 @@ module Clusters
         before_transition any => [:creating] do |provider, transition|
           operation_id = transition.args.first
           raise ArgumentError.new('operation_id is required') unless operation_id.present?
+
           provider.operation_id = operation_id
         end
 
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 6dba154a6ea..307e4fcedfe 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -1,3 +1,4 @@
+# coding: utf-8
 class Commit
   extend ActiveModel::Naming
   extend Gitlab::Cache::RequestCache
@@ -25,7 +26,7 @@ class Commit
   DIFF_HARD_LIMIT_FILES = 1000
   DIFF_HARD_LIMIT_LINES = 50000
 
-  MIN_SHA_LENGTH = 7
+  MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH
   COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze
 
   def banzai_render_context(field)
@@ -80,10 +81,11 @@ class Commit
 
     @raw = raw_commit
     @project = project
+    @statuses = {}
   end
 
   def id
-    @raw.id
+    raw.id
   end
 
   def ==(other)
@@ -108,12 +110,12 @@ class Commit
     @link_reference_pattern ||= super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})/)
   end
 
-  def to_reference(from_project = nil, full: false)
-    commit_reference(from_project, id, full: full)
+  def to_reference(from = nil, full: false)
+    commit_reference(from, id, full: full)
   end
 
-  def reference_link_text(from_project = nil, full: false)
-    commit_reference(from_project, short_id, full: full)
+  def reference_link_text(from = nil, full: false)
+    commit_reference(from, short_id, full: full)
   end
 
   def diff_line_count
@@ -236,11 +238,13 @@ class Commit
   end
 
   def status(ref = nil)
-    @statuses ||= {}
-
     return @statuses[ref] if @statuses.key?(ref)
 
-    @statuses[ref] = pipelines.latest_status(ref)
+    @statuses[ref] = project.pipelines.latest_status_per_commit(id, ref)[id]
+  end
+
+  def set_status_for_ref(ref, status)
+    @statuses[ref] = status
   end
 
   def signature
@@ -358,7 +362,7 @@ class Commit
     @deltas ||= raw.deltas
   end
 
-  def diffs(diff_options = nil)
+  def diffs(diff_options = {})
     Gitlab::Diff::FileCollection::Commit.new(self, diff_options: diff_options)
   end
 
@@ -378,8 +382,8 @@ class Commit
 
   private
 
-  def commit_reference(from_project, referable_commit_id, full: false)
-    reference = project.to_reference(from_project, full: full)
+  def commit_reference(from, referable_commit_id, full: false)
+    reference = project.to_reference(from, full: full)
 
     if reference.present?
       "#{reference}#{self.class.reference_prefix}#{referable_commit_id}"
diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb
new file mode 100644
index 00000000000..dd93af9df64
--- /dev/null
+++ b/app/models/commit_collection.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# A collection of Commit instances for a specific project and Git reference.
+class CommitCollection
+  include Enumerable
+
+  attr_reader :project, :ref, :commits
+
+  # project - The project the commits belong to.
+  # commits - The Commit instances to store.
+  # ref - The name of the ref (e.g. "master").
+  def initialize(project, commits, ref = nil)
+    @project = project
+    @commits = commits
+    @ref = ref
+  end
+
+  def each(&block)
+    commits.each(&block)
+  end
+
+  # Sets the pipeline status for every commit.
+  #
+  # Setting this status ahead of time removes the need for running a query for
+  # every commit we're displaying.
+  def with_pipeline_status
+    statuses = project.pipelines.latest_status_per_commit(map(&:id), ref)
+
+    each do |commit|
+      commit.set_status_for_ref(ref, statuses[commit.id])
+    end
+
+    self
+  end
+
+  def respond_to_missing?(message, inc_private = false)
+    commits.respond_to?(message, inc_private)
+  end
+
+  # rubocop:disable GitlabSecurity/PublicSend
+  def method_missing(message, *args, &block)
+    commits.public_send(message, *args, &block)
+  end
+end
diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb
index 84e2e8a5dd5..b93c111dabc 100644
--- a/app/models/commit_range.rb
+++ b/app/models/commit_range.rb
@@ -89,8 +89,8 @@ class CommitRange
 
   alias_method :id, :to_s
 
-  def to_reference(from_project = nil, full: false)
-    project_reference = project.to_reference(from_project, full: full)
+  def to_reference(from = nil, full: false)
+    project_reference = project.to_reference(from, full: full)
 
     if project_reference.present?
       project_reference + self.class.reference_prefix + self.id
@@ -99,8 +99,8 @@ class CommitRange
     end
   end
 
-  def reference_link_text(from_project = nil)
-    project_reference = project.to_reference(from_project)
+  def reference_link_text(from = nil)
+    project_reference = project.to_reference(from)
     reference         = ref_from + notation + ref_to
 
     if project_reference.present?
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 6b07dbdf3ea..c0263c0b4e2 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -17,6 +17,7 @@ class CommitStatus < ActiveRecord::Base
   validates :name, presence: true, unless: :importing?
 
   alias_attribute :author, :user
+  alias_attribute :pipeline_id, :commit_id
 
   scope :failed_but_allowed, -> do
     where(allow_failure: true, status: [:failed, :canceled])
@@ -42,7 +43,8 @@ class CommitStatus < ActiveRecord::Base
     script_failure: 1,
     api_failure: 2,
     stuck_or_timeout_failure: 3,
-    runner_system_failure: 4
+    runner_system_failure: 4,
+    missing_dependency_failure: 5
   }
 
   ##
@@ -103,26 +105,29 @@ class CommitStatus < ActiveRecord::Base
     end
 
     after_transition do |commit_status, transition|
+      next unless commit_status.project
       next if transition.loopback?
 
       commit_status.run_after_commit do
-        if pipeline
+        if pipeline_id
           if complete? || manual?
-            PipelineProcessWorker.perform_async(pipeline.id)
+            PipelineProcessWorker.perform_async(pipeline_id)
           else
-            PipelineUpdateWorker.perform_async(pipeline.id)
+            PipelineUpdateWorker.perform_async(pipeline_id)
           end
         end
 
-        StageUpdateWorker.perform_async(commit_status.stage_id)
-        ExpireJobCacheWorker.perform_async(commit_status.id)
+        StageUpdateWorker.perform_async(stage_id)
+        ExpireJobCacheWorker.perform_async(id)
       end
     end
 
     after_transition any => :failed do |commit_status|
+      next unless commit_status.project
+
       commit_status.run_after_commit do
         MergeRequests::AddTodoWhenBuildFailsService
-          .new(pipeline.project, nil).execute(self)
+          .new(project, nil).execute(self)
       end
     end
   end
diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb
new file mode 100644
index 00000000000..0460439e9e6
--- /dev/null
+++ b/app/models/concerns/artifact_migratable.rb
@@ -0,0 +1,45 @@
+# Adapter class to unify the interface between mounted uploaders and the
+# Ci::Artifact model
+# Meant to be prepended so the interface can stay the same
+module ArtifactMigratable
+  def artifacts_file
+    job_artifacts_archive&.file || legacy_artifacts_file
+  end
+
+  def artifacts_metadata
+    job_artifacts_metadata&.file || legacy_artifacts_metadata
+  end
+
+  def artifacts?
+    !artifacts_expired? && artifacts_file.exists?
+  end
+
+  def artifacts_metadata?
+    artifacts? && artifacts_metadata.exists?
+  end
+
+  def artifacts_file_changed?
+    job_artifacts_archive&.file_changed? || attribute_changed?(:artifacts_file)
+  end
+
+  def remove_artifacts_file!
+    if job_artifacts_archive
+      job_artifacts_archive.destroy
+    else
+      remove_legacy_artifacts_file!
+    end
+  end
+
+  def remove_artifacts_metadata!
+    if job_artifacts_metadata
+      job_artifacts_metadata.destroy
+    else
+      remove_legacy_artifacts_metadata!
+    end
+  end
+
+  def artifacts_size
+    read_attribute(:artifacts_size).to_i +
+      job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i
+  end
+end
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index 2ec70203710..10659030910 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -4,15 +4,26 @@ module Avatarable
   def avatar_path(only_path: true)
     return unless self[:avatar].present?
 
-    # If only_path is true then use the relative path of avatar.
-    # Otherwise use full path (including host).
     asset_host = ActionController::Base.asset_host
-    gitlab_host = only_path ? gitlab_config.relative_url_root : gitlab_config.url
+    use_asset_host = asset_host.present?
 
-    # If asset_host is set then it is expected that assets are handled by a standalone host.
-    # That means we do not want to get GitLab's relative_url_root option anymore.
-    host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host
+    # Avatars for private and internal groups and projects require authentication to be viewed,
+    # which means they can only be served by Rails, on the regular GitLab host.
+    # If an asset host is configured, we need to return the fully qualified URL
+    # instead of only the avatar path, so that Rails doesn't prefix it with the asset host.
+    if use_asset_host && respond_to?(:public?) && !public?
+      use_asset_host = false
+      only_path = false
+    end
 
-    [host, avatar.url].join
+    url_base = ""
+    if use_asset_host
+      url_base << asset_host unless only_path
+    else
+      url_base << gitlab_config.base_url unless only_path
+      url_base << gitlab_config.relative_url_root
+    end
+
+    url_base + avatar.url
   end
 end
diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb
index 9adc309a22b..d8394415362 100644
--- a/app/models/concerns/awardable.rb
+++ b/app/models/concerns/awardable.rb
@@ -98,6 +98,7 @@ module Awardable
 
   def create_award_emoji(name, current_user)
     return unless emoji_awardable?
+
     award_emoji.create(name: normalize_name(name), user: current_user)
   end
 
diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb
new file mode 100644
index 00000000000..984c4f53bf7
--- /dev/null
+++ b/app/models/concerns/bulk_member_access_load.rb
@@ -0,0 +1,46 @@
+# Returns and caches in thread max member access for a resource
+#
+module BulkMemberAccessLoad
+  extend ActiveSupport::Concern
+
+  included do
+    # Determine the maximum access level for a group of resources in bulk.
+    #
+    # Returns a Hash mapping resource ID -> maximum access level.
+    def max_member_access_for_resource_ids(resource_klass, resource_ids, memoization_index = self.id, &block)
+      raise 'Block is mandatory' unless block_given?
+
+      resource_ids = resource_ids.uniq
+      key = max_member_access_for_resource_key(resource_klass, memoization_index)
+      access = {}
+
+      if RequestStore.active?
+        RequestStore.store[key] ||= {}
+        access = RequestStore.store[key]
+      end
+
+      # Look up only the IDs we need
+      resource_ids = resource_ids - access.keys
+
+      return access if resource_ids.empty?
+
+      resource_access = yield(resource_ids)
+
+      access.merge!(resource_access)
+
+      missing_resource_ids = resource_ids - resource_access.keys
+
+      missing_resource_ids.each do |resource_id|
+        access[resource_id] = Gitlab::Access::NO_ACCESS
+      end
+
+      access
+    end
+
+    private
+
+    def max_member_access_for_resource_key(klass, memoization_index)
+      "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}"
+    end
+  end
+end
diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb
index f5cbb3becad..4b4d519f3df 100644
--- a/app/models/concerns/discussion_on_diff.rb
+++ b/app/models/concerns/discussion_on_diff.rb
@@ -32,6 +32,10 @@ module DiscussionOnDiff
     first_note.position.new_path
   end
 
+  def on_merge_request_commit?
+    for_merge_request? && commit_id.present?
+  end
+
   # Returns an array of at most 16 highlighted lines above a diff note
   def truncated_diff_lines(highlight: true)
     lines = highlight ? highlighted_diff_lines : diff_lines
diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb
index 9585b5583dc..8a241e4374a 100644
--- a/app/models/concerns/has_variable.rb
+++ b/app/models/concerns/has_variable.rb
@@ -16,6 +16,10 @@ module HasVariable
        key: Gitlab::Application.secrets.db_key_base,
        algorithm: 'aes-256-cbc'
 
+    def key=(new_key)
+      super(new_key.to_s.strip)
+    end
+
     def to_runner_variable
       { key: key, value: value, public: false }
     end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index c008fb91a16..5ca4a7086cb 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -122,9 +122,7 @@ module Issuable
     #
     # Returns an ActiveRecord::Relation.
     def search(query)
-      title = to_fuzzy_arel(:title, query)
-
-      where(title)
+      fuzzy_search(query, [:title])
     end
 
     # Searches for records with a matching title or description.
@@ -135,10 +133,7 @@ module Issuable
     #
     # Returns an ActiveRecord::Relation.
     def full_search(query)
-      title = to_fuzzy_arel(:title, query)
-      description = to_fuzzy_arel(:description, query)
-
-      where(title&.or(description))
+      fuzzy_search(query, [:title, :description])
     end
 
     def sort(method, excluded_labels: [])
@@ -255,8 +250,10 @@ module Issuable
     participants(user).include?(user)
   end
 
-  def to_hook_data(user, old_labels: [], old_assignees: [])
+  def to_hook_data(user, old_associations: {})
     changes = previous_changes
+    old_labels = old_associations.fetch(:labels, [])
+    old_assignees = old_associations.fetch(:assignees, [])
 
     if old_labels != labels
       changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)]
@@ -270,6 +267,14 @@ module Issuable
       end
     end
 
+    if self.respond_to?(:total_time_spent)
+      old_total_time_spent = old_associations.fetch(:total_time_spent, nil)
+
+      if old_total_time_spent != total_time_spent
+        changes[:total_time_spent] = [old_total_time_spent, total_time_spent]
+      end
+    end
+
     Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes)
   end
 
@@ -341,4 +346,11 @@ module Issuable
   def first_contribution?
     false
   end
+
+  ##
+  # Overriden in MergeRequest
+  #
+  def wipless_title_changed(old_title)
+    old_title != title
+  end
 end
diff --git a/app/models/concerns/manual_inverse_association.rb b/app/models/concerns/manual_inverse_association.rb
new file mode 100644
index 00000000000..0fca8feaf89
--- /dev/null
+++ b/app/models/concerns/manual_inverse_association.rb
@@ -0,0 +1,17 @@
+module ManualInverseAssociation
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def manual_inverse_association(association, inverse)
+      define_method(association) do |*args|
+        super(*args).tap do |value|
+          next unless value
+
+          child_association = value.association(inverse)
+          child_association.set_inverse_instance(self)
+          child_association.target = self
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb
index 1db6b2d2fa2..b43eaeaeea0 100644
--- a/app/models/concerns/mentionable.rb
+++ b/app/models/concerns/mentionable.rb
@@ -31,11 +31,11 @@ module Mentionable
   #
   # By default this will be the class name and the result of calling
   # `to_reference` on the object.
-  def gfm_reference(from_project = nil)
+  def gfm_reference(from = nil)
     # "MergeRequest" > "merge_request" > "Merge request" > "merge request"
     friendly_name = self.class.to_s.underscore.humanize.downcase
 
-    "#{friendly_name} #{to_reference(from_project)}"
+    "#{friendly_name} #{to_reference(from)}"
   end
 
   # The GFM reference to this Mentionable, which shouldn't be included in its #references.
diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb
index 710fc1ed647..7026f565706 100644
--- a/app/models/concerns/milestoneish.rb
+++ b/app/models/concerns/milestoneish.rb
@@ -86,6 +86,14 @@ module Milestoneish
     false
   end
 
+  def total_issue_time_spent
+    @total_issue_time_spent ||= issues.joins(:timelogs).sum(:time_spent)
+  end
+
+  def human_total_issue_time_spent
+    Gitlab::TimeTrackingFormatter.output(total_issue_time_spent)
+  end
+
   private
 
   def count_issues_by_state(user)
diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb
index fde1cc44afa..e62f42e8e70 100644
--- a/app/models/concerns/protected_branch_access.rb
+++ b/app/models/concerns/protected_branch_access.rb
@@ -1,12 +1,6 @@
 module ProtectedBranchAccess
   extend ActiveSupport::Concern
 
-  ALLOWED_ACCESS_LEVELS ||= [
-    Gitlab::Access::MASTER,
-    Gitlab::Access::DEVELOPER,
-    Gitlab::Access::NO_ACCESS
-  ].freeze
-
   included do
     include ProtectedRefAccess
 
@@ -14,18 +8,6 @@ module ProtectedBranchAccess
 
     delegate :project, to: :protected_branch
 
-    validates :access_level, presence: true, inclusion: {
-      in: ALLOWED_ACCESS_LEVELS
-    }
-
-    def self.human_access_levels
-      {
-        Gitlab::Access::MASTER => "Masters",
-        Gitlab::Access::DEVELOPER => "Developers + Masters",
-        Gitlab::Access::NO_ACCESS => "No one"
-      }.with_indifferent_access
-    end
-
     def check_access(user)
       return false if access_level == Gitlab::Access::NO_ACCESS
 
diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb
index c4f158e569a..80c9f7d4eb4 100644
--- a/app/models/concerns/protected_ref_access.rb
+++ b/app/models/concerns/protected_ref_access.rb
@@ -1,13 +1,35 @@
 module ProtectedRefAccess
   extend ActiveSupport::Concern
 
+  ALLOWED_ACCESS_LEVELS = [
+    Gitlab::Access::MASTER,
+    Gitlab::Access::DEVELOPER,
+    Gitlab::Access::NO_ACCESS
+  ].freeze
+
+  HUMAN_ACCESS_LEVELS = {
+    Gitlab::Access::MASTER => "Masters".freeze,
+    Gitlab::Access::DEVELOPER => "Developers + Masters".freeze,
+    Gitlab::Access::NO_ACCESS => "No one".freeze
+  }.freeze
+
   included do
     scope :master, -> { where(access_level: Gitlab::Access::MASTER) }
     scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) }
+
+    validates :access_level, presence: true, if: :role?, inclusion: {
+      in: ALLOWED_ACCESS_LEVELS
+    }
   end
 
   def humanize
-    self.class.human_access_levels[self.access_level]
+    HUMAN_ACCESS_LEVELS[self.access_level]
+  end
+
+  # CE access levels are always role-based,
+  # where as EE allows groups and users too
+  def role?
+    true
   end
 
   def check_access(user)
diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb
index 78ac4f324e7..b782e85717e 100644
--- a/app/models/concerns/referable.rb
+++ b/app/models/concerns/referable.rb
@@ -7,7 +7,7 @@ module Referable
 
   # Returns the String necessary to reference this object in Markdown
   #
-  # from_project - Refering Project object
+  # from - Referring parent object
   #
   # This should be overridden by the including class.
   #
@@ -17,12 +17,12 @@ module Referable
   #   Issue.last.to_reference(other_project) # => "cross-project#1"
   #
   # Returns a String
-  def to_reference(_from_project = nil, full:)
+  def to_reference(_from = nil, full:)
     ''
   end
 
-  def reference_link_text(from_project = nil)
-    to_reference(from_project)
+  def reference_link_text(from = nil)
+    to_reference(from)
   end
 
   included do
diff --git a/app/models/concerns/throttled_touch.rb b/app/models/concerns/throttled_touch.rb
new file mode 100644
index 00000000000..ad0ff0f20d4
--- /dev/null
+++ b/app/models/concerns/throttled_touch.rb
@@ -0,0 +1,10 @@
+# ThrottledTouch can be used to throttle the number of updates triggered by
+# calling "touch" on an ActiveRecord model.
+module ThrottledTouch
+  # The amount of time to wait before "touch" can update a record again.
+  TOUCH_INTERVAL = 1.minute
+
+  def touch(*args)
+    super if (Time.zone.now - updated_at) > TOUCH_INTERVAL
+  end
+end
diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb
index 6eba87da1a1..4a65738214b 100644
--- a/app/models/diff_discussion.rb
+++ b/app/models/diff_discussion.rb
@@ -24,7 +24,11 @@ class DiffDiscussion < Discussion
     return unless for_merge_request?
     return {} if active?
 
-    noteable.version_params_for(position.diff_refs)
+    if on_merge_request_commit?
+      { commit_id: commit_id }
+    else
+      noteable.version_params_for(position.diff_refs)
+    end
   end
 
   def reply_attributes
diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb
index d88a92dc027..b53d44cda95 100644
--- a/app/models/diff_note.rb
+++ b/app/models/diff_note.rb
@@ -17,8 +17,10 @@ class DiffNote < Note
   validates :noteable_type, inclusion: { in: NOTEABLE_TYPES }
   validate :positions_complete
   validate :verify_supported
+  validate :diff_refs_match_commit, if: :for_commit?
 
-  before_validation :set_original_position, :update_position, on: :create
+  before_validation :set_original_position, on: :create
+  before_validation :update_position, on: :create, if: :on_text?
   before_validation :set_line_code
   after_save :keep_around_commits
 
@@ -134,6 +136,12 @@ class DiffNote < Note
     errors.add(:position, "is invalid")
   end
 
+  def diff_refs_match_commit
+    return if self.original_position.diff_refs == self.commit.diff_refs
+
+    errors.add(:commit_id, 'does not match the diff refs')
+  end
+
   def keep_around_commits
     project.repository.keep_around(self.original_position.base_sha)
     project.repository.keep_around(self.original_position.start_sha)
diff --git a/app/models/discussion.rb b/app/models/discussion.rb
index 437df923d2d..92482a1a875 100644
--- a/app/models/discussion.rb
+++ b/app/models/discussion.rb
@@ -11,6 +11,7 @@ class Discussion
             :author,
 
             :noteable,
+            :commit_id,
             :for_commit?,
             :for_merge_request?,
 
diff --git a/app/models/email.rb b/app/models/email.rb
index 2da8b050149..d6516761f0a 100644
--- a/app/models/email.rb
+++ b/app/models/email.rb
@@ -1,5 +1,6 @@
 class Email < ActiveRecord::Base
   include Sortable
+  include Gitlab::SQL::Pattern
 
   belongs_to :user
 
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 21a028e351c..bf69b4c50f0 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -138,11 +138,11 @@ class Environment < ActiveRecord::Base
   end
 
   def has_terminals?
-    project.deployment_service.present? && available? && last_deployment.present?
+    project.deployment_platform.present? && available? && last_deployment.present?
   end
 
   def terminals
-    project.deployment_service.terminals(self) if has_terminals?
+    project.deployment_platform.terminals(self) if has_terminals?
   end
 
   def has_metrics?
diff --git a/app/models/epic.rb b/app/models/epic.rb
index 62898a02e2d..286b855de3f 100644
--- a/app/models/epic.rb
+++ b/app/models/epic.rb
@@ -1,7 +1,11 @@
 # Placeholder class for model that is implemented in EE
-# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE
+# It reserves '&' as a reference prefix, but the table does not exists in CE
 class Epic < ActiveRecord::Base
-  # TODO: this will be implemented as part of #3853
-  def to_reference
+  def self.reference_prefix
+    '&'
+  end
+
+  def self.reference_prefix_escaped
+    '&amp;'
   end
 end
diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb
index 0bf18e529f0..2aaba2e4c90 100644
--- a/app/models/external_issue.rb
+++ b/app/models/external_issue.rb
@@ -38,13 +38,17 @@ class ExternalIssue
     @project.id
   end
 
-  def to_reference(_from_project = nil, full: nil)
+  def to_reference(_from = nil, full: nil)
     id
   end
 
-  def reference_link_text(from_project = nil)
+  def reference_link_text(from = nil)
     return "##{id}" if id =~ /^\d+$/
 
     id
   end
+
+  def notes
+    Note.none
+  end
 end
diff --git a/app/models/fork_network_member.rb b/app/models/fork_network_member.rb
index 6a9b52a1ef8..eb9417dc34f 100644
--- a/app/models/fork_network_member.rb
+++ b/app/models/fork_network_member.rb
@@ -4,4 +4,14 @@ class ForkNetworkMember < ActiveRecord::Base
   belongs_to :forked_from_project, class_name: 'Project'
 
   validates :fork_network, :project, presence: true
+
+  after_destroy :cleanup_fork_network
+
+  private
+
+  def cleanup_fork_network
+    # Explicitly using `#count` makes sure we have the correct number if the
+    # relation was loaded in the fork_network.
+    fork_network.destroy if fork_network.fork_network_members.count == 0
+  end
 end
diff --git a/app/models/group.rb b/app/models/group.rb
index 8cf632fb566..fddace03387 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord'
 
 class Group < Namespace
   include Gitlab::ConfigHelper
+  include AfterCommitQueue
   include AccessRequestable
   include Avatarable
   include Referable
@@ -50,20 +51,6 @@ class Group < Namespace
       Gitlab::Database.postgresql?
     end
 
-    # Searches for groups matching the given query.
-    #
-    # This method uses ILIKE on PostgreSQL and LIKE on MySQL.
-    #
-    # query - The search query as a String
-    #
-    # Returns an ActiveRecord::Relation.
-    def search(query)
-      table   = Namespace.arel_table
-      pattern = "%#{query}%"
-
-      where(table[:name].matches(pattern).or(table[:path].matches(pattern)))
-    end
-
     def sort(method)
       if method == 'storage_size_desc'
         # storage_size is a virtual column so we need to
@@ -97,7 +84,7 @@ class Group < Namespace
     end
   end
 
-  def to_reference(_from_project = nil, full: nil)
+  def to_reference(_from = nil, full: nil)
     "#{self.class.reference_prefix}#{full_path}"
   end
 
@@ -303,6 +290,18 @@ class Group < Namespace
     "#{parent.full_path}/#{path_was}"
   end
 
+  def group_member(user)
+    if group_members.loaded?
+      group_members.find { |gm| gm.user_id == user.id }
+    else
+      group_members.find_by(user_id: user)
+    end
+  end
+
+  def hashed_storage?(_feature)
+    false
+  end
+
   private
 
   def update_two_factor_requirement
diff --git a/app/models/identity.rb b/app/models/identity.rb
index ac8094b610e..ff811e19f8a 100644
--- a/app/models/identity.rb
+++ b/app/models/identity.rb
@@ -1,18 +1,27 @@
 class Identity < ActiveRecord::Base
   include Sortable
   include CaseSensitivity
+
   belongs_to :user
 
   validates :provider, presence: true
-  validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider }
+  validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider, case_sensitive: false }
   validates :user_id, uniqueness: { scope: :provider }
 
+  scope :with_provider, ->(provider) { where(provider: provider) }
   scope :with_extern_uid, ->(provider, extern_uid) do
-    extern_uid = Gitlab::LDAP::Person.normalize_dn(extern_uid) if provider.starts_with?('ldap')
-    where(extern_uid: extern_uid, provider: provider)
+    iwhere(extern_uid: normalize_uid(provider, extern_uid)).with_provider(provider)
   end
 
   def ldap?
     provider.starts_with?('ldap')
   end
+
+  def self.normalize_uid(provider, uid)
+    if provider.to_s.starts_with?('ldap')
+      Gitlab::LDAP::Person.normalize_dn(uid)
+    else
+      uid.to_s
+    end
+  end
 end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 3b3c7fb7f8b..33db197e612 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -9,6 +9,7 @@ class Issue < ActiveRecord::Base
   include FasterCacheKeys
   include RelativePositioning
   include TimeTrackable
+  include ThrottledTouch
 
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
@@ -49,7 +50,6 @@ class Issue < ActiveRecord::Base
   scope :public_only, -> { where(confidential: false) }
 
   after_save :expire_etag_cache
-  after_commit :update_project_counter_caches, on: :destroy
 
   attr_spammable :title, spam_title: true
   attr_spammable :description, spam_description: true
@@ -246,7 +246,12 @@ class Issue < ActiveRecord::Base
 
   def as_json(options = {})
     super(options).tap do |json|
-      json[:subscribed] = subscribed?(options[:user], project) if options.key?(:user) && options[:user]
+      if options.key?(:sidebar_endpoints) && project
+        url_helper = Gitlab::Routing.url_helpers
+
+        json.merge!(issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'),
+                    toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self))
+      end
 
       if options.key?(:labels)
         json[:labels] = labels.as_json(
@@ -262,10 +267,6 @@ class Issue < ActiveRecord::Base
     true
   end
 
-  def update_project_counter_caches?
-    state_changed? || confidential_changed?
-  end
-
   def update_project_counter_caches
     Projects::OpenIssuesCountService.new(project).refresh_cache
   end
diff --git a/app/models/key.rb b/app/models/key.rb
index f119b15c737..a3f8a5d6dc7 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -2,6 +2,7 @@ require 'digest/md5'
 
 class Key < ActiveRecord::Base
   include Gitlab::CurrentSettings
+  include AfterCommitQueue
   include Sortable
 
   belongs_to :user
@@ -27,8 +28,10 @@ class Key < ActiveRecord::Base
 
   after_commit :add_to_shell, on: :create
   after_create :post_create_hook
+  after_create :refresh_user_cache
   after_commit :remove_from_shell, on: :destroy
   after_destroy :post_destroy_hook
+  after_destroy :refresh_user_cache
 
   def key=(value)
     value&.delete!("\n\r")
@@ -76,6 +79,12 @@ class Key < ActiveRecord::Base
     )
   end
 
+  def refresh_user_cache
+    return unless user
+
+    Users::KeysCountService.new(user).refresh_cache
+  end
+
   def post_destroy_hook
     SystemHooksService.new.execute_hooks_for(self, :destroy)
   end
diff --git a/app/models/label.rb b/app/models/label.rb
index 899028a01a0..b5bfa6ea2dd 100644
--- a/app/models/label.rb
+++ b/app/models/label.rb
@@ -165,12 +165,12 @@ class Label < ActiveRecord::Base
   #
   # Returns a String
   #
-  def to_reference(from_project = nil, target_project: nil, format: :id, full: false)
+  def to_reference(from = nil, target_project: nil, format: :id, full: false)
     format_reference = label_format_reference(format)
     reference = "#{self.class.reference_prefix}#{format_reference}"
 
-    if from_project
-      "#{from_project.to_reference(target_project, full: full)}#{reference}"
+    if from
+      "#{from.to_reference(target_project, full: full)}#{reference}"
     else
       reference
     end
diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb
index 0056b0f80f4..38b8c41024a 100644
--- a/app/models/lfs_object.rb
+++ b/app/models/lfs_object.rb
@@ -8,16 +8,8 @@ class LfsObject < ActiveRecord::Base
 
   mount_uploader :file, LfsObjectUploader
 
-  def storage_project(project)
-    if project && project.forked?
-      storage_project(project.forked_from_project)
-    else
-      project
-    end
-  end
-
   def project_allowed_access?(project)
-    projects.exists?(storage_project(project).id)
+    projects.exists?(project.lfs_storage_project.id)
   end
 
   def self.destroy_unreferenced
diff --git a/app/models/member.rb b/app/models/member.rb
index cbbd58f2eaf..2fe5fda985f 100644
--- a/app/models/member.rb
+++ b/app/models/member.rb
@@ -1,4 +1,5 @@
 class Member < ActiveRecord::Base
+  include AfterCommitQueue
   include Sortable
   include Importable
   include Expirable
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 82d0ae90d77..422f138c4ea 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -5,6 +5,9 @@ class MergeRequest < ActiveRecord::Base
   include Referable
   include IgnorableColumn
   include TimeTrackable
+  include ManualInverseAssociation
+  include EachBatch
+  include ThrottledTouch
 
   ignore_column :locked_at,
                 :ref_fetched
@@ -14,9 +17,28 @@ class MergeRequest < ActiveRecord::Base
   belongs_to :merge_user, class_name: "User"
 
   has_many :merge_request_diffs
+
   has_one :merge_request_diff,
     -> { order('merge_request_diffs.id DESC') }, inverse_of: :merge_request
 
+  belongs_to :latest_merge_request_diff, class_name: 'MergeRequestDiff'
+  manual_inverse_association :latest_merge_request_diff, :merge_request
+
+  # This is the same as latest_merge_request_diff unless:
+  # 1. There are arguments - in which case we might be trying to force-reload.
+  # 2. This association is already loaded.
+  # 3. The latest diff does not exist.
+  #
+  # The second one in particular is important - MergeRequestDiff#merge_request
+  # is the inverse of MergeRequest#merge_request_diff, which means it may not be
+  # the latest diff, because we could have loaded any diff from this particular
+  # MR. If we haven't already loaded a diff, then it's fine to load the latest.
+  def merge_request_diff(*args)
+    fallback = latest_merge_request_diff if args.empty? && !association(:merge_request_diff).loaded?
+
+    fallback || super
+  end
+
   belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline"
 
   has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
@@ -31,7 +53,6 @@ class MergeRequest < ActiveRecord::Base
 
   after_create :ensure_merge_request_diff, unless: :importing?
   after_update :reload_diff_if_branch_changed
-  after_commit :update_project_counter_caches, on: :destroy
 
   # When this attribute is true some MR validation is ignored
   # It allows us to close or modify broken merge requests
@@ -125,6 +146,13 @@ class MergeRequest < ActiveRecord::Base
     '!'
   end
 
+  # Use this method whenever you need to make sure the head_pipeline is synced with the
+  # branch head commit, for example checking if a merge request can be merged.
+  # For more information check: https://gitlab.com/gitlab-org/gitlab-ce/issues/40004
+  def actual_head_pipeline
+    head_pipeline&.sha == diff_head_sha ? head_pipeline : nil
+  end
+
   # Pattern used to extract `!123` merge request references from text
   #
   # This pattern supports cross-project references.
@@ -167,6 +195,22 @@ class MergeRequest < ActiveRecord::Base
     where("merge_requests.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection
   end
 
+  # This is used after project import, to reset the IDs to the correct
+  # values. It is not intended to be called without having already scoped the
+  # relation.
+  def self.set_latest_merge_request_diff_ids!
+    update = '
+      latest_merge_request_diff_id = (
+        SELECT MAX(id)
+        FROM merge_request_diffs
+        WHERE merge_requests.id = merge_request_diffs.merge_request_id
+      )'.squish
+
+    self.each_batch do |batch|
+      batch.update_all(update)
+    end
+  end
+
   WIP_REGEX = /\A\s*(\[WIP\]\s*|WIP:\s*|WIP\s+)+\s*/i.freeze
 
   def self.work_in_progress?(title)
@@ -181,6 +225,12 @@ class MergeRequest < ActiveRecord::Base
     work_in_progress?(title) ? title : "WIP: #{title}"
   end
 
+  # Verifies if title has changed not taking into account WIP prefix
+  # for merge requests.
+  def wipless_title_changed(old_title)
+    self.class.wipless_title(old_title) != self.wipless_title
+  end
+
   def hook_attrs
     Gitlab::HookData::MergeRequestBuilder.new(self).build
   end
@@ -241,9 +291,9 @@ class MergeRequest < ActiveRecord::Base
     if persisted?
       merge_request_diff.commit_shas
     elsif compare_commits
-      compare_commits.reverse.map(&:sha)
+      compare_commits.to_a.reverse.map(&:sha)
     else
-      []
+      Array(diff_head_sha)
     end
   end
 
@@ -322,16 +372,28 @@ class MergeRequest < ActiveRecord::Base
   # We use these attributes to force these to the intended values.
   attr_writer :target_branch_sha, :source_branch_sha
 
+  def source_branch_ref
+    return @source_branch_sha if @source_branch_sha
+    return unless source_branch
+
+    Gitlab::Git::BRANCH_REF_PREFIX + source_branch
+  end
+
+  def target_branch_ref
+    return @target_branch_sha if @target_branch_sha
+    return unless target_branch
+
+    Gitlab::Git::BRANCH_REF_PREFIX + target_branch
+  end
+
   def source_branch_head
     return unless source_project
 
-    source_branch_ref = @source_branch_sha || source_branch
     source_project.repository.commit(source_branch_ref) if source_branch_ref
   end
 
   def target_branch_head
-    target_branch_ref = @target_branch_sha || target_branch
-    target_project.repository.commit(target_branch_ref) if target_branch_ref
+    target_project.repository.commit(target_branch_ref)
   end
 
   def branch_merge_base_commit
@@ -440,7 +502,7 @@ class MergeRequest < ActiveRecord::Base
 
   def merge_request_diff_for(diff_refs_or_sha)
     @merge_request_diffs_by_diff_refs_or_sha ||= Hash.new do |h, diff_refs_or_sha|
-      diffs = merge_request_diffs.viewable.select_without_diff
+      diffs = merge_request_diffs.viewable
       h[diff_refs_or_sha] =
         if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs)
           diffs.find_by_diff_refs(diff_refs_or_sha)
@@ -576,7 +638,7 @@ class MergeRequest < ActiveRecord::Base
     commit_notes = Note
       .except(:order)
       .where(project_id: [source_project_id, target_project_id])
-      .where(noteable_type: 'Commit', commit_id: commit_ids)
+      .for_commit_id(commit_ids)
 
     # We're using a UNION ALL here since this results in better performance
     # compared to using OR statements. We're using UNION ALL since the queries
@@ -587,6 +649,7 @@ class MergeRequest < ActiveRecord::Base
       .to_sql
 
     Note.from("(#{union}) #{Note.table_name}")
+      .includes(:noteable)
   end
 
   alias_method :discussion_notes, :related_notes
@@ -768,8 +831,9 @@ class MergeRequest < ActiveRecord::Base
 
   def mergeable_ci_state?
     return true unless project.only_allow_merge_if_pipeline_succeeds?
+    return true unless head_pipeline
 
-    !head_pipeline || head_pipeline.success? || head_pipeline.skipped?
+    actual_head_pipeline&.success? || actual_head_pipeline&.skipped?
   end
 
   def environments_for(current_user)
@@ -845,7 +909,8 @@ class MergeRequest < ActiveRecord::Base
   def compute_diverged_commits_count
     return 0 unless source_branch_sha && target_branch_sha
 
-    Gitlab::Git::Commit.between(target_project.repository.raw_repository, source_branch_sha, target_branch_sha).size
+    target_project.repository
+      .count_commits_between(source_branch_sha, target_branch_sha)
   end
   private :compute_diverged_commits_count
 
@@ -861,18 +926,26 @@ class MergeRequest < ActiveRecord::Base
       .order(id: :desc)
   end
 
+  def all_commits
+    # MySQL doesn't support LIMIT in a subquery.
+    diffs_relation = if Gitlab::Database.postgresql?
+                       merge_request_diffs.recent
+                     else
+                       merge_request_diffs
+                     end
+
+    MergeRequestDiffCommit
+      .where(merge_request_diff: diffs_relation)
+      .limit(10_000)
+  end
+
   # Note that this could also return SHA from now dangling commits
   #
   def all_commit_shas
-    if persisted?
-      column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).limit(10_000).pluck('sha')
-      serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas)
+    @all_commit_shas ||= begin
+      return commit_shas unless persisted?
 
-      (column_shas + serialised_shas).uniq
-    elsif compare_commits
-      compare_commits.to_a.reverse.map(&:id)
-    else
-      [diff_head_sha]
+      all_commits.pluck(:sha).uniq
     end
   end
 
@@ -940,16 +1013,12 @@ class MergeRequest < ActiveRecord::Base
     return true if autocomplete_precheck
 
     return false unless mergeable?(skip_ci_check: true)
-    return false if head_pipeline && !(head_pipeline.success? || head_pipeline.active?)
+    return false if actual_head_pipeline && !(actual_head_pipeline.success? || actual_head_pipeline.active?)
     return false if last_diff_sha != diff_head_sha
 
     true
   end
 
-  def update_project_counter_caches?
-    state_changed?
-  end
-
   def update_project_counter_caches
     Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache
   end
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 1eda0f9cbbd..c37aa0a594b 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -1,21 +1,21 @@
 class MergeRequestDiff < ActiveRecord::Base
   include Sortable
   include Importable
-  include Gitlab::EncodingHelper
+  include ManualInverseAssociation
+  include IgnorableColumn
 
-  # Prevent store of diff if commits amount more then 500
+  # Don't display more than 100 commits at once
   COMMITS_SAFE_SIZE = 100
 
-  # Valid types of serialized diffs allowed by Gitlab::Git::Diff
-  VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta].freeze
+  ignore_column :st_commits,
+                :st_diffs
 
   belongs_to :merge_request
+  manual_inverse_association :merge_request, :merge_request_diff
+
   has_many :merge_request_diff_files, -> { order(:merge_request_diff_id, :relative_order) }
   has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) }
 
-  serialize :st_commits # rubocop:disable Cop/ActiveRecordSerialize
-  serialize :st_diffs # rubocop:disable Cop/ActiveRecordSerialize
-
   state_machine :state, initial: :empty do
     state :collected
     state :overflow
@@ -29,6 +29,8 @@ class MergeRequestDiff < ActiveRecord::Base
 
   scope :viewable, -> { without_state(:empty) }
 
+  scope :recent, -> { order(id: :desc).limit(100) }
+
   # All diff information is collected from repository after object is created.
   # It allows you to override variables like head_commit_sha before getting diff.
   after_create :save_git_content, unless: :importing?
@@ -37,14 +39,6 @@ class MergeRequestDiff < ActiveRecord::Base
     find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha)
   end
 
-  def self.select_without_diff
-    select(column_names - ['st_diffs'])
-  end
-
-  def st_commits
-    super || []
-  end
-
   # Collect information about commits and diff from repository
   # and save it to the database as serialized data
   def save_git_content
@@ -126,11 +120,7 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def commit_shas
-    if st_commits.present?
-      st_commits.map { |commit| commit[:id] }
-    else
-      merge_request_diff_commits.map(&:sha)
-    end
+    merge_request_diff_commits.map(&:sha)
   end
 
   def diff_refs=(new_diff_refs)
@@ -194,7 +184,7 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def latest?
-    self == merge_request.merge_request_diff
+    self.id == merge_request.latest_merge_request_diff_id
   end
 
   def compare_with(sha)
@@ -205,34 +195,11 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def commits_count
-    if st_commits.present?
-      st_commits.size
-    else
-      merge_request_diff_commits.size
-    end
-  end
-
-  def utf8_st_diffs
-    return [] if st_diffs.blank?
-
-    st_diffs.map do |diff|
-      diff.each do |k, v|
-        diff[k] = encode_utf8(v) if v.respond_to?(:encoding)
-      end
-    end
+    merge_request_diff_commits.size
   end
 
   private
 
-  # Old GitLab implementations may have generated diffs as ["--broken-diff"].
-  # Avoid an error 500 by ignoring bad elements. See:
-  # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776
-  def valid_raw_diff?(raw)
-    return false unless raw.respond_to?(:each)
-
-    raw.any? { |element| VALID_CLASSES.include?(element.class) }
-  end
-
   def create_merge_request_diff_files(diffs)
     rows = diffs.map.with_index do |diff, index|
       diff_hash = diff.to_hash.merge(
@@ -256,9 +223,7 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def load_diffs(options)
-    return Gitlab::Git::DiffCollection.new([]) unless diffs_from_database
-
-    raw = diffs_from_database
+    raw = merge_request_diff_files.map(&:to_hash)
 
     if paths = options[:paths]
       raw = raw.select do |diff|
@@ -269,23 +234,11 @@ class MergeRequestDiff < ActiveRecord::Base
     Gitlab::Git::DiffCollection.new(raw, options)
   end
 
-  def diffs_from_database
-    return @diffs_from_database if defined?(@diffs_from_database)
-
-    @diffs_from_database =
-      if st_diffs.present?
-        if valid_raw_diff?(st_diffs)
-          st_diffs
-        end
-      elsif merge_request_diff_files.present?
-        merge_request_diff_files.map(&:to_hash)
-      end
-  end
-
   def load_commits
-    commits = st_commits.presence || merge_request_diff_commits
+    commits = merge_request_diff_commits.map { |commit| Commit.from_hash(commit.to_hash, project) }
 
-    commits.map { |commit| Commit.from_hash(commit.to_hash, project) }
+    CommitCollection
+      .new(merge_request.source_project, commits, merge_request.source_branch)
   end
 
   def save_diffs
diff --git a/app/models/milestone.rb b/app/models/milestone.rb
index 47e6b785c39..77c19380e66 100644
--- a/app/models/milestone.rb
+++ b/app/models/milestone.rb
@@ -13,6 +13,7 @@ class Milestone < ActiveRecord::Base
   include Referable
   include StripAttribute
   include Milestoneish
+  include Gitlab::SQL::Pattern
 
   cache_markdown_field :title, pipeline: :single_line
   cache_markdown_field :description
@@ -73,10 +74,7 @@ class Milestone < ActiveRecord::Base
     #
     # Returns an ActiveRecord::Relation.
     def search(query)
-      t = arel_table
-      pattern = "%#{query}%"
-
-      where(t[:title].matches(pattern).or(t[:description].matches(pattern)))
+      fuzzy_search(query, [:title, :description])
     end
 
     def filter_by_state(milestones, state)
@@ -86,6 +84,13 @@ class Milestone < ActiveRecord::Base
       else milestones.active
       end
     end
+
+    def predefined?(milestone)
+      milestone == Any ||
+        milestone == None ||
+        milestone == Upcoming ||
+        milestone == Started
+    end
   end
 
   def self.reference_prefix
@@ -162,18 +167,18 @@ class Milestone < ActiveRecord::Base
   #   Milestone.first.to_reference(cross_namespace_project)  # => "gitlab-org/gitlab-ce%1"
   #   Milestone.first.to_reference(same_namespace_project)   # => "gitlab-ce%1"
   #
-  def to_reference(from_project = nil, format: :name, full: false)
+  def to_reference(from = nil, format: :name, full: false)
     format_reference = milestone_format_reference(format)
     reference = "#{self.class.reference_prefix}#{format_reference}"
 
     if project
-      "#{project.to_reference(from_project, full: full)}#{reference}"
+      "#{project.to_reference(from, full: full)}#{reference}"
     else
       reference
     end
   end
 
-  def reference_link_text(from_project = nil)
+  def reference_link_text(from = nil)
     self.title
   end
 
@@ -256,7 +261,7 @@ class Milestone < ActiveRecord::Base
 
   def start_date_should_be_less_than_due_date
     if due_date <= start_date
-      errors.add(:start_date, "Can't be greater than due date")
+      errors.add(:due_date, "must be greater than start date")
     end
   end
 
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 4d401e7ba18..901dbf2ba69 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base
   include Routable
   include AfterCommitQueue
   include Storage::LegacyNamespace
+  include Gitlab::SQL::Pattern
 
   # Prevent users from creating unreasonably deep level of nesting.
   # The number 20 was taken based on maximum nesting level of
@@ -86,10 +87,7 @@ class Namespace < ActiveRecord::Base
     #
     # Returns an ActiveRecord::Relation
     def search(query)
-      t = arel_table
-      pattern = "%#{query}%"
-
-      where(t[:name].matches(pattern).or(t[:path].matches(pattern)))
+      fuzzy_search(query, [:name, :path])
     end
 
     def clean_path(path)
@@ -141,7 +139,17 @@ class Namespace < ActiveRecord::Base
   def find_fork_of(project)
     return nil unless project.fork_network
 
-    project.fork_network.find_forks_in(projects).first
+    if RequestStore.active?
+      forks_in_namespace = RequestStore.fetch("namespaces:#{id}:forked_projects") do
+        Hash.new do |found_forks, project|
+          found_forks[project] = project.fork_network.find_forks_in(projects).first
+        end
+      end
+
+      forks_in_namespace[project]
+    else
+      project.fork_network.find_forks_in(projects).first
+    end
   end
 
   def lfs_enabled?
diff --git a/app/models/note.rb b/app/models/note.rb
index f9676361072..c4c2ab8e67d 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -14,6 +14,8 @@ class Note < ActiveRecord::Base
   include ResolvableNote
   include IgnorableColumn
   include Editable
+  include Gitlab::SQL::Pattern
+  include ThrottledTouch
 
   module SpecialRole
     FIRST_TIME_CONTRIBUTOR = :first_time_contributor
@@ -54,7 +56,7 @@ class Note < ActiveRecord::Base
   participant :author
 
   belongs_to :project
-  belongs_to :noteable, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations
+  belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
   belongs_to :author, class_name: "User"
   belongs_to :updated_by, class_name: "User"
   belongs_to :last_edited_by, class_name: 'User'
@@ -110,12 +112,14 @@ class Note < ActiveRecord::Base
     includes(:author, :noteable, :updated_by,
              project: [:project_members, { group: [:group_members] }])
   end
+  scope :with_metadata, -> { includes(:system_note_metadata) }
 
   after_initialize :ensure_discussion_id
   before_validation :nullify_blank_type, :nullify_blank_line_code
   before_validation :set_discussion_id, on: :create
   after_save :keep_around_commit, if: :for_project_noteable?
   after_save :expire_etag_cache
+  after_save :touch_noteable
   after_destroy :expire_etag_cache
 
   class << self
@@ -166,10 +170,20 @@ class Note < ActiveRecord::Base
     def has_special_role?(role, note)
       note.special_role == role
     end
+
+    def search(query)
+      fuzzy_search(query, [:note])
+    end
   end
 
   def cross_reference?
-    system? && matches_cross_reference_regex?
+    return unless system?
+
+    if force_cross_reference_regex_check?
+      matches_cross_reference_regex?
+    else
+      SystemNoteService.cross_reference?(note)
+    end
   end
 
   def diff_note?
@@ -216,16 +230,18 @@ class Note < ActiveRecord::Base
     for_personal_snippet?
   end
 
+  def commit
+    @commit ||= project.commit(commit_id) if commit_id.present?
+  end
+
   # override to return commits, which are not active record
   def noteable
-    if for_commit?
-      @commit ||= project.commit(commit_id)
-    else
-      super
-    end
-  # Temp fix to prevent app crash
-  # if note commit id doesn't exist
+    return commit if for_commit?
+
+    super
   rescue
+    # Temp fix to prevent app crash
+    # if note commit id doesn't exist
     nil
   end
 
@@ -355,6 +371,42 @@ class Note < ActiveRecord::Base
     Gitlab::EtagCaching::Store.new.touch(key)
   end
 
+  def touch(*args)
+    # We're not using an explicit transaction here because this would in all
+    # cases result in all future queries going to the primary, even if no writes
+    # are performed.
+    #
+    # We touch the noteable first so its SELECT query can run before our writes,
+    # ensuring it runs on a secondary (if no prior write took place).
+    touch_noteable
+    super
+  end
+
+  # By default Rails will issue an "SELECT *" for the relation, which is
+  # overkill for just updating the timestamps. To work around this we manually
+  # touch the data so we can SELECT only the columns we need.
+  def touch_noteable
+    # Commits are not stored in the DB so we can't touch them.
+    return if for_commit?
+
+    assoc = association(:noteable)
+
+    noteable_object =
+      if assoc.loaded?
+        noteable
+      else
+        # If the object is not loaded (e.g. when notes are loaded async) we
+        # _only_ want the data we actually need.
+        assoc.scope.select(:id, :updated_at).take
+      end
+
+    noteable_object&.touch
+  end
+
+  def banzai_render_context(field)
+    super.merge(noteable: noteable)
+  end
+
   private
 
   def keep_around_commit
@@ -382,4 +434,10 @@ class Note < ActiveRecord::Base
   def set_discussion_id
     self.discussion_id ||= discussion_class.discussion_id(self)
   end
+
+  def force_cross_reference_regex_check?
+    return unless system?
+
+    SystemNoteMetadata::TYPES_WITH_CROSS_REFERENCES.include?(system_note_metadata&.action)
+  end
 end
diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb
index 2e824cda525..8de42ff9d2e 100644
--- a/app/models/pages_domain.rb
+++ b/app/models/pages_domain.rb
@@ -65,12 +65,18 @@ class PagesDomain < ActiveRecord::Base
 
   def expired?
     return false unless x509
+
     current = Time.new
     current < x509.not_before || x509.not_after < current
   end
 
+  def expiration
+    x509&.not_after
+  end
+
   def subject
     return unless x509
+
     x509.subject.to_s
   end
 
@@ -98,6 +104,7 @@ class PagesDomain < ActiveRecord::Base
 
   def validate_pages_domain
     return unless domain
+
     if domain.downcase.ends_with?(Settings.pages.host.downcase)
       self.errors.add(:domain, "*.#{Settings.pages.host} is restricted")
     end
@@ -105,6 +112,7 @@ class PagesDomain < ActiveRecord::Base
 
   def x509
     return unless certificate
+
     @x509 ||= OpenSSL::X509::Certificate.new(certificate)
   rescue OpenSSL::X509::CertificateError
     nil
@@ -112,6 +120,7 @@ class PagesDomain < ActiveRecord::Base
 
   def pkey
     return unless key
+
     @pkey ||= OpenSSL::PKey::RSA.new(key)
   rescue OpenSSL::PKey::PKeyError, OpenSSL::Cipher::CipherError
     nil
diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb
index cfcb03138b7..063dc521324 100644
--- a/app/models/personal_access_token.rb
+++ b/app/models/personal_access_token.rb
@@ -3,6 +3,8 @@ class PersonalAccessToken < ActiveRecord::Base
   include TokenAuthenticatable
   add_authentication_token_field :token
 
+  REDIS_EXPIRY_TIME = 3.minutes
+
   serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize
 
   belongs_to :user
@@ -27,6 +29,21 @@ class PersonalAccessToken < ActiveRecord::Base
     !revoked? && !expired?
   end
 
+  def self.redis_getdel(user_id)
+    Gitlab::Redis::SharedState.with do |redis|
+      token = redis.get(redis_shared_state_key(user_id))
+      redis.del(redis_shared_state_key(user_id))
+      token
+    end
+  end
+
+  def self.redis_store!(user_id, token)
+    Gitlab::Redis::SharedState.with do |redis|
+      redis.set(redis_shared_state_key(user_id), token, ex: REDIS_EXPIRY_TIME)
+      token
+    end
+  end
+
   protected
 
   def validate_scopes
@@ -38,4 +55,8 @@ class PersonalAccessToken < ActiveRecord::Base
   def set_default_scopes
     self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty?
   end
+
+  def self.redis_shared_state_key(user_id)
+    "gitlab:personal_access_token:#{user_id}"
+  end
 end
diff --git a/app/models/project.rb b/app/models/project.rb
index 53df29dab02..6ae15a0a50f 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -18,6 +18,7 @@ class Project < ActiveRecord::Base
   include SelectForProjectAuthorization
   include Routable
   include GroupDescendant
+  include Gitlab::SQL::Pattern
 
   extend Gitlab::ConfigHelper
   extend Gitlab::CurrentSettings
@@ -188,7 +189,6 @@ class Project < ActiveRecord::Base
   has_one :statistics, class_name: 'ProjectStatistics'
 
   has_one :cluster_project, class_name: 'Clusters::Project'
-  has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster'
   has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
 
   # Container repositories need to remove data from the container registry,
@@ -227,14 +227,13 @@ class Project < ActiveRecord::Base
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
   delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team
-  delegate :empty_repo?, to: :repository
 
   # Validations
   validates :creator, presence: true, on: :create
   validates :description, length: { maximum: 2000 }, allow_blank: true
   validates :ci_config_path,
-    format: { without: /\.{2}/,
-              message: 'cannot include directory traversal.' },
+    format: { without: /(\.{2}|\A\/)/,
+              message: 'cannot include leading slash or directory traversal.' },
     length: { maximum: 255 },
     allow_blank: true
   validates :name,
@@ -272,8 +271,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
-  scope :with_hashed_storage, -> { where('storage_version >= 1') }
-  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+  scope :with_storage_feature, ->(feature) { where('storage_version >= :version', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :without_storage_feature, ->(feature) { where('storage_version < :version OR storage_version IS NULL', version: HASHED_STORAGE_FEATURES[feature]) }
+  scope :with_unmigrated_storage, -> { where('storage_version < :version OR storage_version IS NULL', version: LATEST_STORAGE_VERSION) }
 
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
@@ -365,6 +365,7 @@ class Project < ActiveRecord::Base
   scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 
   scope :excluding_project, ->(project) { where.not(id: project) }
+  scope :import_started, -> { where(import_status: 'started') }
 
   state_machine :import_status, initial: :none do
     event :import_schedule do
@@ -423,32 +424,11 @@ class Project < ActiveRecord::Base
     #
     # query - The search query as a String.
     def search(query)
-      ptable  = arel_table
-      ntable  = Namespace.arel_table
-      pattern = "%#{query}%"
-
-      # unscoping unnecessary conditions that'll be applied
-      # when executing `where("projects.id IN (#{union.to_sql})")`
-      projects = unscoped.select(:id).where(
-        ptable[:path].matches(pattern)
-          .or(ptable[:name].matches(pattern))
-          .or(ptable[:description].matches(pattern))
-      )
-
-      namespaces = unscoped.select(:id)
-        .joins(:namespace)
-        .where(ntable[:name].matches(pattern))
-
-      union = Gitlab::SQL::Union.new([projects, namespaces])
-
-      where("projects.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection
+      fuzzy_search(query, [:path, :name, :description])
     end
 
     def search_by_title(query)
-      pattern = "%#{query}%"
-      table   = Project.arel_table
-
-      non_archived.where(table[:name].matches(pattern))
+      non_archived.fuzzy_search(query, [:name])
     end
 
     def visibility_levels
@@ -518,6 +498,10 @@ class Project < ActiveRecord::Base
     auto_devops&.enabled.nil? && !current_application_settings.auto_devops_enabled?
   end
 
+  def empty_repo?
+    repository.empty?
+  end
+
   def repository_storage_path
     Gitlab.config.repositories.storages[repository_storage].try(:[], 'path')
   end
@@ -580,8 +564,7 @@ class Project < ActiveRecord::Base
       if forked?
         RepositoryForkWorker.perform_async(id,
                                            forked_from_project.repository_storage_path,
-                                           forked_from_project.full_path,
-                                           self.namespace.full_path)
+                                           forked_from_project.disk_path)
       else
         RepositoryImportWorker.perform_async(self.id)
       end
@@ -617,7 +600,7 @@ class Project < ActiveRecord::Base
 
   def ci_config_path=(value)
     # Strip all leading slashes so that //foo -> foo
-    super(value&.sub(%r{\A/+}, '')&.delete("\0"))
+    super(value&.delete("\0"))
   end
 
   def import_url=(value)
@@ -703,10 +686,6 @@ class Project < ActiveRecord::Base
     import_type == 'gitea'
   end
 
-  def github_import?
-    import_type == 'github'
-  end
-
   def check_limit
     unless creator.can_create_project? || namespace.kind == 'group'
       projects_limit = creator.projects_limit
@@ -763,10 +742,10 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def to_human_reference(from_project = nil)
-    if cross_namespace_reference?(from_project)
+  def to_human_reference(from = nil)
+    if cross_namespace_reference?(from)
       name_with_namespace
-    elsif cross_project_reference?(from_project)
+    elsif cross_project_reference?(from)
       name
     end
   end
@@ -775,13 +754,14 @@ class Project < ActiveRecord::Base
     Gitlab::Routing.url_helpers.project_url(self)
   end
 
-  def new_issue_address(author)
+  def new_issuable_address(author, address_type)
     return unless Gitlab::IncomingEmail.supports_issue_creation? && author
 
     author.ensure_incoming_email_token!
 
+    suffix = address_type == 'merge_request' ? '+merge-request' : ''
     Gitlab::IncomingEmail.reply_address(
-      "#{full_path}+#{author.incoming_email_token}")
+      "#{full_path}#{suffix}+#{author.incoming_email_token}")
   end
 
   def build_commit_note(commit)
@@ -919,12 +899,10 @@ class Project < ActiveRecord::Base
     @ci_service ||= ci_services.reorder(nil).find_by(active: true)
   end
 
-  def deployment_services
-    services.where(category: :deployment)
-  end
-
-  def deployment_service
-    @deployment_service ||= deployment_services.reorder(nil).find_by(active: true)
+  # TODO: This will be extended for multiple enviroment clusters
+  def deployment_platform
+    @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes
+    @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true)
   end
 
   def monitoring_services
@@ -1046,6 +1024,18 @@ class Project < ActiveRecord::Base
     forked_from_project || fork_network&.root_project
   end
 
+  def lfs_storage_project
+    @lfs_storage_project ||= begin
+      result = self
+
+      # TODO: Make this go to the fork_network root immeadiatly
+      # dependant on the discussion in: https://gitlab.com/gitlab-org/gitlab-ce/issues/39769
+      result = result.fork_source while result&.forked?
+
+      result || self
+    end
+  end
+
   def personal?
     !group
   end
@@ -1126,7 +1116,11 @@ class Project < ActiveRecord::Base
   end
 
   def project_member(user)
-    project_members.find_by(user_id: user)
+    if project_members.loaded?
+      project_members.find { |member| member.user_id == user.id }
+    else
+      project_members.find_by(user_id: user)
+    end
   end
 
   def default_branch
@@ -1190,6 +1184,10 @@ class Project < ActiveRecord::Base
     !!repository.exists?
   end
 
+  def wiki_repository_exists?
+    wiki.repository_exists?
+  end
+
   # update visibility_level of forks
   def update_forks_visibility_level
     return unless visibility_level < visibility_level_was
@@ -1433,6 +1431,31 @@ class Project < ActiveRecord::Base
     reload_repository!
   end
 
+  def after_import
+    repository.after_import
+    import_finish
+    remove_import_jid
+    update_project_counter_caches
+  end
+
+  def update_project_counter_caches
+    classes = [
+      Projects::OpenIssuesCountService,
+      Projects::OpenMergeRequestsCountService
+    ]
+
+    classes.each do |klass|
+      klass.new(self).refresh_cache
+    end
+  end
+
+  def remove_import_jid
+    return unless import_jid
+
+    Gitlab::SidekiqStatus.unset(import_jid)
+    update_column(:import_jid, nil)
+  end
+
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1528,9 +1551,9 @@ class Project < ActiveRecord::Base
   end
 
   def deployment_variables
-    return [] unless deployment_service
+    return [] unless deployment_platform
 
-    deployment_service.predefined_variables
+    deployment_platform.predefined_variables
   end
 
   def auto_devops_variables
@@ -1690,6 +1713,17 @@ class Project < ActiveRecord::Base
     Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
   end
 
+  # Refreshes the expiration time of the associated import job ID.
+  #
+  # This method can be used by asynchronous importers to refresh the status,
+  # preventing the StuckImportJobsWorker from marking the import as failed.
+  def refresh_import_jid_expiration
+    return unless import_jid
+
+    Gitlab::SidekiqStatus
+      .set(import_jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
+  end
+
   private
 
   def storage
diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb
index 976d85246a8..768f0a7472e 100644
--- a/app/models/project_services/hipchat_service.rb
+++ b/app/models/project_services/hipchat_service.rb
@@ -51,8 +51,10 @@ class HipchatService < Service
 
   def execute(data)
     return unless supported_events.include?(data[:object_kind])
+
     message = create_message(data)
     return unless message.present?
+
     gate[room].send('GitLab', message, message_options(data)) # rubocop:disable GitlabSecurity/PublicSend
   end
 
diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb
index b487378edd2..1c065e1ddbd 100644
--- a/app/models/project_services/jira_service.rb
+++ b/app/models/project_services/jira_service.rb
@@ -176,6 +176,7 @@ class JiraService < IssueTrackerService
 
   def test_settings
     return unless client_url.present?
+
     # Test settings by getting the project
     jira_request { client.ServerInfo.all.attrs }
   end
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index 5080acffb3c..b82567ce2b3 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -1,3 +1,8 @@
+##
+# NOTE:
+# We'll move this class to Clusters::Platforms::Kubernetes, which contains exactly the same logic.
+# After we've migrated data, we'll remove KubernetesService. This would happen in a few months.
+# If you're modyfiyng this class, please note that you should update the same change in Clusters::Platforms::Kubernetes.
 class KubernetesService < DeploymentService
   include Gitlab::CurrentSettings
   include Gitlab::Kubernetes
@@ -182,6 +187,7 @@ class KubernetesService < DeploymentService
     kubeclient.get_pods(namespace: actual_namespace).as_json
   rescue KubeException => err
     raise err unless err.error_code == 404
+
     []
   end
 
diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb
index 217f753f05f..fa7b3f2bcaf 100644
--- a/app/models/project_services/prometheus_service.rb
+++ b/app/models/project_services/prometheus_service.rb
@@ -25,7 +25,7 @@ class PrometheusService < MonitoringService
   end
 
   def description
-    'Prometheus monitoring'
+    s_('PrometheusService|Prometheus monitoring')
   end
 
   def self.to_param
@@ -38,8 +38,8 @@ class PrometheusService < MonitoringService
         type: 'text',
         name: 'api_url',
         title: 'API URL',
-        placeholder: 'Prometheus API Base URL, like http://prometheus.example.com/',
-        help: 'By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.',
+        placeholder: s_('PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/'),
+        help: s_('PrometheusService|By default, Prometheus listens on ‘http://localhost:9090’. It’s not recommended to change the default address and port as this might affect or conflict with other services running on the GitLab server.'),
         required: true
       }
     ]
diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb
index 715b215d1db..17b9d2cf7b4 100644
--- a/app/models/project_statistics.rb
+++ b/app/models/project_statistics.rb
@@ -35,7 +35,9 @@ class ProjectStatistics < ActiveRecord::Base
   end
 
   def update_build_artifacts_size
-    self.build_artifacts_size = project.builds.sum(:artifacts_size)
+    self.build_artifacts_size =
+      project.builds.sum(:artifacts_size) +
+      Ci::JobArtifact.artifacts_size_for(self)
   end
 
   def update_storage_size
diff --git a/app/models/project_team.rb b/app/models/project_team.rb
index 1d35426050e..c679758973a 100644
--- a/app/models/project_team.rb
+++ b/app/models/project_team.rb
@@ -1,4 +1,6 @@
 class ProjectTeam
+  include BulkMemberAccessLoad
+
   attr_accessor :project
 
   def initialize(project)
@@ -157,39 +159,16 @@ class ProjectTeam
   #
   # Returns a Hash mapping user ID -> maximum access level.
   def max_member_access_for_user_ids(user_ids)
-    user_ids = user_ids.uniq
-    key = "max_member_access:#{project.id}"
-
-    access = {}
-
-    if RequestStore.active?
-      RequestStore.store[key] ||= {}
-      access = RequestStore.store[key]
+    max_member_access_for_resource_ids(User, user_ids, project.id) do |user_ids|
+      project.project_authorizations
+             .where(user: user_ids)
+             .group(:user_id)
+             .maximum(:access_level)
     end
-
-    # Look up only the IDs we need
-    user_ids = user_ids - access.keys
-
-    return access if user_ids.empty?
-
-    users_access = project.project_authorizations
-      .where(user: user_ids)
-      .group(:user_id)
-      .maximum(:access_level)
-
-    access.merge!(users_access)
-
-    missing_user_ids = user_ids - users_access.keys
-
-    missing_user_ids.each do |user_id|
-      access[user_id] = Gitlab::Access::NO_ACCESS
-    end
-
-    access
   end
 
   def max_member_access(user_id)
-    max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS
+    max_member_access_for_user_ids([user_id])[user_id]
   end
 
   private
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 43de6809178..a0af749a93f 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -21,7 +21,7 @@ class ProjectWiki
   end
 
   delegate :empty?, to: :pages
-  delegate :repository_storage_path, to: :project
+  delegate :repository_storage_path, :hashed_storage?, to: :project
 
   def path
     @project.path + '.wiki'
@@ -76,8 +76,8 @@ class ProjectWiki
 
   # Returns an Array of Gitlab WikiPage instances or an
   # empty Array if this Wiki has no pages.
-  def pages
-    wiki.pages.map { |page| WikiPage.new(self, page, true) }
+  def pages(limit: nil)
+    wiki.pages(limit: limit).map { |page| WikiPage.new(self, page, true) }
   end
 
   # Finds a page within the repository based on a tile
diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb
index 89bfc5f9a9c..d28fed11ca8 100644
--- a/app/models/protected_branch.rb
+++ b/app/models/protected_branch.rb
@@ -10,7 +10,9 @@ class ProtectedBranch < ActiveRecord::Base
   def self.protected?(project, ref_name)
     return true if project.empty_repo? && default_branch_protected?
 
-    self.matching(ref_name, protected_refs: project.protected_branches).present?
+    refs = project.protected_branches.select(:name)
+
+    self.matching(ref_name, protected_refs: refs).present?
   end
 
   def self.default_branch_protected?
diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb
index f38109c0e52..42a9bcf7723 100644
--- a/app/models/protected_tag.rb
+++ b/app/models/protected_tag.rb
@@ -5,6 +5,8 @@ class ProtectedTag < ActiveRecord::Base
   protected_ref_access_levels :create
 
   def self.protected?(project, ref_name)
-    self.matching(ref_name, protected_refs: project.protected_tags).present?
+    refs = project.protected_tags.select(:name)
+
+    self.matching(ref_name, protected_refs: refs).present?
   end
 end
diff --git a/app/models/protected_tag/create_access_level.rb b/app/models/protected_tag/create_access_level.rb
index c7e1319719d..6b6ab3d8279 100644
--- a/app/models/protected_tag/create_access_level.rb
+++ b/app/models/protected_tag/create_access_level.rb
@@ -1,18 +1,6 @@
 class ProtectedTag::CreateAccessLevel < ActiveRecord::Base
   include ProtectedTagAccess
 
-  validates :access_level, presence: true, inclusion: { in: [Gitlab::Access::MASTER,
-                                                             Gitlab::Access::DEVELOPER,
-                                                             Gitlab::Access::NO_ACCESS] }
-
-  def self.human_access_levels
-    {
-      Gitlab::Access::MASTER => "Masters",
-      Gitlab::Access::DEVELOPER => "Developers + Masters",
-      Gitlab::Access::NO_ACCESS => "No one"
-    }.with_indifferent_access
-  end
-
   def check_access(user)
     return false if access_level == Gitlab::Access::NO_ACCESS
 
diff --git a/app/models/repository.rb b/app/models/repository.rb
index eb7766d040c..751306188a0 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -37,7 +37,7 @@ class Repository
                       issue_template_names merge_request_template_names).freeze
 
   # Methods that use cache_method but only memoize the value
-  MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
+  MEMOIZED_CACHED_METHODS = %i(license).freeze
 
   # Certain method caches should be refreshed when certain types of files are
   # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
@@ -132,7 +132,8 @@ class Repository
 
     commits = Gitlab::Git::Commit.where(options)
     commits = Commit.decorate(commits, @project) if commits.present?
-    commits
+
+    CommitCollection.new(project, commits, ref)
   end
 
   def commits_between(from, to)
@@ -148,11 +149,14 @@ class Repository
     end
 
     raw_repository.gitaly_migrate(:commits_by_message) do |is_enabled|
-      if is_enabled
-        find_commits_by_message_by_gitaly(query, ref, path, limit, offset)
-      else
-        find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
-      end
+      commits =
+        if is_enabled
+          find_commits_by_message_by_gitaly(query, ref, path, limit, offset)
+        else
+          find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
+        end
+
+      CommitCollection.new(project, commits, ref)
     end
   end
 
@@ -213,11 +217,7 @@ class Repository
   def branch_exists?(branch_name)
     return false unless raw_repository
 
-    @branch_exists_memo ||= Hash.new do |hash, key|
-      hash[key] = raw_repository.branch_exists?(key)
-    end
-
-    @branch_exists_memo[branch_name]
+    branch_names.include?(branch_name)
   end
 
   def ref_exists?(ref)
@@ -242,6 +242,7 @@ class Repository
       Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}"
     rescue Rugged::OSError => ex
       raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/
+
       Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}"
     end
   end
@@ -255,7 +256,7 @@ class Repository
   end
 
   def diverging_commit_counts(branch)
-    root_ref_hash = raw_repository.rev_parse_target(root_ref).oid
+    root_ref_hash = raw_repository.commit(root_ref).id
     cache.fetch(:"diverging_commit_counts_#{branch.name}") do
       # Rugged seems to throw a `ReferenceError` when given branch_names rather
       # than SHA-1 hashes
@@ -473,6 +474,11 @@ class Repository
     nil
   end
 
+  # items is an Array like: [[oid, path], [oid1, path1]]
+  def blobs_at(items)
+    raw_repository.batch_blobs(items).map { |blob| Blob.decorate(blob, project) }
+  end
+
   def root_ref
     if raw_repository
       raw_repository.root_ref
@@ -491,7 +497,11 @@ class Repository
   end
   cache_method :exists?
 
-  delegate :empty?, to: :raw_repository
+  def empty?
+    return true unless exists?
+
+    !has_visible_content?
+  end
   cache_method :empty?
 
   # The size of this repository in megabytes.
@@ -662,6 +672,7 @@ class Repository
   def next_branch(name, opts = {})
     branch_ids = self.branch_names.map do |n|
       next 1 if n == name
+
       result = n.match(/\A#{name}-([0-9]+)\z/)
       result[1].to_i if result
     end.compact
@@ -902,19 +913,13 @@ class Repository
     end
   end
 
-  def merged_to_root_ref?(branch_or_name, pre_loaded_merged_branches = nil)
+  def merged_to_root_ref?(branch_or_name)
     branch = Gitlab::Git::Branch.find(self, branch_or_name)
 
     if branch
       @root_ref_sha ||= commit(root_ref).sha
       same_head = branch.target == @root_ref_sha
-      merged =
-        if pre_loaded_merged_branches
-          pre_loaded_merged_branches.include?(branch.name)
-        else
-          ancestor?(branch.target, @root_ref_sha)
-        end
-
+      merged = ancestor?(branch.target, @root_ref_sha)
       !same_head && merged
     else
       nil
@@ -943,13 +948,8 @@ class Repository
     end
   end
 
-  def empty_repo?
-    !exists? || !has_visible_content?
-  end
-  cache_method :empty_repo?, memoize_only: true
-
   def search_files_by_content(query, ref)
-    return [] if empty_repo? || query.blank?
+    return [] if empty? || query.blank?
 
     offset = 2
     args = %W(grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
@@ -958,13 +958,26 @@ class Repository
   end
 
   def search_files_by_name(query, ref)
-    return [] if empty_repo? || query.blank?
+    return [] if empty? || query.blank?
 
     args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)})
 
     run_git(args).first.lines.map(&:strip)
   end
 
+  def fetch_as_mirror(url, forced: false, refmap: :all_refs, remote_name: nil)
+    unless remote_name
+      remote_name = "tmp-#{SecureRandom.hex}"
+      tmp_remote_name = true
+    end
+
+    add_remote(remote_name, url)
+    set_remote_as_mirror(remote_name, refmap: refmap)
+    fetch_remote(remote_name, forced: forced)
+  ensure
+    remove_remote(remote_name) if tmp_remote_name
+  end
+
   def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
     gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
@@ -973,6 +986,10 @@ class Repository
     raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
   end
 
+  def remote_exists?(name)
+    raw_repository.remote_exists?(name)
+  end
+
   def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
     raw_repository.compare_source_branch(target_branch_name, source_repository.raw_repository, source_branch_name, straight: straight)
   end
@@ -986,10 +1003,6 @@ class Repository
     raw_repository.ls_files(actual_ref)
   end
 
-  def gitattribute(path, name)
-    raw_repository.attributes(path)[name]
-  end
-
   def copy_gitattributes(ref)
     actual_ref = ref || root_ref
     begin
@@ -1058,6 +1071,14 @@ class Repository
     blob_data_at(sha, path)
   end
 
+  def fetch_ref(source_repository, source_ref:, target_ref:)
+    raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref)
+  end
+
+  def repository_storage_path
+    @project.repository_storage_path
+  end
+
   private
 
   # TODO Generice finder, later split this on finders by Ref or Oid
@@ -1123,10 +1144,6 @@ class Repository
     raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
   end
 
-  def repository_storage_path
-    @project.repository_storage_path
-  end
-
   def initialize_raw_repository
     Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
   end
diff --git a/app/models/service.rb b/app/models/service.rb
index fdd2605e3e3..3c4f1885dd0 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -211,7 +211,7 @@ class Service < ActiveRecord::Base
   def async_execute(data)
     return unless supported_events.include?(data[:object_kind])
 
-    Sidekiq::Client.enqueue(ProjectServiceWorker, id, data)
+    ProjectServiceWorker.perform_async(id, data)
   end
 
   def issue_tracker?
diff --git a/app/models/snippet.rb b/app/models/snippet.rb
index 9533aa7f555..05a16f11b59 100644
--- a/app/models/snippet.rb
+++ b/app/models/snippet.rb
@@ -9,6 +9,7 @@ class Snippet < ActiveRecord::Base
   include Mentionable
   include Spammable
   include Editable
+  include Gitlab::SQL::Pattern
 
   extend Gitlab::CurrentSettings
 
@@ -75,11 +76,11 @@ class Snippet < ActiveRecord::Base
     @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/)
   end
 
-  def to_reference(from_project = nil, full: false)
+  def to_reference(from = nil, full: false)
     reference = "#{self.class.reference_prefix}#{id}"
 
     if project.present?
-      "#{project.to_reference(from_project, full: full)}#{reference}"
+      "#{project.to_reference(from, full: full)}#{reference}"
     else
       reference
     end
@@ -135,10 +136,7 @@ class Snippet < ActiveRecord::Base
     #
     # Returns an ActiveRecord::Relation.
     def search(query)
-      t = arel_table
-      pattern = "%#{query}%"
-
-      where(t[:title].matches(pattern).or(t[:file_name].matches(pattern)))
+      fuzzy_search(query, [:title, :file_name])
     end
 
     # Searches for snippets with matching content.
@@ -149,10 +147,7 @@ class Snippet < ActiveRecord::Base
     #
     # Returns an ActiveRecord::Relation.
     def search_code(query)
-      table   = Snippet.arel_table
-      pattern = "%#{query}%"
-
-      where(table[:content].matches(pattern))
+      fuzzy_search(query, [:content])
     end
   end
 end
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index f025f40994e..fae1b64961a 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,7 +4,6 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
-    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb
index 1f9f8d7286b..29035480371 100644
--- a/app/models/system_note_metadata.rb
+++ b/app/models/system_note_metadata.rb
@@ -1,4 +1,14 @@
 class SystemNoteMetadata < ActiveRecord::Base
+  # These notes's action text might contain a reference that is external.
+  # We should always force a deep validation upon references that are found
+  # in this note type.
+  # Other notes can always be safely shown as all its references are
+  # in the same project (i.e. with the same permissions)
+  TYPES_WITH_CROSS_REFERENCES = %w[
+    commit cross_reference
+    close duplicate
+  ].freeze
+
   ICON_TYPES = %w[
     commit description merge confidential visible label assignee cross_reference
     title time_tracking branch milestone discussion task moved
diff --git a/app/models/user.rb b/app/models/user.rb
index aa88cda4dc0..af1c36d9c93 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -7,6 +7,7 @@ class User < ActiveRecord::Base
   include Gitlab::ConfigHelper
   include Gitlab::CurrentSettings
   include Gitlab::SQL::Pattern
+  include AfterCommitQueue
   include Avatarable
   include Referable
   include Sortable
@@ -16,6 +17,7 @@ class User < ActiveRecord::Base
   include FeatureGate
   include CreatedAtFilterable
   include IgnorableColumn
+  include BulkMemberAccessLoad
 
   DEFAULT_NOTIFICATION_LEVEL = :participating
 
@@ -170,6 +172,7 @@ class User < ActiveRecord::Base
   after_save :ensure_namespace_correct
   after_update :username_changed_hook, if: :username_changed?
   after_destroy :post_destroy_hook
+  after_destroy :remove_key_cache
   after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') }
   after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') }
 
@@ -267,18 +270,22 @@ class User < ActiveRecord::Base
       end
     end
 
+    def for_github_id(id)
+      joins(:identities).merge(Identity.with_extern_uid(:github, id))
+    end
+
     # Find a User by their primary email or any associated secondary email
     def find_by_any_email(email)
-      sql = 'SELECT *
-      FROM users
-      WHERE id IN (
-        SELECT id FROM users WHERE email = :email
-        UNION
-        SELECT emails.user_id FROM emails WHERE email = :email
-      )
-      LIMIT 1;'
+      by_any_email(email).take
+    end
+
+    # Returns a relation containing all the users for the given Email address
+    def by_any_email(email)
+      users = where(email: email)
+      emails = joins(:emails).where(emails: { email: email })
+      union = Gitlab::SQL::Union.new([users, emails])
 
-      User.find_by_sql([sql, { email: email }]).first
+      from("(#{union.to_sql}) #{table_name}")
     end
 
     def filter(filter_name)
@@ -308,9 +315,6 @@ class User < ActiveRecord::Base
     #
     # Returns an ActiveRecord::Relation.
     def search(query)
-      table   = arel_table
-      pattern = User.to_pattern(query)
-
       order = <<~SQL
         CASE
           WHEN users.name = %{query} THEN 0
@@ -320,11 +324,8 @@ class User < ActiveRecord::Base
         END
       SQL
 
-      where(
-        table[:name].matches(pattern)
-          .or(table[:email].matches(pattern))
-          .or(table[:username].matches(pattern))
-      ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name)
+      fuzzy_search(query, [:name, :email, :username])
+        .reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name)
     end
 
     # searches user by given pattern
@@ -332,16 +333,16 @@ class User < ActiveRecord::Base
     # This method uses ILIKE on PostgreSQL and LIKE on MySQL.
 
     def search_with_secondary_emails(query)
-      table = arel_table
       email_table = Email.arel_table
-      pattern = "%#{query}%"
-      matched_by_emails_user_ids = email_table.project(email_table[:user_id]).where(email_table[:email].matches(pattern))
+      matched_by_emails_user_ids = email_table
+        .project(email_table[:user_id])
+        .where(Email.fuzzy_arel_match(:email, query))
 
       where(
-        table[:name].matches(pattern)
-          .or(table[:email].matches(pattern))
-          .or(table[:username].matches(pattern))
-          .or(table[:id].in(matched_by_emails_user_ids))
+        fuzzy_arel_match(:name, query)
+          .or(fuzzy_arel_match(:email, query))
+          .or(fuzzy_arel_match(:username, query))
+          .or(arel_table[:id].in(matched_by_emails_user_ids))
       )
     end
 
@@ -432,7 +433,7 @@ class User < ActiveRecord::Base
     username
   end
 
-  def to_reference(_from_project = nil, target_project: nil, full: nil)
+  def to_reference(_from = nil, target_project: nil, full: nil)
     "#{self.class.reference_prefix}#{username}"
   end
 
@@ -440,6 +441,10 @@ class User < ActiveRecord::Base
     skip_confirmation! if bool
   end
 
+  def skip_reconfirmation=(bool)
+    skip_reconfirmation! if bool
+  end
+
   def generate_reset_token
     @reset_token, enc = Devise.token_generator.generate(self.class, :reset_password_token)
 
@@ -484,7 +489,11 @@ class User < ActiveRecord::Base
   end
 
   def two_factor_u2f_enabled?
-    u2f_registrations.exists?
+    if u2f_registrations.loaded?
+      u2f_registrations.any?
+    else
+      u2f_registrations.exists?
+    end
   end
 
   def namespace_uniq
@@ -619,21 +628,39 @@ class User < ActiveRecord::Base
   end
 
   def require_ssh_key?
-    keys.count == 0 && Gitlab::ProtocolAccess.allowed?('ssh')
+    count = Users::KeysCountService.new(self).count
+
+    count.zero? && Gitlab::ProtocolAccess.allowed?('ssh')
+  end
+
+  def require_password_creation_for_web?
+    allow_password_authentication_for_web? && password_automatically_set?
   end
 
-  def require_password_creation?
-    password_automatically_set? && allow_password_authentication?
+  def require_password_creation_for_git?
+    allow_password_authentication_for_git? && password_automatically_set?
   end
 
   def require_personal_access_token_creation_for_git_auth?
-    return false if current_application_settings.password_authentication_enabled? || ldap_user?
+    return false if allow_password_authentication_for_git? || ldap_user?
 
     PersonalAccessTokensFinder.new(user: self, impersonation: false, state: 'active').execute.none?
   end
 
+  def require_extra_setup_for_git_auth?
+    require_password_creation_for_git? || require_personal_access_token_creation_for_git_auth?
+  end
+
   def allow_password_authentication?
-    !ldap_user? && current_application_settings.password_authentication_enabled?
+    allow_password_authentication_for_web? || allow_password_authentication_for_git?
+  end
+
+  def allow_password_authentication_for_web?
+    current_application_settings.password_authentication_enabled_for_web? && !ldap_user?
+  end
+
+  def allow_password_authentication_for_git?
+    current_application_settings.password_authentication_enabled_for_git? && !ldap_user?
   end
 
   def can_change_username?
@@ -878,9 +905,14 @@ class User < ActiveRecord::Base
 
   def post_destroy_hook
     log_info("User \"#{name}\" (#{email})  was removed")
+
     system_hook_service.execute_hooks_for(self, :destroy)
   end
 
+  def remove_key_cache
+    Users::KeysCountService.new(self).delete_cache
+  end
+
   def delete_async(deleted_by:, params: {})
     block if params[:hard_delete]
     DeleteUserWorker.perform_async(deleted_by.id, id, params)
@@ -916,7 +948,16 @@ class User < ActiveRecord::Base
   end
 
   def manageable_namespaces
-    @manageable_namespaces ||= [namespace] + owned_groups + masters_groups
+    @manageable_namespaces ||= [namespace] + manageable_groups
+  end
+
+  def manageable_groups
+    union = Gitlab::SQL::Union.new([owned_groups.select(:id),
+                                    masters_groups.select(:id)])
+    arel_union = Arel::Nodes::SqlLiteral.new(union.to_sql)
+    owned_and_master_groups = Group.where(Group.arel_table[:id].in(arel_union))
+
+    Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants
   end
 
   def namespaces
@@ -964,7 +1005,11 @@ class User < ActiveRecord::Base
   end
 
   def notification_settings_for(source)
-    notification_settings.find_or_initialize_by(source: source)
+    if notification_settings.loaded?
+      notification_settings.find { |notification| notification.source == source }
+    else
+      notification_settings.find_or_initialize_by(source: source)
+    end
   end
 
   # Lazy load global notification setting
@@ -1100,11 +1145,40 @@ class User < ActiveRecord::Base
     super
   end
 
+  # Determine the maximum access level for a group of projects in bulk.
+  #
+  # Returns a Hash mapping project ID -> maximum access level.
+  def max_member_access_for_project_ids(project_ids)
+    max_member_access_for_resource_ids(Project, project_ids) do |project_ids|
+      project_authorizations.where(project: project_ids)
+                            .group(:project_id)
+                            .maximum(:access_level)
+    end
+  end
+
+  def max_member_access_for_project(project_id)
+    max_member_access_for_project_ids([project_id])[project_id]
+  end
+
+  # Determine the maximum access level for a group of groups in bulk.
+  #
+  # Returns a Hash mapping project ID -> maximum access level.
+  def max_member_access_for_group_ids(group_ids)
+    max_member_access_for_resource_ids(Group, group_ids) do |group_ids|
+      group_members.where(source: group_ids).group(:source_id).maximum(:access_level)
+    end
+  end
+
+  def max_member_access_for_group(group_id)
+    max_member_access_for_group_ids([group_id])[group_id]
+  end
+
   protected
 
   # override, from Devise::Validatable
   def password_required?
     return false if internal?
+
     super
   end
 
@@ -1122,6 +1196,7 @@ class User < ActiveRecord::Base
   # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration
   def send_devise_notification(notification, *args)
     return true unless can?(:receive_notifications)
+
     devise_mailer.__send__(notification, self, *args).deliver_later # rubocop:disable GitlabSecurity/PublicSend
   end
 
diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb
index 5f710961f95..bdfef677ef3 100644
--- a/app/models/wiki_page.rb
+++ b/app/models/wiki_page.rb
@@ -127,19 +127,24 @@ class WikiPage
     @version ||= @page.version
   end
 
-  # Returns an array of Gitlab Commit instances.
-  def versions
+  def versions(options = {})
     return [] unless persisted?
 
-    wiki.wiki.page_versions(@page.path)
+    wiki.wiki.page_versions(@page.path, options)
   end
 
-  def commit
-    versions.first
+  def count_versions
+    return [] unless persisted?
+
+    wiki.wiki.count_page_versions(@page.path)
+  end
+
+  def last_version
+    @last_version ||= versions(limit: 1).first
   end
 
   def last_commit_sha
-    commit&.sha
+    last_version&.sha
   end
 
   # Returns the Date that this latest version was
@@ -151,7 +156,7 @@ class WikiPage
   # Returns boolean True or False if this instance
   # is an old version of the page.
   def historical?
-    @page.historical? && versions.first.sha != version.sha
+    @page.historical? && last_version.sha != version.sha
   end
 
   # Returns boolean True or False if this instance