Merge commit 'f2f58a60b76acd479e37bdbc9246ec9f9b2bea82' into object-storage-ee-to-ce-backport

49 files changed, 1053 insertions, 370 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index c0cc60d5ebf..5e16badabec 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -33,6 +33,8 @@ class ApplicationSetting < ActiveRecord::Base
 
   attr_accessor :domain_whitelist_raw, :domain_blacklist_raw
 
+  default_value_for :id, 1
+
   validates :uuid, presence: true
 
   validates :session_expire_delay,
@@ -151,6 +153,25 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { greater_than_or_equal_to: 0 }
 
+  validates :circuitbreaker_backoff_threshold,
+            :circuitbreaker_failure_count_threshold,
+            :circuitbreaker_failure_wait_time,
+            :circuitbreaker_failure_reset_time,
+            :circuitbreaker_storage_timeout,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+
+  validates :circuitbreaker_access_retries,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 1 }
+
+  validates_each :circuitbreaker_backoff_threshold do |record, attr, value|
+    if value.to_i >= record.circuitbreaker_failure_count_threshold
+      record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\
+                                "lower than the failure count threshold"))
+    end
+  end
+
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
   end
@@ -194,7 +215,10 @@ class ApplicationSetting < ActiveRecord::Base
     ensure_cache_setup
 
     Rails.cache.fetch(CACHE_KEY) do
-      ApplicationSetting.last
+      ApplicationSetting.last.tap do |settings|
+        # do not cache nils
+        raise 'missing settings' unless settings
+      end
     end
   rescue
     # Fall back to an uncached value if there are any problems (e.g. redis down)
@@ -396,7 +420,7 @@ class ApplicationSetting < ActiveRecord::Base
   #   the enabling/disabling is `performance_bar_allowed_group_id`
   # - If `enable` is false, we set `performance_bar_allowed_group_id` to `nil`
   def performance_bar_enabled=(enable)
-    return if enable
+    return if Gitlab::Utils.to_boolean(enable)
 
     self.performance_bar_allowed_group_id = nil
   end
diff --git a/app/models/blob.rb b/app/models/blob.rb
index 954d4e4d779..ad0bc2e2ead 100644
--- a/app/models/blob.rb
+++ b/app/models/blob.rb
@@ -156,7 +156,9 @@ class Blob < SimpleDelegator
   end
 
   def file_type
-    Gitlab::FileDetector.type_of(path)
+    name = File.basename(path)
+
+    Gitlab::FileDetector.type_of(path) || Gitlab::FileDetector.type_of(name)
   end
 
   def video?
diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb
index 8b66531ec7b..ec56cc53aea 100644
--- a/app/models/ci/artifact_blob.rb
+++ b/app/models/ci/artifact_blob.rb
@@ -2,7 +2,7 @@ module Ci
   class ArtifactBlob
     include BlobLike
 
-    EXTENTIONS_SERVED_BY_PAGES = %w[.html .htm .txt .json].freeze
+    EXTENSIONS_SERVED_BY_PAGES = %w[.html .htm .txt .json].freeze
 
     attr_reader :entry
 
@@ -36,17 +36,22 @@ module Ci
     def external_url(project, job)
       return unless external_link?(job)
 
-      components = project.full_path_components
-      components << "-/jobs/#{job.id}/artifacts/file/#{path}"
-      artifact_path = components[1..-1].join('/')
+      full_path_parts = project.full_path_components
+      top_level_group = full_path_parts.shift
 
-      "#{pages_config.protocol}://#{components[0]}.#{pages_config.host}/#{artifact_path}"
+      artifact_path = [
+        '-', *full_path_parts, '-',
+        'jobs', job.id,
+        'artifacts', path
+      ].join('/')
+
+      "#{pages_config.protocol}://#{top_level_group}.#{pages_config.host}/#{artifact_path}"
     end
 
     def external_link?(job)
       pages_config.enabled &&
         pages_config.artifacts_server &&
-        EXTENTIONS_SERVED_BY_PAGES.include?(File.extname(name)) &&
+        EXTENSIONS_SERVED_BY_PAGES.include?(File.extname(name)) &&
         job.project.public?
     end
 
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index cf3ce3c9e54..ca65e81f27a 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -249,9 +249,7 @@ module Ci
     end
 
     def commit
-      @commit ||= project.commit(sha)
-    rescue
-      nil
+      @commit ||= project.commit_by(oid: sha)
     end
 
     def branch?
diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb
new file mode 100644
index 00000000000..c7949d11ef8
--- /dev/null
+++ b/app/models/clusters/applications/helm.rb
@@ -0,0 +1,35 @@
+module Clusters
+  module Applications
+    class Helm < ActiveRecord::Base
+      self.table_name = 'clusters_applications_helm'
+
+      include ::Clusters::Concerns::ApplicationStatus
+
+      belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
+
+      default_value_for :version, Gitlab::Kubernetes::Helm::HELM_VERSION
+
+      validates :cluster, presence: true
+
+      after_initialize :set_initial_status
+
+      def self.application_name
+        self.to_s.demodulize.underscore
+      end
+
+      def set_initial_status
+        return unless not_installable?
+
+        self.status = 'installable' if cluster&.platform_kubernetes_active?
+      end
+
+      def name
+        self.class.application_name
+      end
+
+      def install_command
+        Gitlab::Kubernetes::Helm::InstallCommand.new(name, true)
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb
new file mode 100644
index 00000000000..44bd979741e
--- /dev/null
+++ b/app/models/clusters/applications/ingress.rb
@@ -0,0 +1,44 @@
+module Clusters
+  module Applications
+    class Ingress < ActiveRecord::Base
+      self.table_name = 'clusters_applications_ingress'
+
+      include ::Clusters::Concerns::ApplicationStatus
+
+      belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
+
+      validates :cluster, presence: true
+
+      default_value_for :ingress_type, :nginx
+      default_value_for :version, :nginx
+
+      after_initialize :set_initial_status
+
+      enum ingress_type: {
+        nginx: 1
+      }
+
+      def self.application_name
+        self.to_s.demodulize.underscore
+      end
+
+      def set_initial_status
+        return unless not_installable?
+
+        self.status = 'installable' if cluster&.application_helm_installed?
+      end
+
+      def name
+        self.class.application_name
+      end
+
+      def chart
+        'stable/nginx-ingress'
+      end
+
+      def install_command
+        Gitlab::Kubernetes::Helm::InstallCommand.new(name, false, chart)
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
new file mode 100644
index 00000000000..185d9473aab
--- /dev/null
+++ b/app/models/clusters/cluster.rb
@@ -0,0 +1,102 @@
+module Clusters
+  class Cluster < ActiveRecord::Base
+    include Presentable
+
+    self.table_name = 'clusters'
+
+    APPLICATIONS = {
+      Applications::Helm.application_name => Applications::Helm,
+      Applications::Ingress.application_name => Applications::Ingress
+    }.freeze
+
+    belongs_to :user
+
+    has_many :cluster_projects, class_name: 'Clusters::Project'
+    has_many :projects, through: :cluster_projects, class_name: '::Project'
+
+    # we force autosave to happen when we save `Cluster` model
+    has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
+
+    # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration
+    has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+
+    has_one :application_helm, class_name: 'Clusters::Applications::Helm'
+    has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
+
+    accepts_nested_attributes_for :provider_gcp, update_only: true
+    accepts_nested_attributes_for :platform_kubernetes, update_only: true
+
+    validates :name, cluster_name: true
+    validate :restrict_modification, on: :update
+
+    # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3
+    # We need callback here because `enabled` belongs to Clusters::Cluster
+    # Callbacks in Clusters::Platforms::Kubernetes will not be called after update
+    after_save :update_kubernetes_integration!
+
+    delegate :status, to: :provider, allow_nil: true
+    delegate :status_reason, to: :provider, allow_nil: true
+    delegate :on_creation?, to: :provider, allow_nil: true
+    delegate :update_kubernetes_integration!, to: :platform, allow_nil: true
+
+    delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true
+    delegate :installed?, to: :application_helm, prefix: true, allow_nil: true
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+
+    scope :enabled, -> { where(enabled: true) }
+    scope :disabled, -> { where(enabled: false) }
+
+    def status_name
+      if provider
+        provider.status_name
+      else
+        :created
+      end
+    end
+
+    def applications
+      [
+        application_helm || build_application_helm,
+        application_ingress || build_application_ingress
+      ]
+    end
+
+    def provider
+      return provider_gcp if gcp?
+    end
+
+    def platform
+      return platform_kubernetes if kubernetes?
+    end
+
+    def first_project
+      return @first_project if defined?(@first_project)
+
+      @first_project = projects.first
+    end
+    alias_method :project, :first_project
+
+    def kubeclient
+      platform_kubernetes.kubeclient if kubernetes?
+    end
+
+    private
+
+    def restrict_modification
+      if provider&.on_creation?
+        errors.add(:base, "cannot modify during creation")
+        return false
+      end
+
+      true
+    end
+  end
+end
diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb
new file mode 100644
index 00000000000..7b7c8eac773
--- /dev/null
+++ b/app/models/clusters/concerns/application_status.rb
@@ -0,0 +1,43 @@
+module Clusters
+  module Concerns
+    module ApplicationStatus
+      extend ActiveSupport::Concern
+
+      included do
+        state_machine :status, initial: :not_installable do
+          state :not_installable, value: -2
+          state :errored, value: -1
+          state :installable, value: 0
+          state :scheduled, value: 1
+          state :installing, value: 2
+          state :installed, value: 3
+
+          event :make_scheduled do
+            transition [:installable, :errored] => :scheduled
+          end
+
+          event :make_installing do
+            transition [:scheduled] => :installing
+          end
+
+          event :make_installed do
+            transition [:installing] => :installed
+          end
+
+          event :make_errored do
+            transition any => :errored
+          end
+
+          before_transition any => [:scheduled] do |app_status, _|
+            app_status.status_reason = nil
+          end
+
+          before_transition any => [:errored] do |app_status, transition|
+            status_reason = transition.args.first
+            app_status.status_reason = status_reason if status_reason
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
new file mode 100644
index 00000000000..6dc1ee810d3
--- /dev/null
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -0,0 +1,109 @@
+module Clusters
+  module Platforms
+    class Kubernetes < ActiveRecord::Base
+      self.table_name = 'cluster_platforms_kubernetes'
+
+      belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster'
+
+      attr_encrypted :password,
+        mode: :per_attribute_iv,
+        key: Gitlab::Application.secrets.db_key_base,
+        algorithm: 'aes-256-cbc'
+
+      attr_encrypted :token,
+        mode: :per_attribute_iv,
+        key: Gitlab::Application.secrets.db_key_base,
+        algorithm: 'aes-256-cbc'
+
+      before_validation :enforce_namespace_to_lower_case
+
+      validates :namespace,
+        allow_blank: true,
+        length: 1..63,
+        format: {
+          with: Gitlab::Regex.kubernetes_namespace_regex,
+          message: Gitlab::Regex.kubernetes_namespace_regex_message
+        }
+
+      # We expect to be `active?` only when enabled and cluster is created (the api_url is assigned)
+      validates :api_url, url: true, presence: true
+      validates :token, presence: true
+
+      # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes
+      after_destroy :destroy_kubernetes_integration!
+
+      alias_attribute :ca_pem, :ca_cert
+
+      delegate :project, to: :cluster, allow_nil: true
+      delegate :enabled?, to: :cluster, allow_nil: true
+
+      class << self
+        def namespace_for_project(project)
+          "#{project.path}-#{project.id}"
+        end
+      end
+
+      def actual_namespace
+        if namespace.present?
+          namespace
+        else
+          default_namespace
+        end
+      end
+
+      def default_namespace
+        self.class.namespace_for_project(project) if project
+      end
+
+      def kubeclient
+        @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service?
+      end
+
+      def update_kubernetes_integration!
+        raise 'Kubernetes service already configured' unless manages_kubernetes_service?
+
+        # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false
+        cluster.reload
+
+        ensure_kubernetes_service&.update!(
+          active: enabled?,
+          api_url: api_url,
+          namespace: namespace,
+          token: token,
+          ca_pem: ca_cert
+        )
+      end
+
+      def active?
+        manages_kubernetes_service?
+      end
+
+      private
+
+      def enforce_namespace_to_lower_case
+        self.namespace = self.namespace&.downcase
+      end
+
+      # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class
+      def manages_kubernetes_service?
+        return true unless kubernetes_service&.active?
+
+        kubernetes_service.api_url == api_url
+      end
+
+      def destroy_kubernetes_integration!
+        return unless manages_kubernetes_service?
+
+        kubernetes_service&.destroy!
+      end
+
+      def kubernetes_service
+        @kubernetes_service ||= project&.kubernetes_service
+      end
+
+      def ensure_kubernetes_service
+        @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/project.rb b/app/models/clusters/project.rb
new file mode 100644
index 00000000000..eeb734b20b8
--- /dev/null
+++ b/app/models/clusters/project.rb
@@ -0,0 +1,8 @@
+module Clusters
+  class Project < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'Clusters::Cluster'
+    belongs_to :project, class_name: '::Project'
+  end
+end
diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb
new file mode 100644
index 00000000000..ee2e43ee9dd
--- /dev/null
+++ b/app/models/clusters/providers/gcp.rb
@@ -0,0 +1,79 @@
+module Clusters
+  module Providers
+    class Gcp < ActiveRecord::Base
+      self.table_name = 'cluster_providers_gcp'
+
+      belongs_to :cluster, inverse_of: :provider_gcp, class_name: 'Clusters::Cluster'
+
+      default_value_for :zone, 'us-central1-a'
+      default_value_for :num_nodes, 3
+      default_value_for :machine_type, 'n1-standard-2'
+
+      attr_encrypted :access_token,
+        mode: :per_attribute_iv,
+        key: Gitlab::Application.secrets.db_key_base,
+        algorithm: 'aes-256-cbc'
+
+      validates :gcp_project_id,
+        length: 1..63,
+        format: {
+          with: Gitlab::Regex.kubernetes_namespace_regex,
+          message: Gitlab::Regex.kubernetes_namespace_regex_message
+        }
+
+      validates :zone, presence: true
+
+      validates :num_nodes,
+        presence: true,
+        numericality: {
+          only_integer: true,
+          greater_than: 0
+        }
+
+      state_machine :status, initial: :scheduled do
+        state :scheduled, value: 1
+        state :creating, value: 2
+        state :created, value: 3
+        state :errored, value: 4
+
+        event :make_creating do
+          transition any - [:creating] => :creating
+        end
+
+        event :make_created do
+          transition any - [:created] => :created
+        end
+
+        event :make_errored do
+          transition any - [:errored] => :errored
+        end
+
+        before_transition any => [:errored, :created] do |provider|
+          provider.access_token = nil
+          provider.operation_id = nil
+        end
+
+        before_transition any => [:creating] do |provider, transition|
+          operation_id = transition.args.first
+          raise ArgumentError.new('operation_id is required') unless operation_id.present?
+          provider.operation_id = operation_id
+        end
+
+        before_transition any => [:errored] do |provider, transition|
+          status_reason = transition.args.first
+          provider.status_reason = status_reason if status_reason
+        end
+      end
+
+      def on_creation?
+        scheduled? || creating?
+      end
+
+      def api_client
+        return unless access_token
+
+        @api_client ||= GoogleApi::CloudPlatform::Client.new(access_token, nil)
+      end
+    end
+  end
+end
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index f3888528940..6b07dbdf3ea 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -14,7 +14,6 @@ class CommitStatus < ActiveRecord::Base
   delegate :sha, :short_sha, to: :pipeline
 
   validates :pipeline, presence: true, unless: :importing?
-
   validates :name, presence: true, unless: :importing?
 
   alias_attribute :author, :user
@@ -46,6 +45,17 @@ class CommitStatus < ActiveRecord::Base
     runner_system_failure: 4
   }
 
+  ##
+  # We still create some CommitStatuses outside of CreatePipelineService.
+  #
+  # These are pages deployments and external statuses.
+  #
+  before_create unless: :importing? do
+    Ci::EnsureStageService.new(project, user).execute(self) do |stage|
+      self.run_after_commit { StageUpdateWorker.perform_async(stage.id) }
+    end
+  end
+
   state_machine :status do
     event :process do
       transition [:skipped, :manual] => :created
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb
index 8fbfed11bdf..2ec70203710 100644
--- a/app/models/concerns/avatarable.rb
+++ b/app/models/concerns/avatarable.rb
@@ -11,7 +11,7 @@ module Avatarable
 
     # If asset_host is set then it is expected that assets are handled by a standalone host.
     # That means we do not want to get GitLab's relative_url_root option anymore.
-    host = asset_host.present? ? asset_host : gitlab_host
+    host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host
 
     [host, avatar.url].join
   end
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
index 9417033d1f6..98776eab424 100644
--- a/app/models/concerns/cache_markdown_field.rb
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -49,7 +49,8 @@ module CacheMarkdownField
 
     # Always include a project key, or Banzai complains
     project = self.project if self.respond_to?(:project)
-    context = cached_markdown_fields[field].merge(project: project)
+    group = self.group if self.respond_to?(:group)
+    context = cached_markdown_fields[field].merge(project: project, group: group)
 
     # Banzai is less strict about authors, so don't always have an author key
     context[:author] = self.author if self.respond_to?(:author)
diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb
new file mode 100644
index 00000000000..01957da0bf3
--- /dev/null
+++ b/app/models/concerns/group_descendant.rb
@@ -0,0 +1,56 @@
+module GroupDescendant
+  # Returns the hierarchy of a project or group in the from of a hash upto a
+  # given top.
+  #
+  # > project.hierarchy
+  # => { parent_group => { child_group => project } }
+  def hierarchy(hierarchy_top = nil, preloaded = nil)
+    preloaded ||= ancestors_upto(hierarchy_top)
+    expand_hierarchy_for_child(self, self, hierarchy_top, preloaded)
+  end
+
+  # Merges all hierarchies of the given groups or projects into an array of
+  # hashes. All ancestors need to be loaded into the given `descendants` to avoid
+  # queries down the line.
+  #
+  # > GroupDescendant.merge_hierarchy([project, child_group, child_group2, parent])
+  # => { parent => [{ child_group => project}, child_group2] }
+  def self.build_hierarchy(descendants, hierarchy_top = nil)
+    descendants = Array.wrap(descendants).uniq
+    return [] if descendants.empty?
+
+    unless descendants.all? { |hierarchy| hierarchy.is_a?(GroupDescendant) }
+      raise ArgumentError.new('element is not a hierarchy')
+    end
+
+    all_hierarchies = descendants.map do |descendant|
+      descendant.hierarchy(hierarchy_top, descendants)
+    end
+
+    Gitlab::Utils::MergeHash.merge(all_hierarchies)
+  end
+
+  private
+
+  def expand_hierarchy_for_child(child, hierarchy, hierarchy_top, preloaded)
+    parent = hierarchy_top if hierarchy_top && child.parent_id == hierarchy_top.id
+    parent ||= preloaded.detect { |possible_parent| possible_parent.is_a?(Group) && possible_parent.id == child.parent_id }
+
+    if parent.nil? && !child.parent_id.nil?
+      raise ArgumentError.new('parent was not preloaded')
+    end
+
+    if parent.nil? && hierarchy_top.present?
+      raise ArgumentError.new('specified top is not part of the tree')
+    end
+
+    if parent && parent != hierarchy_top
+      expand_hierarchy_for_child(parent,
+                                 { parent => hierarchy },
+                                 hierarchy_top,
+                                 preloaded)
+    else
+      hierarchy
+    end
+  end
+end
diff --git a/app/models/concerns/ignorable_column.rb b/app/models/concerns/ignorable_column.rb
index eb9f3423e48..03793e8bcbb 100644
--- a/app/models/concerns/ignorable_column.rb
+++ b/app/models/concerns/ignorable_column.rb
@@ -21,8 +21,8 @@ module IgnorableColumn
       @ignored_columns ||= Set.new
     end
 
-    def ignore_column(name)
-      ignored_columns << name.to_s
+    def ignore_column(*names)
+      ignored_columns.merge(names.map(&:to_s))
     end
   end
 end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index fc30d008dea..c008fb91a16 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -14,10 +14,11 @@ module Issuable
   include StripAttribute
   include Awardable
   include Taskable
-  include TimeTrackable
   include Importable
   include Editable
   include AfterCommitQueue
+  include Sortable
+  include CreatedAtFilterable
 
   # This object is used to gather issuable meta data for displaying
   # upvotes, downvotes, notes and closing merge requests count for issues and merge requests
@@ -95,8 +96,6 @@ module Issuable
 
     strip_attributes :title
 
-    acts_as_paranoid
-
     after_save :record_metrics, unless: :imported?
 
     # We want to use optimistic lock for cases when only title or description are involved
@@ -256,23 +255,22 @@ module Issuable
     participants(user).include?(user)
   end
 
-  def to_hook_data(user)
-    hook_data = {
-      object_kind: self.class.name.underscore,
-      user: user.hook_attrs,
-      project: project.hook_attrs,
-      object_attributes: hook_attrs,
-      labels: labels.map(&:hook_attrs),
-      # DEPRECATED
-      repository: project.hook_attrs.slice(:name, :url, :description, :homepage)
-    }
-    if self.is_a?(Issue)
-      hook_data[:assignees] = assignees.map(&:hook_attrs) if assignees.any?
-    else
-      hook_data[:assignee] = assignee.hook_attrs if assignee
+  def to_hook_data(user, old_labels: [], old_assignees: [])
+    changes = previous_changes
+
+    if old_labels != labels
+      changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)]
+    end
+
+    if old_assignees != assignees
+      if self.is_a?(Issue)
+        changes[:assignees] = [old_assignees.map(&:hook_attrs), assignees.map(&:hook_attrs)]
+      else
+        changes[:assignee] = [old_assignees&.first&.hook_attrs, assignee&.hook_attrs]
+      end
     end
 
-    hook_data
+    Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes)
   end
 
   def labels_array
diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb
new file mode 100644
index 00000000000..dcb3b2b5ff3
--- /dev/null
+++ b/app/models/concerns/loaded_in_group_list.rb
@@ -0,0 +1,72 @@
+module LoadedInGroupList
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def with_counts(archived:)
+      selects_including_counts = [
+        'namespaces.*',
+        "(#{project_count_sql(archived).to_sql}) AS preloaded_project_count",
+        "(#{member_count_sql.to_sql}) AS preloaded_member_count",
+        "(#{subgroup_count_sql.to_sql}) AS preloaded_subgroup_count"
+      ]
+
+      select(selects_including_counts)
+    end
+
+    def with_selects_for_list(archived: nil)
+      with_route.with_counts(archived: archived)
+    end
+
+    private
+
+    def project_count_sql(archived = nil)
+      projects = Project.arel_table
+      namespaces = Namespace.arel_table
+
+      base_count = projects.project(Arel.star.count.as('preloaded_project_count'))
+                     .where(projects[:namespace_id].eq(namespaces[:id]))
+      if archived == 'only'
+        base_count.where(projects[:archived].eq(true))
+      elsif Gitlab::Utils.to_boolean(archived)
+        base_count
+      else
+        base_count.where(projects[:archived].not_eq(true))
+      end
+    end
+
+    def subgroup_count_sql
+      namespaces = Namespace.arel_table
+      children = namespaces.alias('children')
+
+      namespaces.project(Arel.star.count.as('preloaded_subgroup_count'))
+        .from(children)
+        .where(children[:parent_id].eq(namespaces[:id]))
+    end
+
+    def member_count_sql
+      members = Member.arel_table
+      namespaces = Namespace.arel_table
+
+      members.project(Arel.star.count.as('preloaded_member_count'))
+        .where(members[:source_type].eq(Namespace.name))
+        .where(members[:source_id].eq(namespaces[:id]))
+        .where(members[:requested_at].eq(nil))
+    end
+  end
+
+  def children_count
+    @children_count ||= project_count + subgroup_count
+  end
+
+  def project_count
+    @project_count ||= try(:preloaded_project_count) || projects.non_archived.count
+  end
+
+  def subgroup_count
+    @subgroup_count ||= try(:preloaded_subgroup_count) || children.count
+  end
+
+  def member_count
+    @member_count ||= try(:preloaded_member_count) || users.count
+  end
+end
diff --git a/app/models/concerns/repository_mirroring.rb b/app/models/concerns/repository_mirroring.rb
deleted file mode 100644
index f6aba91bc4c..00000000000
--- a/app/models/concerns/repository_mirroring.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-module RepositoryMirroring
-  IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-  IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-
-  def set_remote_as_mirror(name)
-    # This is used to define repository as equivalent as "git clone --mirror"
-    raw_repository.rugged.config["remote.#{name}.fetch"] = 'refs/*:refs/*'
-    raw_repository.rugged.config["remote.#{name}.mirror"] = true
-    raw_repository.rugged.config["remote.#{name}.prune"] = true
-  end
-
-  def set_import_remote_as_mirror(remote_name)
-    # Add first fetch with Rugged so it does not create its own.
-    raw_repository.rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-    add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
-
-    raw_repository.rugged.config["remote.#{remote_name}.mirror"] = true
-    raw_repository.rugged.config["remote.#{remote_name}.prune"] = true
-  end
-
-  def add_remote_fetch_config(remote_name, refspec)
-    run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
-  end
-
-  def fetch_mirror(remote, url)
-    add_remote(remote, url)
-    set_remote_as_mirror(remote)
-    fetch_remote(remote, forced: true)
-    remove_remote(remote)
-  end
-end
diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb
index 5ab5c80a2f5..b3020484738 100644
--- a/app/models/concerns/storage/legacy_namespace.rb
+++ b/app/models/concerns/storage/legacy_namespace.rb
@@ -7,6 +7,8 @@ module Storage
         raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry')
       end
 
+      expires_full_path_cache
+
       # Move the namespace directory in all storage paths used by member projects
       repository_storage_paths.each do |repository_storage_path|
         # Ensure old directory exists before moving it
diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb
index 274b38a7708..f478c8ede18 100644
--- a/app/models/concerns/subscribable.rb
+++ b/app/models/concerns/subscribable.rb
@@ -13,6 +13,8 @@ module Subscribable
   end
 
   def subscribed?(user, project = nil)
+    return false unless user
+
     if subscription = subscriptions.find_by(user: user, project: project)
       subscription.subscribed
     else
diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb
index b517ddaebd7..9f403d96ed5 100644
--- a/app/models/concerns/time_trackable.rb
+++ b/app/models/concerns/time_trackable.rb
@@ -9,7 +9,7 @@ module TimeTrackable
   extend ActiveSupport::Concern
 
   included do
-    attr_reader :time_spent, :time_spent_user
+    attr_reader :time_spent, :time_spent_user, :spent_at
 
     alias_method :time_spent?, :time_spent
 
@@ -24,6 +24,7 @@ module TimeTrackable
   def spend_time(options)
     @time_spent = options[:duration]
     @time_spent_user = options[:user]
+    @spent_at = options[:spent_at]
     @original_total_time_spent = nil
 
     return if @time_spent == 0
@@ -55,7 +56,11 @@ module TimeTrackable
   end
 
   def add_or_subtract_spent_time
-    timelogs.new(time_spent: time_spent, user: @time_spent_user)
+    timelogs.new(
+      time_spent: time_spent,
+      user: @time_spent_user,
+      spent_at: @spent_at
+    )
   end
 
   def check_negative_time_spent
diff --git a/app/models/email.rb b/app/models/email.rb
index 384f38f2db7..2da8b050149 100644
--- a/app/models/email.rb
+++ b/app/models/email.rb
@@ -14,6 +14,8 @@ class Email < ActiveRecord::Base
   devise :confirmable
   self.reconfirmable = false  # currently email can't be changed, no need to reconfirm
 
+  delegate :username, to: :user
+
   def email=(value)
     write_attribute(:email, value.downcase.strip)
   end
diff --git a/app/models/environment.rb b/app/models/environment.rb
index b6868ccbe8f..21a028e351c 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -30,7 +30,6 @@ class Environment < ActiveRecord::Base
                       message: Gitlab::Regex.environment_slug_regex_message }
 
   validates :external_url,
-            uniqueness: { scope: :project_id },
             length: { maximum: 255 },
             allow_nil: true,
             addressable_url: true
@@ -110,7 +109,7 @@ class Environment < ActiveRecord::Base
   end
 
   def ref_path
-    "refs/#{Repository::REF_ENVIRONMENTS}/#{Shellwords.shellescape(name)}"
+    "refs/#{Repository::REF_ENVIRONMENTS}/#{slug}"
   end
 
   def formatted_external_url
@@ -164,6 +163,10 @@ class Environment < ActiveRecord::Base
     end
   end
 
+  def slug
+    super.presence || generate_slug
+  end
+
   # An environment name is not necessarily suitable for use in URLs, DNS
   # or other third-party contexts, so provide a slugified version. A slug has
   # the following properties:
diff --git a/app/models/epic.rb b/app/models/epic.rb
new file mode 100644
index 00000000000..62898a02e2d
--- /dev/null
+++ b/app/models/epic.rb
@@ -0,0 +1,7 @@
+# Placeholder class for model that is implemented in EE
+# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE
+class Epic < ActiveRecord::Base
+  # TODO: this will be implemented as part of #3853
+  def to_reference
+  end
+end
diff --git a/app/models/fork_network.rb b/app/models/fork_network.rb
index 218e37a5312..7f1728e8c77 100644
--- a/app/models/fork_network.rb
+++ b/app/models/fork_network.rb
@@ -12,4 +12,8 @@ class ForkNetwork < ActiveRecord::Base
   def find_forks_in(other_projects)
     projects.where(id: other_projects)
   end
+
+  def merge_requests
+    MergeRequest.where(target_project: projects)
+  end
 end
diff --git a/app/models/gcp/cluster.rb b/app/models/gcp/cluster.rb
deleted file mode 100644
index 18bd6a6dcb4..00000000000
--- a/app/models/gcp/cluster.rb
+++ /dev/null
@@ -1,113 +0,0 @@
-module Gcp
-  class Cluster < ActiveRecord::Base
-    extend Gitlab::Gcp::Model
-    include Presentable
-
-    belongs_to :project, inverse_of: :cluster
-    belongs_to :user
-    belongs_to :service
-
-    default_value_for :gcp_cluster_zone, 'us-central1-a'
-    default_value_for :gcp_cluster_size, 3
-    default_value_for :gcp_machine_type, 'n1-standard-4'
-
-    attr_encrypted :password,
-      mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
-      algorithm: 'aes-256-cbc'
-
-    attr_encrypted :kubernetes_token,
-      mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
-      algorithm: 'aes-256-cbc'
-
-    attr_encrypted :gcp_token,
-      mode: :per_attribute_iv,
-      key: Gitlab::Application.secrets.db_key_base,
-      algorithm: 'aes-256-cbc'
-
-    validates :gcp_project_id,
-      length: 1..63,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message
-      }
-
-    validates :gcp_cluster_name,
-      length: 1..63,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message
-      }
-
-    validates :gcp_cluster_zone, presence: true
-
-    validates :gcp_cluster_size,
-      presence: true,
-      numericality: {
-        only_integer: true,
-        greater_than: 0
-      }
-
-    validates :project_namespace,
-      allow_blank: true,
-      length: 1..63,
-      format: {
-        with: Gitlab::Regex.kubernetes_namespace_regex,
-        message: Gitlab::Regex.kubernetes_namespace_regex_message
-      }
-
-    # if we do not do status transition we prevent change
-    validate :restrict_modification, on: :update, unless: :status_changed?
-
-    state_machine :status, initial: :scheduled do
-      state :scheduled, value: 1
-      state :creating, value: 2
-      state :created, value: 3
-      state :errored, value: 4
-
-      event :make_creating do
-        transition any - [:creating] => :creating
-      end
-
-      event :make_created do
-        transition any - [:created] => :created
-      end
-
-      event :make_errored do
-        transition any - [:errored] => :errored
-      end
-
-      before_transition any => [:errored, :created] do |cluster|
-        cluster.gcp_token = nil
-        cluster.gcp_operation_id = nil
-      end
-
-      before_transition any => [:errored] do |cluster, transition|
-        status_reason = transition.args.first
-        cluster.status_reason = status_reason if status_reason
-      end
-    end
-
-    def project_namespace_placeholder
-      "#{project.path}-#{project.id}"
-    end
-
-    def on_creation?
-      scheduled? || creating?
-    end
-
-    def api_url
-      'https://' + endpoint if endpoint
-    end
-
-    def restrict_modification
-      if on_creation?
-        errors.add(:base, "cannot modify during creation")
-        return false
-      end
-
-      true
-    end
-  end
-end
diff --git a/app/models/group.rb b/app/models/group.rb
index e746e4a12c9..8cf632fb566 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -6,6 +6,8 @@ class Group < Namespace
   include Avatarable
   include Referable
   include SelectForProjectAuthorization
+  include LoadedInGroupList
+  include GroupDescendant
 
   has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   alias_method :members, :group_members
@@ -24,6 +26,7 @@ class Group < Namespace
   has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   has_many :labels, class_name: 'GroupLabel'
   has_many :variables, class_name: 'Ci::GroupVariable'
+  has_many :custom_attributes, class_name: 'GroupCustomAttribute'
 
   validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
   validate :visibility_level_allowed_by_projects
@@ -40,6 +43,7 @@ class Group < Namespace
   after_create :post_create_hook
   after_destroy :post_destroy_hook
   after_save :update_two_factor_requirement
+  after_update :path_changed_hook, if: :path_changed?
 
   class << self
     def supports_nested_groups?
@@ -178,6 +182,12 @@ class Group < Namespace
     add_user(user, :owner, current_user: current_user)
   end
 
+  def member?(user, min_access_level = Gitlab::Access::GUEST)
+    return false unless user
+
+    max_member_access_for_user(user) >= min_access_level
+  end
+
   def has_owner?(user)
     return false unless user
 
@@ -287,6 +297,12 @@ class Group < Namespace
     list_of_ids.reverse.map { |group| variables[group.id] }.compact.flatten
   end
 
+  def full_path_was
+    return path_was unless has_parent?
+
+    "#{parent.full_path}/#{path_was}"
+  end
+
   private
 
   def update_two_factor_requirement
@@ -295,6 +311,10 @@ class Group < Namespace
     users.find_each(&:update_two_factor_requirement)
   end
 
+  def path_changed_hook
+    system_hook_service.execute_hooks_for(self, :rename)
+  end
+
   def visibility_level_allowed_by_parent
     return if visibility_level_allowed_by_parent?
 
diff --git a/app/models/group_custom_attribute.rb b/app/models/group_custom_attribute.rb
new file mode 100644
index 00000000000..8157d602d67
--- /dev/null
+++ b/app/models/group_custom_attribute.rb
@@ -0,0 +1,6 @@
+class GroupCustomAttribute < ActiveRecord::Base
+  belongs_to :group
+
+  validates :group, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:group_id] }
+end
diff --git a/app/models/identity.rb b/app/models/identity.rb
index 920a25932b4..ac8094b610e 100644
--- a/app/models/identity.rb
+++ b/app/models/identity.rb
@@ -7,7 +7,10 @@ class Identity < ActiveRecord::Base
   validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider }
   validates :user_id, uniqueness: { scope: :provider }
 
-  scope :with_extern_uid, ->(provider, extern_uid) { where(extern_uid: extern_uid, provider: provider) }
+  scope :with_extern_uid, ->(provider, extern_uid) do
+    extern_uid = Gitlab::LDAP::Person.normalize_dn(extern_uid) if provider.starts_with?('ldap')
+    where(extern_uid: extern_uid, provider: provider)
+  end
 
   def ldap?
     provider.starts_with?('ldap')
diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb
new file mode 100644
index 00000000000..b30b707e5fe
--- /dev/null
+++ b/app/models/instance_configuration.rb
@@ -0,0 +1,71 @@
+require 'resolv'
+
+class InstanceConfiguration
+  SSH_ALGORITHMS = %w(DSA ECDSA ED25519 RSA).freeze
+  SSH_ALGORITHMS_PATH = '/etc/ssh/'.freeze
+  CACHE_KEY = 'instance_configuration'.freeze
+  EXPIRATION_TIME = 24.hours
+
+  def settings
+    @configuration ||= Rails.cache.fetch(CACHE_KEY, expires_in: EXPIRATION_TIME) do
+      { ssh_algorithms_hashes: ssh_algorithms_hashes,
+        host: host,
+        gitlab_pages: gitlab_pages,
+        gitlab_ci: gitlab_ci }.deep_symbolize_keys
+    end
+  end
+
+  private
+
+  def ssh_algorithms_hashes
+    SSH_ALGORITHMS.map { |algo| ssh_algorithm_hashes(algo) }.compact
+  end
+
+  def host
+    Settings.gitlab.host
+  end
+
+  def gitlab_pages
+    Settings.pages.to_h.merge(ip_address: resolv_dns(Settings.pages.host))
+  end
+
+  def resolv_dns(dns)
+    Resolv.getaddress(dns)
+  rescue Resolv::ResolvError
+  end
+
+  def gitlab_ci
+    Settings.gitlab_ci
+            .to_h
+            .merge(artifacts_max_size: { value: Settings.artifacts.max_size&.megabytes,
+                                         default: 100.megabytes })
+  end
+
+  def ssh_algorithm_file(algorithm)
+    File.join(SSH_ALGORITHMS_PATH, "ssh_host_#{algorithm.downcase}_key.pub")
+  end
+
+  def ssh_algorithm_hashes(algorithm)
+    content = ssh_algorithm_file_content(algorithm)
+    return unless content.present?
+
+    { name: algorithm,
+      md5: ssh_algorithm_md5(content),
+      sha256: ssh_algorithm_sha256(content) }
+  end
+
+  def ssh_algorithm_file_content(algorithm)
+    file = ssh_algorithm_file(algorithm)
+    return unless File.exist?(file)
+
+    File.read(file)
+  end
+
+  def ssh_algorithm_md5(ssh_file_content)
+    OpenSSL::Digest::MD5.hexdigest(ssh_file_content).scan(/../).join(':')
+  end
+
+  def ssh_algorithm_sha256(ssh_file_content)
+    OpenSSL::Digest::SHA256.hexdigest(ssh_file_content)
+  end
+end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 155c5d972b7..3b3c7fb7f8b 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -5,11 +5,10 @@ class Issue < ActiveRecord::Base
   include Issuable
   include Noteable
   include Referable
-  include Sortable
   include Spammable
   include FasterCacheKeys
   include RelativePositioning
-  include CreatedAtFilterable
+  include TimeTrackable
 
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
@@ -74,19 +73,7 @@ class Issue < ActiveRecord::Base
     end
   end
 
-  def hook_attrs
-    assignee_ids = self.assignee_ids
-
-    attrs = {
-      total_time_spent: total_time_spent,
-      human_total_time_spent: human_total_time_spent,
-      human_time_estimate: human_time_estimate,
-      assignee_ids: assignee_ids,
-      assignee_id: assignee_ids.first # This key is deprecated
-    }
-
-    attributes.merge!(attrs)
-  end
+  acts_as_paranoid
 
   def self.reference_prefix
     '#'
@@ -131,6 +118,10 @@ class Issue < ActiveRecord::Base
               "id DESC")
   end
 
+  def hook_attrs
+    Gitlab::HookData::IssueBuilder.new(self).build
+  end
+
   # Returns a Hash of attributes to be used for Twitter card metadata
   def card_attributes
     {
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 292122f779e..82d0ae90d77 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -3,11 +3,11 @@ class MergeRequest < ActiveRecord::Base
   include Issuable
   include Noteable
   include Referable
-  include Sortable
   include IgnorableColumn
-  include CreatedAtFilterable
+  include TimeTrackable
 
-  ignore_column :locked_at
+  ignore_column :locked_at,
+                :ref_fetched
 
   belongs_to :target_project, class_name: "Project"
   belongs_to :source_project, class_name: "Project"
@@ -119,6 +119,8 @@ class MergeRequest < ActiveRecord::Base
 
   after_save :keep_around_commit
 
+  acts_as_paranoid
+
   def self.reference_prefix
     '!'
   end
@@ -179,6 +181,10 @@ class MergeRequest < ActiveRecord::Base
     work_in_progress?(title) ? title : "WIP: #{title}"
   end
 
+  def hook_attrs
+    Gitlab::HookData::MergeRequestBuilder.new(self).build
+  end
+
   # Returns a Hash of attributes to be used for Twitter card metadata
   def card_attributes
     {
@@ -392,7 +398,11 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def merge_ongoing?
-    !!merge_jid && !merged?
+    # While the MergeRequest is locked, it should present itself as 'merge ongoing'.
+    # The unlocking process is handled by StuckMergeJobsWorker scheduled in Cron.
+    return true if locked?
+
+    !!merge_jid && !merged? && Gitlab::SidekiqStatus.running?(merge_jid)
   end
 
   def closed_without_fork?
@@ -415,7 +425,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def create_merge_request_diff
-    fetch_ref
+    fetch_ref!
 
     # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37435
     Gitlab::GitalyClient.allow_n_plus_1_calls do
@@ -587,24 +597,6 @@ class MergeRequest < ActiveRecord::Base
     !discussions_to_be_resolved?
   end
 
-  def hook_attrs
-    attrs = {
-      source: source_project.try(:hook_attrs),
-      target: target_project.hook_attrs,
-      last_commit: nil,
-      work_in_progress: work_in_progress?,
-      total_time_spent: total_time_spent,
-      human_total_time_spent: human_total_time_spent,
-      human_time_estimate: human_time_estimate
-    }
-
-    if diff_head_commit
-      attrs[:last_commit] = diff_head_commit.hook_attrs
-    end
-
-    attributes.merge!(attrs)
-  end
-
   def for_fork?
     target_project != source_project
   end
@@ -689,13 +681,13 @@ class MergeRequest < ActiveRecord::Base
   def source_branch_exists?
     return false unless self.source_project
 
-    self.source_project.repository.branch_names.include?(self.source_branch)
+    self.source_project.repository.branch_exists?(self.source_branch)
   end
 
   def target_branch_exists?
     return false unless self.target_project
 
-    self.target_project.repository.branch_names.include?(self.target_branch)
+    self.target_project.repository.branch_exists?(self.target_branch)
   end
 
   def merge_commit_message(include_description: false)
@@ -818,29 +810,14 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
-  def fetch_ref
-    write_ref
-    update_column(:ref_fetched, true)
+  def fetch_ref!
+    target_project.repository.fetch_source_branch!(source_project.repository, source_branch, ref_path)
   end
 
   def ref_path
     "refs/#{Repository::REF_MERGE_REQUEST}/#{iid}/head"
   end
 
-  def ref_fetched?
-    super ||
-      begin
-        computed_value = project.repository.ref_exists?(ref_path)
-        update_column(:ref_fetched, true) if computed_value
-
-        computed_value
-      end
-  end
-
-  def ensure_ref_fetched
-    fetch_ref unless ref_fetched?
-  end
-
   def in_locked_state
     begin
       lock_mr
@@ -888,7 +865,7 @@ class MergeRequest < ActiveRecord::Base
   #
   def all_commit_shas
     if persisted?
-      column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).pluck('DISTINCT(sha)')
+      column_shas = MergeRequestDiffCommit.where(merge_request_diff: merge_request_diffs).limit(10_000).pluck('sha')
       serialised_shas = merge_request_diffs.where.not(st_commits: nil).flat_map(&:commit_shas)
 
       (column_shas + serialised_shas).uniq
@@ -982,10 +959,4 @@ class MergeRequest < ActiveRecord::Base
 
     project.merge_requests.merged.where(author_id: author_id).empty?
   end
-
-  private
-
-  def write_ref
-    target_project.repository.fetch_source_branch(source_project.repository, source_branch, ref_path)
-  end
 end
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index faf0b95f842..1eda0f9cbbd 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -48,6 +48,10 @@ class MergeRequestDiff < ActiveRecord::Base
   # Collect information about commits and diff from repository
   # and save it to the database as serialized data
   def save_git_content
+    MergeRequest
+      .where('id = ? AND COALESCE(latest_merge_request_diff_id, 0) < ?', self.merge_request_id, self.id)
+      .update_all(latest_merge_request_diff_id: self.id)
+
     ensure_commit_shas
     save_commits
     save_diffs
diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb
index 670b26d4ca3..b75387e236e 100644
--- a/app/models/merge_request_diff_commit.rb
+++ b/app/models/merge_request_diff_commit.rb
@@ -17,7 +17,9 @@ class MergeRequestDiffCommit < ActiveRecord::Base
       commit_hash.merge(
         merge_request_diff_id: merge_request_diff_id,
         relative_order: index,
-        sha: sha_attribute.type_cast_for_database(sha)
+        sha: sha_attribute.type_cast_for_database(sha),
+        authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]),
+        committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date])
       )
     end
 
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 4672881e220..4d401e7ba18 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -36,7 +36,7 @@ class Namespace < ActiveRecord::Base
   validates :path,
     presence: true,
     length: { maximum: 255 },
-    dynamic_path: true
+    namespace_path: true
 
   validate :nesting_level_allowed
 
@@ -162,6 +162,13 @@ class Namespace < ActiveRecord::Base
       .base_and_ancestors
   end
 
+  # returns all ancestors upto but excluding the the given namespace
+  # when no namespace is given, all ancestors upto the top are returned
+  def ancestors_upto(top = nil)
+    Gitlab::GroupHierarchy.new(self.class.where(id: id))
+      .ancestors(upto: top)
+  end
+
   def self_and_ancestors
     return self.class.where(id: id) unless parent_id
 
diff --git a/app/models/note.rb b/app/models/note.rb
index ceded9f2aef..f9676361072 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -69,7 +69,7 @@ class Note < ActiveRecord::Base
   delegate :title, to: :noteable, allow_nil: true
 
   validates :note, presence: true
-  validates :project, presence: true, unless: :for_personal_snippet?
+  validates :project, presence: true, if: :for_project_noteable?
 
   # Attachments are deprecated and are handled by Markdown uploader
   validates :attachment, file_size: { maximum: :max_attachment_size }
@@ -114,7 +114,7 @@ class Note < ActiveRecord::Base
   after_initialize :ensure_discussion_id
   before_validation :nullify_blank_type, :nullify_blank_line_code
   before_validation :set_discussion_id, on: :create
-  after_save :keep_around_commit, unless: :for_personal_snippet?
+  after_save :keep_around_commit, if: :for_project_noteable?
   after_save :expire_etag_cache
   after_destroy :expire_etag_cache
 
@@ -169,7 +169,7 @@ class Note < ActiveRecord::Base
   end
 
   def cross_reference?
-    system? && SystemNoteService.cross_reference?(note)
+    system? && matches_cross_reference_regex?
   end
 
   def diff_note?
@@ -208,6 +208,10 @@ class Note < ActiveRecord::Base
     noteable.is_a?(PersonalSnippet)
   end
 
+  def for_project_noteable?
+    !for_personal_snippet?
+  end
+
   def skip_project_check?
     for_personal_snippet?
   end
diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb
index b85f5dbaf2e..e8595b13d6d 100644
--- a/app/models/oauth_access_token.rb
+++ b/app/models/oauth_access_token.rb
@@ -1,4 +1,14 @@
 class OauthAccessToken < Doorkeeper::AccessToken
   belongs_to :resource_owner, class_name: 'User'
   belongs_to :application, class_name: 'Doorkeeper::Application'
+
+  alias_attribute :user, :resource_owner
+
+  def scopes=(value)
+    if value.is_a?(Array)
+      super(Doorkeeper::OAuth::Scopes.from_array(value).to_s)
+    else
+      super
+    end
+  end
 end
diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb
index 5d798247863..2e824cda525 100644
--- a/app/models/pages_domain.rb
+++ b/app/models/pages_domain.rb
@@ -16,9 +16,9 @@ class PagesDomain < ActiveRecord::Base
     key: Gitlab::Application.secrets.db_key_base,
     algorithm: 'aes-256-cbc'
 
-  after_create :update
-  after_save :update
-  after_destroy :update
+  after_create :update_daemon
+  after_save :update_daemon
+  after_destroy :update_daemon
 
   def to_param
     domain
@@ -80,7 +80,7 @@ class PagesDomain < ActiveRecord::Base
 
   private
 
-  def update
+  def update_daemon
     ::Projects::UpdatePagesConfigurationService.new(project).execute
   end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index 57e91ab3b88..53df29dab02 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -17,6 +17,7 @@ class Project < ActiveRecord::Base
   include ProjectFeaturesCompatibility
   include SelectForProjectAuthorization
   include Routable
+  include GroupDescendant
 
   extend Gitlab::ConfigHelper
   extend Gitlab::CurrentSettings
@@ -25,7 +26,15 @@ class Project < ActiveRecord::Base
 
   NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze
-  LATEST_STORAGE_VERSION = 1
+  # Hashed Storage versions handle rolling out new storage to project and dependents models:
+  # nil: legacy
+  # 1: repository
+  # 2: attachments
+  LATEST_STORAGE_VERSION = 2
+  HASHED_STORAGE_FEATURES = {
+    repository: 1,
+    attachments: 2
+  }.freeze
 
   cache_markdown_field :description, pipeline: :description
 
@@ -81,6 +90,8 @@ class Project < ActiveRecord::Base
   belongs_to :creator, class_name: 'User'
   belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id'
   belongs_to :namespace
+  alias_method :parent, :namespace
+  alias_attribute :parent_id, :namespace_id
 
   has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event'
   has_many :boards, before_add: :validate_board_limit
@@ -117,6 +128,7 @@ class Project < ActiveRecord::Base
   has_one :mock_deployment_service
   has_one :mock_monitoring_service
   has_one :microsoft_teams_service
+  has_one :packagist_service
 
   # TODO: replace these relations with the fork network versions
   has_one  :forked_project_link,  foreign_key: "forked_to_project_id"
@@ -174,7 +186,10 @@ class Project < ActiveRecord::Base
   has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true
   has_one :project_feature, inverse_of: :project
   has_one :statistics, class_name: 'ProjectStatistics'
-  has_one :cluster, class_name: 'Gcp::Cluster', inverse_of: :project
+
+  has_one :cluster_project, class_name: 'Clusters::Project'
+  has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster'
+  has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
 
   # Container repositories need to remove data from the container registry,
   # which is not managed by the DB. Hence we're still using dependent: :destroy
@@ -201,6 +216,7 @@ class Project < ActiveRecord::Base
   has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
 
   has_one :auto_devops, class_name: 'ProjectAutoDevops'
+  has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
@@ -228,10 +244,8 @@ class Project < ActiveRecord::Base
               message: Gitlab::Regex.project_name_regex_message }
   validates :path,
     presence: true,
-    dynamic_path: true,
+    project_path: true,
     length: { maximum: 255 },
-    format: { with: Gitlab::PathRegex.project_path_format_regex,
-              message: Gitlab::PathRegex.project_path_format_message },
     uniqueness: { scope: :namespace_id }
 
   validates :namespace, presence: true
@@ -479,6 +493,13 @@ class Project < ActiveRecord::Base
     end
   end
 
+  # returns all ancestor-groups upto but excluding the given namespace
+  # when no namespace is given, all ancestors upto the top are returned
+  def ancestors_upto(top = nil)
+    Gitlab::GroupHierarchy.new(Group.where(id: namespace_id))
+      .base_and_ancestors(upto: top)
+  end
+
   def lfs_enabled?
     return namespace.lfs_enabled? if self[:lfs_enabled].nil?
 
@@ -530,6 +551,10 @@ class Project < ActiveRecord::Base
     repository.commit(ref)
   end
 
+  def commit_by(oid:)
+    repository.commit_by(oid: oid)
+  end
+
   # ref can't be HEAD, can only be branch/tag name or SHA
   def latest_successful_builds_for(ref = default_branch)
     latest_pipeline = pipelines.latest_successful_for(ref)
@@ -543,7 +568,7 @@ class Project < ActiveRecord::Base
 
   def merge_base_commit(first_commit_id, second_commit_id)
     sha = repository.merge_base(first_commit_id, second_commit_id)
-    repository.commit(sha) if sha
+    commit_by(oid: sha) if sha
   end
 
   def saved?
@@ -1017,6 +1042,10 @@ class Project < ActiveRecord::Base
     !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?)
   end
 
+  def fork_source
+    forked_from_project || fork_network&.root_project
+  end
+
   def personal?
     !group
   end
@@ -1069,6 +1098,7 @@ class Project < ActiveRecord::Base
 
   def hook_attrs(backward: true)
     attrs = {
+      id: id,
       name: name,
       description: description,
       web_url: web_url,
@@ -1262,7 +1292,7 @@ class Project < ActiveRecord::Base
 
     # self.forked_from_project will be nil before the project is saved, so
     # we need to go through the relation
-    original_project = forked_project_link.forked_from_project
+    original_project = forked_project_link&.forked_from_project
     return true unless original_project
 
     level <= original_project.visibility_level
@@ -1380,6 +1410,19 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def after_rename_repo
+    path_before_change = previous_changes['path'].first
+
+    # We need to check if project had been rolled out to move resource to hashed storage or not and decide
+    # if we need execute any take action or no-op.
+
+    unless hashed_storage?(:attachments)
+      Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+    end
+
+    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+  end
+
   def rename_repo_notify!
     send_move_instructions(full_path_was)
     expires_full_path_cache
@@ -1390,13 +1433,6 @@ class Project < ActiveRecord::Base
     reload_repository!
   end
 
-  def after_rename_repo
-    path_before_change = previous_changes['path'].first
-
-    Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
-    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
-  end
-
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1458,7 +1494,8 @@ class Project < ActiveRecord::Base
       { key: 'CI_PROJECT_PATH', value: full_path, public: true },
       { key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug, public: true },
       { key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path, public: true },
-      { key: 'CI_PROJECT_URL', value: web_url, public: true }
+      { key: 'CI_PROJECT_URL', value: web_url, public: true },
+      { key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level), public: true }
     ]
   end
 
@@ -1549,10 +1586,6 @@ class Project < ActiveRecord::Base
     map.public_path_for_source_path(path)
   end
 
-  def parent
-    namespace
-  end
-
   def parent_changed?
     namespace_id_changed?
   end
@@ -1590,8 +1623,13 @@ class Project < ActiveRecord::Base
     [nil, 0].include?(self.storage_version)
   end
 
-  def hashed_storage?
-    self.storage_version && self.storage_version >= 1
+  # Check if Hashed Storage is enabled for the project with at least informed feature rolled out
+  #
+  # @param [Symbol] feature that needs to be rolled out for the project (:repository, :attachments)
+  def hashed_storage?(feature)
+    raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature)
+
+    self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature]
   end
 
   def renamed?
@@ -1627,7 +1665,7 @@ class Project < ActiveRecord::Base
   end
 
   def migrate_to_hashed_storage!
-    return if hashed_storage?
+    return if hashed_storage?(:repository)
 
     update!(repository_read_only: true)
 
@@ -1648,11 +1686,15 @@ class Project < ActiveRecord::Base
     Gitlab::GlRepository.gl_repository(self, is_wiki)
   end
 
+  def reference_counter(wiki: false)
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
+  end
+
   private
 
   def storage
     @storage ||=
-      if hashed_storage?
+      if hashed_storage?(:repository)
         Storage::HashedProject.new(self)
       else
         Storage::LegacyProject.new(self)
@@ -1666,11 +1708,11 @@ class Project < ActiveRecord::Base
   end
 
   def repo_reference_count
-    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+    reference_counter.value
   end
 
   def wiki_reference_count
-    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+    reference_counter(wiki: true).value
   end
 
   def check_repository_absence!
diff --git a/app/models/project_custom_attribute.rb b/app/models/project_custom_attribute.rb
new file mode 100644
index 00000000000..3f1a7b86a82
--- /dev/null
+++ b/app/models/project_custom_attribute.rb
@@ -0,0 +1,6 @@
+class ProjectCustomAttribute < ActiveRecord::Base
+  belongs_to :project
+
+  validates :project, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:project_id] }
+end
diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb
index 1327b075858..3273f41dbd2 100644
--- a/app/models/project_services/chat_message/issue_message.rb
+++ b/app/models/project_services/chat_message/issue_message.rb
@@ -39,7 +39,7 @@ module ChatMessage
     private
 
     def message
-      if state == 'opened'
+      if opened_issue?
         "[#{project_link}] Issue #{state} by #{user_combined_name}"
       else
         "[#{project_link}] Issue #{issue_link} #{state} by #{user_combined_name}"
diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb
index 9ee3a533c1e..b487378edd2 100644
--- a/app/models/project_services/jira_service.rb
+++ b/app/models/project_services/jira_service.rb
@@ -3,6 +3,8 @@ class JiraService < IssueTrackerService
 
   validates :url, url: true, presence: true, if: :activated?
   validates :api_url, url: true, allow_blank: true
+  validates :username, presence: true, if: :activated?
+  validates :password, presence: true, if: :activated?
 
   prop_accessor :username, :password, :url, :api_url, :jira_issue_transition_id, :title, :description
 
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index 8ba07173c74..5080acffb3c 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -136,6 +136,10 @@ class KubernetesService < DeploymentService
     { pods: read_pods }
   end
 
+  def kubeclient
+    @kubeclient ||= build_kubeclient!
+  end
+
   TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze
 
   private
@@ -153,7 +157,10 @@ class KubernetesService < DeploymentService
   end
 
   def default_namespace
-    "#{project.path}-#{project.id}" if project.present?
+    return unless project
+
+    slug = "#{project.path}-#{project.id}".downcase
+    slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
   end
 
   def build_kubeclient!(api_path: 'api', api_version: 'v1')
diff --git a/app/models/project_services/packagist_service.rb b/app/models/project_services/packagist_service.rb
new file mode 100644
index 00000000000..f68a0c1a3c3
--- /dev/null
+++ b/app/models/project_services/packagist_service.rb
@@ -0,0 +1,65 @@
+class PackagistService < Service
+  include HTTParty
+
+  prop_accessor :username, :token, :server
+
+  validates :username, presence: true, if: :activated?
+  validates :token, presence: true, if: :activated?
+
+  default_value_for :push_events, true
+  default_value_for :tag_push_events, true
+
+  after_save :compose_service_hook, if: :activated?
+
+  def title
+    'Packagist'
+  end
+
+  def description
+    'Update your project on Packagist, the main Composer repository'
+  end
+
+  def self.to_param
+    'packagist'
+  end
+
+  def fields
+    [
+      { type: 'text', name: 'username', placeholder: '', required: true },
+      { type: 'text', name: 'token', placeholder: '', required: true },
+      { type: 'text', name: 'server', placeholder: 'https://packagist.org', required: false }
+    ]
+  end
+
+  def self.supported_events
+    %w(push merge_request tag_push)
+  end
+
+  def execute(data)
+    return unless supported_events.include?(data[:object_kind])
+
+    service_hook.execute(data)
+  end
+
+  def test(data)
+    begin
+      result = execute(data)
+      return { success: false, result: result[:message] } if result[:http_status] != 202
+    rescue StandardError => error
+      return { success: false, result: error }
+    end
+
+    { success: true, result: result[:message] }
+  end
+
+  def compose_service_hook
+    hook = service_hook || build_service_hook
+    hook.url = hook_url
+    hook.save
+  end
+
+  def hook_url
+    base_url = server.present? ? server : 'https://packagist.org'
+    "#{base_url}/api/update-package?username=#{username}&apiToken=#{token}"
+  end
+end
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index bb7be29ef66..43de6809178 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -135,7 +135,7 @@ class ProjectWiki
   end
 
   def repository
-    @repository ||= Repository.new(full_path, @project, disk_path: disk_path)
+    @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true)
   end
 
   def default_branch
diff --git a/app/models/repository.rb b/app/models/repository.rb
index d725c65081d..eb7766d040c 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -15,9 +15,8 @@ class Repository
   ].freeze
 
   include Gitlab::ShellAdapter
-  include RepositoryMirroring
 
-  attr_accessor :full_path, :disk_path, :project
+  attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
 
@@ -34,7 +33,8 @@ class Repository
   CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide
                       changelog license_blob license_key gitignore koding_yml
                       gitlab_ci_yml branch_names tag_names branch_count
-                      tag_count avatar exists? empty? root_ref has_visible_content?).freeze
+                      tag_count avatar exists? empty? root_ref has_visible_content?
+                      issue_template_names merge_request_template_names).freeze
 
   # Methods that use cache_method but only memoize the value
   MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
@@ -50,7 +50,9 @@ class Repository
     gitignore: :gitignore,
     koding: :koding_yml,
     gitlab_ci: :gitlab_ci_yml,
-    avatar: :avatar
+    avatar: :avatar,
+    issue_template: :issue_template_names,
+    merge_request_template: :merge_request_template_names
   }.freeze
 
   # Wraps around the given method and caches its output in Redis and an instance
@@ -69,10 +71,12 @@ class Repository
     end
   end
 
-  def initialize(full_path, project, disk_path: nil)
+  def initialize(full_path, project, disk_path: nil, is_wiki: false)
     @full_path = full_path
     @disk_path = disk_path || full_path
     @project = project
+    @commit_cache = {}
+    @is_wiki = is_wiki
   end
 
   def ==(other)
@@ -100,18 +104,17 @@ class Repository
 
   def commit(ref = 'HEAD')
     return nil unless exists?
+    return ref if ref.is_a?(::Commit)
 
-    commit =
-      if ref.is_a?(Gitlab::Git::Commit)
-        ref
-      else
-        Gitlab::Git::Commit.find(raw_repository, ref)
-      end
+    find_commit(ref)
+  end
 
-    commit = ::Commit.new(commit, @project) if commit
-    commit
-  rescue Rugged::OdbError, Rugged::TreeError
-    nil
+  # Finding a commit by the passed SHA
+  # Also takes care of caching, based on the SHA
+  def commit_by(oid:)
+    return @commit_cache[oid] if @commit_cache.key?(oid)
+
+    @commit_cache[oid] = find_commit(oid)
   end
 
   def commits(ref, path: nil, limit: nil, offset: nil, skip_merges: false, after: nil, before: nil)
@@ -228,7 +231,7 @@ class Repository
   # branches or tags, but we want to keep some of these commits around, for
   # example if they have comments or CI builds.
   def keep_around(sha)
-    return unless sha && commit(sha)
+    return unless sha && commit_by(oid: sha)
 
     return if kept_around?(sha)
 
@@ -465,9 +468,7 @@ class Repository
   end
 
   def blob_at(sha, path)
-    unless Gitlab::Git.blank_ref?(sha)
-      Blob.decorate(Gitlab::Git::Blob.find(self, sha, path), project)
-    end
+    Blob.decorate(raw_repository.blob_at(sha, path), project)
   rescue Gitlab::Git::Repository::NoRepository
     nil
   end
@@ -535,6 +536,16 @@ class Repository
   end
   cache_method :avatar
 
+  def issue_template_names
+    Gitlab::Template::IssueTemplate.dropdown_names(project)
+  end
+  cache_method :issue_template_names, fallback: []
+
+  def merge_request_template_names
+    Gitlab::Template::MergeRequestTemplate.dropdown_names(project)
+  end
+  cache_method :merge_request_template_names, fallback: []
+
   def readme
     if readme = tree(:head)&.readme
       ReadmeBlob.new(readme, self)
@@ -851,22 +862,12 @@ class Repository
   end
 
   def ff_merge(user, source, target_branch, merge_request: nil)
-    our_commit = rugged.branches[target_branch].target
-    their_commit =
-      if source.is_a?(Gitlab::Git::Commit)
-        source.raw_commit
-      else
-        rugged.lookup(source)
-      end
-
-    raise 'Invalid merge target' if our_commit.nil?
-    raise 'Invalid merge source' if their_commit.nil?
+    their_commit_id = commit(source)&.id
+    raise 'Invalid merge source' if their_commit_id.nil?
 
-    with_branch(user, target_branch) do |start_commit|
-      merge_request&.update(in_progress_merge_commit_sha: their_commit.oid)
+    merge_request&.update(in_progress_merge_commit_sha: their_commit_id)
 
-      their_commit.oid
-    end
+    with_cache_hooks { raw.ff_merge(user, their_commit_id, target_branch) }
   end
 
   def revert(
@@ -901,26 +902,27 @@ class Repository
     end
   end
 
-  def resolve_conflicts(user, branch_name, params)
-    with_branch(user, branch_name) do
-      committer = user_to_committer(user)
+  def merged_to_root_ref?(branch_or_name, pre_loaded_merged_branches = nil)
+    branch = Gitlab::Git::Branch.find(self, branch_or_name)
 
-      create_commit(params.merge(author: committer, committer: committer))
-    end
-  end
-
-  def merged_to_root_ref?(branch_name)
-    branch_commit = commit(branch_name)
-    root_ref_commit = commit(root_ref)
+    if branch
+      @root_ref_sha ||= commit(root_ref).sha
+      same_head = branch.target == @root_ref_sha
+      merged =
+        if pre_loaded_merged_branches
+          pre_loaded_merged_branches.include?(branch.name)
+        else
+          ancestor?(branch.target, @root_ref_sha)
+        end
 
-    if branch_commit
-      same_head = branch_commit.id == root_ref_commit.id
-      !same_head && ancestor?(branch_commit.id, root_ref_commit.id)
+      !same_head && merged
     else
       nil
     end
   end
 
+  delegate :merged_branch_names, to: :raw_repository
+
   def merge_base(first_commit_id, second_commit_id)
     first_commit_id = commit(first_commit_id).try(:id) || first_commit_id
     second_commit_id = commit(second_commit_id).try(:id) || second_commit_id
@@ -963,25 +965,12 @@ class Repository
     run_git(args).first.lines.map(&:strip)
   end
 
-  def add_remote(name, url)
-    raw_repository.remote_add(name, url)
-  rescue Rugged::ConfigError
-    raw_repository.remote_update(name, url: url)
+  def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
+    gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
 
-  def remove_remote(name)
-    raw_repository.remote_delete(name)
-    true
-  rescue Rugged::ConfigError
-    false
-  end
-
-  def fetch_remote(remote, forced: false, no_tags: false)
-    gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags)
-  end
-
-  def fetch_source_branch(source_repository, source_branch, local_ref)
-    raw_repository.fetch_source_branch(source_repository.raw_repository, source_branch, local_ref)
+  def fetch_source_branch!(source_repository, source_branch, local_ref)
+    raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
   end
 
   def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:)
@@ -1028,6 +1017,10 @@ class Repository
     if instance_variable_defined?(ivar)
       instance_variable_get(ivar)
     else
+      # If the repository doesn't exist and a fallback was specified we return
+      # that value inmediately. This saves us Rugged/gRPC invocations.
+      return fallback unless fallback.nil? || exists?
+
       begin
         value =
           if memoize_only
@@ -1037,8 +1030,9 @@ class Repository
           end
         instance_variable_set(ivar, value)
       rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository
-        # if e.g. HEAD or the entire repository doesn't exist we want to
-        # gracefully handle this and not cache anything.
+        # Even if the above `#exists?` check passes these errors might still
+        # occur (for example because of a non-existing HEAD). We want to
+        # gracefully handle this and not cache anything
         fallback
       end
     end
@@ -1066,6 +1060,18 @@ class Repository
 
   private
 
+  # TODO Generice finder, later split this on finders by Ref or Oid
+  # gitlab-org/gitlab-ce#39239
+  def find_commit(oid_or_ref)
+    commit = if oid_or_ref.is_a?(Gitlab::Git::Commit)
+               oid_or_ref
+             else
+               Gitlab::Git::Commit.find(raw_repository, oid_or_ref)
+             end
+
+    ::Commit.new(commit, @project) if commit
+  end
+
   def blob_data_at(sha, path)
     blob = blob_at(sha, path)
     return unless blob
@@ -1104,17 +1110,17 @@ class Repository
 
   def last_commit_for_path_by_gitaly(sha, path)
     c = raw_repository.gitaly_commit_client.last_commit_for_path(sha, path)
-    commit(c)
+    commit_by(oid: c)
   end
 
   def last_commit_for_path_by_rugged(sha, path)
     sha = last_commit_id_for_path_by_shelling_out(sha, path)
-    commit(sha)
+    commit_by(oid: sha)
   end
 
   def last_commit_id_for_path_by_shelling_out(sha, path)
     args = %W(rev-list --max-count=1 #{sha} -- #{path})
-    run_git(args).first.strip
+    raw_repository.run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
   end
 
   def repository_storage_path
@@ -1122,7 +1128,7 @@ class Repository
   end
 
   def initialize_raw_repository
-    Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false))
+    Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
   end
 
   def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
diff --git a/app/models/service.rb b/app/models/service.rb
index 6b64079215f..fdd2605e3e3 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -238,6 +238,7 @@ class Service < ActiveRecord::Base
       kubernetes
       mattermost_slash_commands
       mattermost
+      packagist
       pipelines_email
       pivotaltracker
       prometheus
diff --git a/app/models/user.rb b/app/models/user.rb
index 533a776bc65..aa88cda4dc0 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -21,8 +21,8 @@ class User < ActiveRecord::Base
 
   ignore_column :external_email
   ignore_column :email_provider
+  ignore_column :authentication_token
 
-  add_authentication_token_field :authentication_token
   add_authentication_token_field :incoming_email_token
   add_authentication_token_field :rss_token
 
@@ -146,7 +146,7 @@ class User < ActiveRecord::Base
     presence: true,
     numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: Gitlab::Database::MAX_INT_VALUE }
   validates :username,
-    dynamic_path: true,
+    user_path: true,
     presence: true,
     uniqueness: { case_sensitive: false }
 
@@ -163,11 +163,12 @@ class User < ActiveRecord::Base
   before_validation :sanitize_attrs
   before_validation :set_notification_email, if: :email_changed?
   before_validation :set_public_email, if: :public_email_changed?
-  before_save :ensure_authentication_token, :ensure_incoming_email_token
-  before_save :ensure_user_rights_and_limits, if: :external_changed?
+  before_save :ensure_incoming_email_token
+  before_save :ensure_user_rights_and_limits, if: ->(user) { user.new_record? || user.external_changed? }
   before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) }
   before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? }
   after_save :ensure_namespace_correct
+  after_update :username_changed_hook, if: :username_changed?
   after_destroy :post_destroy_hook
   after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') }
   after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') }
@@ -182,15 +183,8 @@ class User < ActiveRecord::Base
   enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos]
 
   # User's Project preference
-  #
-  # Note: When adding an option, it MUST go on the end of the hash with a
-  # number higher than the current max. We cannot move options and/or change
-  # their numbers.
-  #
-  # We skip 0 because this was used by an option that has since been removed.
-  enum project_view: { activity: 1, files: 2 }
-
-  alias_attribute :private_token, :authentication_token
+  # Note: When adding an option, it MUST go on the end of the array.
+  enum project_view: [:readme, :activity, :files]
 
   delegate :path, to: :namespace, allow_nil: true, prefix: true
 
@@ -878,6 +872,10 @@ class User < ActiveRecord::Base
     end
   end
 
+  def username_changed_hook
+    system_hook_service.execute_hooks_for(self, :rename)
+  end
+
   def post_destroy_hook
     log_info("User \"#{name}\" (#{email})  was removed")
     system_hook_service.execute_hooks_for(self, :destroy)
@@ -1141,8 +1139,9 @@ class User < ActiveRecord::Base
       self.can_create_group = false
       self.projects_limit   = 0
     else
-      self.can_create_group = gitlab_config.default_can_create_group
-      self.projects_limit = current_application_settings.default_projects_limit
+      # Only revert these back to the default if they weren't specifically changed in this update.
+      self.can_create_group = gitlab_config.default_can_create_group unless can_create_group_changed?
+      self.projects_limit = current_application_settings.default_projects_limit unless projects_limit_changed?
     end
   end