Merge branch 'master' into feature/sm/35954-create-kubernetes-cluster-on-gke-from-k8s-service

13 files changed, 115 insertions, 69 deletions

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index acaa028eaa2..3d5acc00f8f 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -434,7 +434,7 @@ module Ci
     def update_duration
       return unless started_at
 
-      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
+      self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
     end
 
     def execute_hooks
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index a0d07902ba2..c6509f89117 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -174,7 +174,7 @@ module Ci
     end
 
     def assignable_for?(project)
-      !locked? || projects.exists?(id: project.id)
+      is_shared? || projects.exists?(id: project.id)
     end
 
     def accepting_tags?(build)
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 2ae8890c1b3..6dba154a6ea 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -25,8 +25,8 @@ class Commit
   DIFF_HARD_LIMIT_FILES = 1000
   DIFF_HARD_LIMIT_LINES = 50000
 
-  # The SHA can be between 7 and 40 hex characters.
-  COMMIT_SHA_PATTERN = '\h{7,40}'.freeze
+  MIN_SHA_LENGTH = 7
+  COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze
 
   def banzai_render_context(field)
     context = { pipeline: :single_line, project: self.project }
@@ -53,7 +53,7 @@ class Commit
 
     # Truncate sha to 8 characters
     def truncate_sha(sha)
-      sha[0..7]
+      sha[0..MIN_SHA_LENGTH]
     end
 
     def max_diff_options
@@ -100,7 +100,7 @@ class Commit
   def self.reference_pattern
     @reference_pattern ||= %r{
       (?:#{Project.reference_pattern}#{reference_prefix})?
-      (?<commit>\h{7,40})
+      (?<commit>#{COMMIT_SHA_PATTERN})
     }x
   end
 
@@ -216,9 +216,8 @@ class Commit
     @raw.respond_to?(method, include_private) || super
   end
 
-  # Truncate sha to 8 characters
   def short_id
-    @raw.short_id(7)
+    @raw.short_id(MIN_SHA_LENGTH)
   end
 
   def diff_refs
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 265f6e48540..fc30d008dea 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -143,16 +143,18 @@ module Issuable
     end
 
     def sort(method, excluded_labels: [])
-      sorted = case method.to_s
-               when 'milestone_due_asc' then order_milestone_due_asc
-               when 'milestone_due_desc' then order_milestone_due_desc
-               when 'downvotes_desc' then order_downvotes_desc
-               when 'upvotes_desc' then order_upvotes_desc
-               when 'label_priority' then order_labels_priority(excluded_labels: excluded_labels)
-               when 'priority' then order_due_date_and_labels_priority(excluded_labels: excluded_labels)
-               else
-                 order_by(method)
-               end
+      sorted =
+        case method.to_s
+        when 'downvotes_desc'     then order_downvotes_desc
+        when 'label_priority'     then order_labels_priority(excluded_labels: excluded_labels)
+        when 'milestone'          then order_milestone_due_asc
+        when 'milestone_due_asc'  then order_milestone_due_asc
+        when 'milestone_due_desc' then order_milestone_due_desc
+        when 'popularity'         then order_upvotes_desc
+        when 'priority'           then order_due_date_and_labels_priority(excluded_labels: excluded_labels)
+        when 'upvotes_desc'       then order_upvotes_desc
+        else order_by(method)
+        end
 
       # Break ties with the ID column for pagination
       sorted.order(id: :desc)
@@ -214,7 +216,7 @@ module Issuable
     def grouping_columns(sort)
       grouping_columns = [arel_table[:id]]
 
-      if %w(milestone_due_desc milestone_due_asc).include?(sort)
+      if %w(milestone_due_desc milestone_due_asc milestone).include?(sort)
         milestone_table = Milestone.arel_table
         grouping_columns << milestone_table[:id]
         grouping_columns << milestone_table[:due_date]
diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb
index db3cd257584..cefa5c13c5f 100644
--- a/app/models/concerns/sortable.rb
+++ b/app/models/concerns/sortable.rb
@@ -19,14 +19,15 @@ module Sortable
   module ClassMethods
     def order_by(method)
       case method.to_s
-      when 'name_asc' then order_name_asc
-      when 'name_desc' then order_name_desc
-      when 'updated_asc' then order_updated_asc
-      when 'updated_desc' then order_updated_desc
-      when 'created_asc' then order_created_asc
+      when 'created_asc'  then order_created_asc
+      when 'created_date' then order_created_desc
       when 'created_desc' then order_created_desc
-      when 'id_desc' then order_id_desc
-      when 'id_asc' then order_id_asc
+      when 'id_asc'       then order_id_asc
+      when 'id_desc'      then order_id_desc
+      when 'name_asc'     then order_name_asc
+      when 'name_desc'    then order_name_desc
+      when 'updated_asc'  then order_updated_asc
+      when 'updated_desc' then order_updated_desc
       else
         all
       end
diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb
index 44deae4234b..54bd5b68777 100644
--- a/app/models/gpg_key.rb
+++ b/app/models/gpg_key.rb
@@ -73,7 +73,7 @@ class GpgKey < ActiveRecord::Base
   end
 
   def verified_and_belongs_to_email?(email)
-    emails_with_verified_status.fetch(email, false)
+    emails_with_verified_status.fetch(email.downcase, false)
   end
 
   def update_invalid_gpg_signatures
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 92a454300af..155c5d972b7 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -116,7 +116,8 @@ class Issue < ActiveRecord::Base
 
   def self.sort(method, excluded_labels: [])
     case method.to_s
-    when 'due_date_asc' then order_due_date_asc
+    when 'due_date'      then order_due_date_asc
+    when 'due_date_asc'  then order_due_date_asc
     when 'due_date_desc' then order_due_date_desc
     else
       super
diff --git a/app/models/project.rb b/app/models/project.rb
index 5d6a8bdbaeb..fd171cc7941 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -72,6 +72,7 @@ class Project < ActiveRecord::Base
   attr_accessor :old_path_with_namespace
   attr_accessor :template_name
   attr_writer :pipeline_status
+  attr_accessor :skip_disk_validation
 
   alias_attribute :title, :name
 
@@ -228,7 +229,7 @@ class Project < ActiveRecord::Base
   validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?]
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
-  validate :can_create_repository?, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? }
+  validate :check_repository_path_availability, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? }
   validate :avatar_type,
     if: ->(project) { project.avatar.present? && project.avatar_changed? }
   validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
@@ -246,6 +247,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
+  scope :with_hashed_storage, -> { where('storage_version >= 1') }
+  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
 
@@ -1016,7 +1020,8 @@ class Project < ActiveRecord::Base
   end
 
   # Check if repository already exists on disk
-  def can_create_repository?
+  def check_repository_path_availability
+    return true if skip_disk_validation
     return false unless repository_storage_path
 
     expires_full_path_cache # we need to clear cache to validate renames correctly
@@ -1033,7 +1038,7 @@ class Project < ActiveRecord::Base
     # Forked import is handled asynchronously
     return if forked? && !force
 
-    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+    if gitlab_shell.add_repository(repository_storage, disk_path)
       repository.after_create
       true
     else
@@ -1551,18 +1556,44 @@ class Project < ActiveRecord::Base
   end
 
   def legacy_storage?
-    self.storage_version.nil?
+    [nil, 0].include?(self.storage_version)
+  end
+
+  def hashed_storage?
+    self.storage_version && self.storage_version >= 1
   end
 
   def renamed?
     persisted? && path_changed?
   end
 
+  def migrate_to_hashed_storage!
+    return if hashed_storage?
+
+    update!(repository_read_only: true)
+
+    if repo_reference_count > 0 || wiki_reference_count > 0
+      ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
+    else
+      ProjectMigrateHashedStorageWorker.perform_async(id)
+    end
+  end
+
+  def storage_version=(value)
+    super
+
+    @storage = nil if storage_version_changed?
+  end
+
+  def gl_repository(is_wiki:)
+    Gitlab::GlRepository.gl_repository(self, is_wiki)
+  end
+
   private
 
   def storage
     @storage ||=
-      if self.storage_version && self.storage_version >= 1
+      if hashed_storage?
         Storage::HashedProject.new(self)
       else
         Storage::LegacyProject.new(self)
@@ -1575,6 +1606,14 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def repo_reference_count
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+  end
+
+  def wiki_reference_count
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+  end
+
   # set last_activity_at to the same as created_at
   def set_last_activity_at
     update_column(:last_activity_at, self.created_at)
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 698fdf7a20c..c4cc1c1cf22 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -174,7 +174,7 @@ class ProjectWiki
   private
 
   def init_repo(disk_path)
-    gitlab_shell.add_repository(project.repository_storage_path, disk_path)
+    gitlab_shell.add_repository(project.repository_storage, disk_path)
   end
 
   def commit_details(action, message = nil, title = nil)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 90cede9d3d4..a0f57f1e54d 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -34,7 +34,10 @@ class Repository
   CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide
                       changelog license_blob license_key gitignore koding_yml
                       gitlab_ci_yml branch_names tag_names branch_count
-                      tag_count avatar exists? empty? root_ref).freeze
+                      tag_count avatar exists? empty? root_ref has_visible_content?).freeze
+
+  # Methods that use cache_method but only memoize the value
+  MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
 
   # Certain method caches should be refreshed when certain types of files are
   # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
@@ -91,12 +94,6 @@ class Repository
     )
   end
 
-  # we need to have this method here because it is not cached in ::Git and
-  # the method is called multiple times for every request
-  def has_visible_content?
-    branch_count > 0
-  end
-
   def inspect
     "#<#{self.class.name}:#{@disk_path}>"
   end
@@ -275,7 +272,7 @@ class Repository
   end
 
   def expire_branches_cache
-    expire_method_caches(%i(branch_names branch_count))
+    expire_method_caches(%i(branch_names branch_count has_visible_content?))
     @local_branches = nil
     @branch_exists_memo = nil
   end
@@ -346,7 +343,7 @@ class Repository
   def expire_emptiness_caches
     return unless empty?
 
-    expire_method_caches(%i(empty?))
+    expire_method_caches(%i(empty? has_visible_content?))
   end
 
   def lookup_cache
@@ -489,13 +486,7 @@ class Repository
   def exists?
     return false unless full_path
 
-    Gitlab::GitalyClient.migrate(:repository_exists) do |enabled|
-      if enabled
-        raw_repository.exists?
-      else
-        refs_directory_exists?
-      end
-    end
+    raw_repository.exists?
   end
   cache_method :exists?
 
@@ -529,13 +520,17 @@ class Repository
   delegate :tag_names, to: :raw_repository
   cache_method :tag_names, fallback: []
 
-  delegate :branch_count, :tag_count, to: :raw_repository
+  delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository
   cache_method :branch_count, fallback: 0
   cache_method :tag_count, fallback: 0
+  cache_method :has_visible_content?, fallback: false
 
   def avatar
-    if tree = file_on_head(:avatar)
-      tree.path
+    # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38327
+    Gitlab::GitalyClient.allow_n_plus_1_calls do
+      if tree = file_on_head(:avatar)
+        tree.path
+      end
     end
   end
   cache_method :avatar
@@ -1060,12 +1055,6 @@ class Repository
     blob.data
   end
 
-  def refs_directory_exists?
-    circuit_breaker.perform do
-      File.exist?(File.join(path_to_repo, 'refs'))
-    end
-  end
-
   def cache
     # TODO: should we use UUIDs here? We could move repositories without clearing this cache
     @cache ||= RepositoryCache.new(full_path, @project.id)
@@ -1117,10 +1106,6 @@ class Repository
     Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false))
   end
 
-  def circuit_breaker
-    @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(project.repository_storage)
-  end
-
   def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
     ref ||= root_ref
 
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index fae1b64961a..f025f40994e 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,6 +4,7 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
+    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/models/user.rb b/app/models/user.rb
index 09c9b3250eb..4e71a3e11c2 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -60,7 +60,7 @@ class User < ActiveRecord::Base
     lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i)
     return unless lease.try_obtain
 
-    Users::UpdateService.new(self).execute(validate: false)
+    Users::UpdateService.new(self, user: self).execute(validate: false)
   end
 
   attr_accessor :force_random_password
@@ -130,6 +130,8 @@ class User < ActiveRecord::Base
   has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue
   has_many :assigned_merge_requests,  dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :custom_attributes, class_name: 'UserCustomAttribute'
+
   #
   # Validations
   #
@@ -526,8 +528,8 @@ class User < ActiveRecord::Base
   def update_emails_with_primary_email
     primary_email_record = emails.find_by(email: email)
     if primary_email_record
-      Emails::DestroyService.new(self, email: email).execute
-      Emails::CreateService.new(self, email: email_was).execute
+      Emails::DestroyService.new(self, user: self, email: email).execute
+      Emails::CreateService.new(self, user: self, email: email_was).execute
     end
   end
 
@@ -690,7 +692,11 @@ class User < ActiveRecord::Base
   end
 
   def ldap_user?
-    identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"])
+    if identities.loaded?
+      identities.find { |identity| identity.provider.start_with?('ldap') && !identity.extern_uid.nil? }
+    else
+      identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"])
+    end
   end
 
   def ldap_identity
@@ -1000,7 +1006,7 @@ class User < ActiveRecord::Base
     if attempts_exceeded?
       lock_access! unless access_locked?
     else
-      Users::UpdateService.new(self).execute(validate: false)
+      Users::UpdateService.new(self, user: self).execute(validate: false)
     end
   end
 
@@ -1061,6 +1067,12 @@ class User < ActiveRecord::Base
     user_synced_attributes_metadata&.read_only?(attribute)
   end
 
+  # override, from Devise
+  def lock_access!
+    Gitlab::AppLogger.info("Account Locked: username=#{username}")
+    super
+  end
+
   protected
 
   # override, from Devise::Validatable
@@ -1186,7 +1198,7 @@ class User < ActiveRecord::Base
       &creation_block
     )
 
-    Users::UpdateService.new(user).execute(validate: false)
+    Users::UpdateService.new(user, user: user).execute(validate: false)
     user
   ensure
     Gitlab::ExclusiveLease.cancel(lease_key, uuid)
diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb
new file mode 100644
index 00000000000..eff25b31f9b
--- /dev/null
+++ b/app/models/user_custom_attribute.rb
@@ -0,0 +1,6 @@
+class UserCustomAttribute < ActiveRecord::Base
+  belongs_to :user
+
+  validates :user_id, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:user_id] }
+end