18 files changed, 111 insertions, 114 deletions

diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb
index 2340453831e..0d7c2d20029 100644
--- a/app/models/abuse_report.rb
+++ b/app/models/abuse_report.rb
@@ -16,7 +16,7 @@ class AbuseReport < ActiveRecord::Base
 
   def remove_user(deleted_by:)
     user.block
-    DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true)
+    DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true, hard_delete: true)
   end
 
   def notify
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 2961e16f5e0..dd1a6922968 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -238,7 +238,8 @@ class ApplicationSetting < ActiveRecord::Base
       terminal_max_session_time: 0,
       two_factor_grace_period: 48,
       user_default_external: false,
-      polling_interval_multiplier: 1
+      polling_interval_multiplier: 1,
+      usage_ping_enabled: true
     }
   end
 
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
index 8ea95beed79..2eedc143968 100644
--- a/app/models/concerns/cache_markdown_field.rb
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -8,6 +8,14 @@
 #
 # Corresponding foo_html, bar_html and baz_html fields should exist.
 module CacheMarkdownField
+  extend ActiveSupport::Concern
+
+  # Increment this number every time the renderer changes its output
+  CACHE_VERSION = 1
+
+  # changes to these attributes cause the cache to be invalidates
+  INVALIDATED_BY = %w[author project].freeze
+
   # Knows about the relationship between markdown and html field names, and
   # stores the rendering contexts for the latter
   class FieldData
@@ -30,60 +38,71 @@ module CacheMarkdownField
     end
   end
 
-  # Dynamic registries don't really work in Rails as it's not guaranteed that
-  # every class will be loaded, so hardcode the list.
-  CACHING_CLASSES = %w[
-    AbuseReport
-    Appearance
-    ApplicationSetting
-    BroadcastMessage
-    Issue
-    Label
-    MergeRequest
-    Milestone
-    Namespace
-    Note
-    Project
-    Release
-    Snippet
-  ].freeze
-
-  def self.caching_classes
-    CACHING_CLASSES.map(&:constantize)
-  end
-
   def skip_project_check?
     false
   end
 
-  extend ActiveSupport::Concern
+  # Returns the default Banzai render context for the cached markdown field.
+  def banzai_render_context(field)
+    raise ArgumentError.new("Unknown field: #{field.inspect}") unless
+      cached_markdown_fields.markdown_fields.include?(field)
 
-  included do
-    cattr_reader :cached_markdown_fields do
-      FieldData.new
-    end
+    # Always include a project key, or Banzai complains
+    project = self.project if self.respond_to?(:project)
+    context = cached_markdown_fields[field].merge(project: project)
 
-    # Returns the default Banzai render context for the cached markdown field.
-    def banzai_render_context(field)
-      raise ArgumentError.new("Unknown field: #{field.inspect}") unless
-        cached_markdown_fields.markdown_fields.include?(field)
+    # Banzai is less strict about authors, so don't always have an author key
+    context[:author] = self.author if self.respond_to?(:author)
 
-      # Always include a project key, or Banzai complains
-      project = self.project if self.respond_to?(:project)
-      context = cached_markdown_fields[field].merge(project: project)
+    context
+  end
 
-      # Banzai is less strict about authors, so don't always have an author key
-      context[:author] = self.author if self.respond_to?(:author)
+  # Update every column in a row if any one is invalidated, as we only store
+  # one version per row
+  def refresh_markdown_cache!(do_update: false)
+    options = { skip_project_check: skip_project_check? }
 
-      context
-    end
+    updates = cached_markdown_fields.markdown_fields.map do |markdown_field|
+      [
+        cached_markdown_fields.html_field(markdown_field),
+        Banzai::Renderer.cacheless_render_field(self, markdown_field, options)
+      ]
+    end.to_h
+    updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION
 
-    # Allow callers to look up the cache field name, rather than hardcoding it
-    def markdown_cache_field_for(field)
-      raise ArgumentError.new("Unknown field: #{field}") unless
-        cached_markdown_fields.markdown_fields.include?(field)
+    updates.each {|html_field, data| write_attribute(html_field, data) }
 
-      cached_markdown_fields.html_field(field)
+    update_columns(updates) if persisted? && do_update
+  end
+
+  def cached_html_up_to_date?(markdown_field)
+    html_field = cached_markdown_fields.html_field(markdown_field)
+
+    markdown_changed = attribute_changed?(markdown_field) || false
+    html_changed = attribute_changed?(html_field) || false
+
+    CacheMarkdownField::CACHE_VERSION == cached_markdown_version &&
+      (html_changed || markdown_changed == html_changed)
+  end
+
+  def invalidated_markdown_cache?
+    cached_markdown_fields.html_fields.any? {|html_field| attribute_invalidated?(html_field) }
+  end
+
+  def attribute_invalidated?(attr)
+    __send__("#{attr}_invalidated?")
+  end
+
+  def cached_html_for(markdown_field)
+    raise ArgumentError.new("Unknown field: #{field}") unless
+      cached_markdown_fields.markdown_fields.include?(markdown_field)
+
+    __send__(cached_markdown_fields.html_field(markdown_field))
+  end
+
+  included do
+    cattr_reader :cached_markdown_fields do
+      FieldData.new
     end
 
     # Always exclude _html fields from attributes (including serialization).
@@ -92,12 +111,16 @@ module CacheMarkdownField
     def attributes
       attrs = attributes_before_markdown_cache
 
+      attrs.delete('cached_markdown_version')
+
       cached_markdown_fields.html_fields.each do |field|
         attrs.delete(field)
       end
 
       attrs
     end
+
+    before_save :refresh_markdown_cache!, if: :invalidated_markdown_cache?
   end
 
   class_methods do
@@ -107,31 +130,18 @@ module CacheMarkdownField
     # a corresponding _html field. Any custom rendering options may be provided
     # as a context.
     def cache_markdown_field(markdown_field, context = {})
-      raise "Add #{self} to CacheMarkdownField::CACHING_CLASSES" unless
-        CacheMarkdownField::CACHING_CLASSES.include?(self.to_s)
-
       cached_markdown_fields[markdown_field] = context
 
       html_field = cached_markdown_fields.html_field(markdown_field)
-      cache_method = "#{markdown_field}_cache_refresh".to_sym
       invalidation_method = "#{html_field}_invalidated?".to_sym
 
-      define_method(cache_method) do
-        options = { skip_project_check: skip_project_check? }
-        html = Banzai::Renderer.cacheless_render_field(self, markdown_field, options)
-        __send__("#{html_field}=", html)
-        true
-      end
-
       # The HTML becomes invalid if any dependent fields change. For now, assume
       # author and project invalidate the cache in all circumstances.
       define_method(invalidation_method) do
         changed_fields = changed_attributes.keys
-        invalidations = changed_fields & [markdown_field.to_s, "author", "project"]
-        !invalidations.empty?
+        invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY]
+        !invalidations.empty? || !cached_html_up_to_date?(markdown_field)
       end
-
-      before_save cache_method, if: invalidation_method
     end
   end
 end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 3d2258d5e3e..26dbf4d9570 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -23,7 +23,7 @@ module Issuable
 
   included do
     cache_markdown_field :title, pipeline: :single_line
-    cache_markdown_field :description
+    cache_markdown_field :description, issuable_state_filter_enabled: true
 
     belongs_to :author, class_name: "User"
     belongs_to :assignee, class_name: "User"
diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb
index 82f4182d59a..d0c94d3b694 100644
--- a/app/models/container_repository.rb
+++ b/app/models/container_repository.rb
@@ -20,7 +20,8 @@ class ContainerRepository < ActiveRecord::Base
   end
 
   def path
-    @path ||= [project.full_path, name].select(&:present?).join('/')
+    @path ||= [project.full_path, name]
+      .select(&:present?).join('/').downcase
   end
 
   def location
diff --git a/app/models/group.rb b/app/models/group.rb
index 106084175ff..cbc10b00cf5 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -125,7 +125,7 @@ class Group < Namespace
   end
 
   def add_users(users, access_level, current_user: nil, expires_at: nil)
-    GroupMember.add_users_to_group(
+    GroupMember.add_users(
       self,
       users,
       access_level,
diff --git a/app/models/identity.rb b/app/models/identity.rb
index 3bacc450e6e..920a25932b4 100644
--- a/app/models/identity.rb
+++ b/app/models/identity.rb
@@ -7,6 +7,8 @@ class Identity < ActiveRecord::Base
   validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider }
   validates :user_id, uniqueness: { scope: :provider }
 
+  scope :with_extern_uid, ->(provider, extern_uid) { where(extern_uid: extern_uid, provider: provider) }
+
   def ldap?
     provider.starts_with?('ldap')
   end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index d39ae3a6c92..305fc01f041 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -199,7 +199,7 @@ class Issue < ActiveRecord::Base
   # Returns `true` if the current issue can be viewed by either a logged in User
   # or an anonymous user.
   def visible_to_user?(user = nil)
-    return false unless project.feature_available?(:issues, user)
+    return false unless project && project.feature_available?(:issues, user)
 
     user ? readable_by?(user) : publicly_visible?
   end
diff --git a/app/models/member.rb b/app/models/member.rb
index 0545bd4eedf..97fba501759 100644
--- a/app/models/member.rb
+++ b/app/models/member.rb
@@ -151,6 +151,22 @@ class Member < ActiveRecord::Base
       member
     end
 
+    def add_users(source, users, access_level, current_user: nil, expires_at: nil)
+      return [] unless users.present?
+
+      self.transaction do
+        users.map do |user|
+          add_user(
+            source,
+            user,
+            access_level,
+            current_user: current_user,
+            expires_at: expires_at
+          )
+        end
+      end
+    end
+
     def access_levels
       Gitlab::Access.sym_options
     end
@@ -173,18 +189,6 @@ class Member < ActiveRecord::Base
       # There is no current user for bulk actions, in which case anything is allowed
       !current_user || current_user.can?(:"update_#{member.type.underscore}", member)
     end
-
-    def add_users_to_source(source, users, access_level, current_user: nil, expires_at: nil)
-      users.each do |user|
-        add_user(
-          source,
-          user,
-          access_level,
-          current_user: current_user,
-          expires_at: expires_at
-        )
-      end
-    end
   end
 
   def real_source_type
diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb
index 483425cd30f..28e10bc6172 100644
--- a/app/models/members/group_member.rb
+++ b/app/models/members/group_member.rb
@@ -21,18 +21,6 @@ class GroupMember < Member
     Gitlab::Access.sym_options_with_owner
   end
 
-  def self.add_users_to_group(group, users, access_level, current_user: nil, expires_at: nil)
-    self.transaction do
-      add_users_to_source(
-        group,
-        users,
-        access_level,
-        current_user: current_user,
-        expires_at: expires_at
-      )
-    end
-  end
-
   def group
     source
   end
diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb
index 912820b51ac..b3a91feb091 100644
--- a/app/models/members/project_member.rb
+++ b/app/models/members/project_member.rb
@@ -16,7 +16,7 @@ class ProjectMember < Member
   before_destroy :delete_member_todos
 
   class << self
-    # Add users to project teams with passed access option
+    # Add users to projects with passed access option
     #
     # access can be an integer representing a access code
     # or symbol like :master representing role
@@ -39,7 +39,7 @@ class ProjectMember < Member
         project_ids.each do |project_id|
           project = Project.find(project_id)
 
-          add_users_to_source(
+          add_users(
             project,
             users,
             access_level,
diff --git a/app/models/note.rb b/app/models/note.rb
index 630d0adbece..e720bfba030 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -16,7 +16,7 @@ class Note < ActiveRecord::Base
 
   ignore_column :original_discussion_id
 
-  cache_markdown_field :note, pipeline: :note
+  cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true
 
   # Attribute containing rendered and redacted Markdown as generated by
   # Banzai::ObjectRenderer.
diff --git a/app/models/project.rb b/app/models/project.rb
index a160efba912..73593f04283 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -181,7 +181,7 @@ class Project < ActiveRecord::Base
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :count, to: :forks, prefix: true
   delegate :members, to: :team, prefix: true
-  delegate :add_user, to: :team
+  delegate :add_user, :add_users, to: :team
   delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team
   delegate :empty_repo?, to: :repository
 
diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb
index fa782c6fbb7..f2dfb87dbda 100644
--- a/app/models/project_services/chat_notification_service.rb
+++ b/app/models/project_services/chat_notification_service.rb
@@ -22,7 +22,7 @@ class ChatNotificationService < Service
   end
 
   def can_test?
-    valid?
+    super && valid?
   end
 
   def self.supported_events
diff --git a/app/models/project_team.rb b/app/models/project_team.rb
index 6d6644053f8..543b9b293e0 100644
--- a/app/models/project_team.rb
+++ b/app/models/project_team.rb
@@ -50,8 +50,8 @@ class ProjectTeam
   end
 
   def add_users(users, access_level, current_user: nil, expires_at: nil)
-    ProjectMember.add_users_to_projects(
-      [project.id],
+    ProjectMember.add_users(
+      project,
       users,
       access_level,
       current_user: current_user,
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 2b11ed6128e..7bb874d7744 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -19,7 +19,7 @@ class Repository
   #
   # For example, for entry `:readme` there's a method called `readme` which
   # stores its data in the `readme` cache key.
-  CACHED_METHODS = %i(size commit_count readme version contribution_guide
+  CACHED_METHODS = %i(size commit_count readme contribution_guide
                       changelog license_blob license_key gitignore koding_yml
                       gitlab_ci_yml branch_names tag_names branch_count
                       tag_count avatar exists? empty? root_ref).freeze
@@ -32,7 +32,6 @@ class Repository
     changelog: :changelog,
     license: %i(license_blob license_key),
     contributing: :contribution_guide,
-    version: :version,
     gitignore: :gitignore,
     koding: :koding_yml,
     gitlab_ci: :gitlab_ci_yml,
@@ -109,7 +108,7 @@ class Repository
       offset: offset,
       after: after,
       before: before,
-      follow: path.present?,
+      follow: Array(path).length == 1,
       skip_merges: skip_merges
     }
 
@@ -530,11 +529,6 @@ class Repository
   end
   cache_method :readme
 
-  def version
-    file_on_head(:version)
-  end
-  cache_method :version
-
   def contribution_guide
     file_on_head(:contributing)
   end
diff --git a/app/models/spam_log.rb b/app/models/spam_log.rb
index 3b8b9833565..dd21ee15c6c 100644
--- a/app/models/spam_log.rb
+++ b/app/models/spam_log.rb
@@ -3,9 +3,9 @@ class SpamLog < ActiveRecord::Base
 
   validates :user, presence: true
 
-  def remove_user
+  def remove_user(deleted_by:)
     user.block
-    user.destroy
+    DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true, hard_delete: true)
   end
 
   def text
diff --git a/app/models/user.rb b/app/models/user.rb
index 457ba05fb04..774d4caa806 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -99,9 +99,6 @@ class User < ActiveRecord::Base
   has_many :award_emoji,              dependent: :destroy
   has_many :triggers,                 dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id
 
-  has_many :assigned_issues,          dependent: :nullify, foreign_key: :assignee_id, class_name: "Issue"
-  has_many :assigned_merge_requests,  dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest"
-
   # Issues that a user owns are expected to be moved to the "ghost" user before
   # the user is destroyed. If the user owns any issues during deletion, this
   # should be treated as an exceptional condition.
@@ -197,7 +194,7 @@ class User < ActiveRecord::Base
   scope :admins, -> { where(admin: true) }
   scope :blocked, -> { with_states(:blocked, :ldap_blocked) }
   scope :external, -> { where(external: true) }
-  scope :active, -> { with_state(:active) }
+  scope :active, -> { with_state(:active).non_internal }
   scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all }
   scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') }
   scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) }
@@ -891,20 +888,20 @@ class User < ActiveRecord::Base
     @global_notification_setting
   end
 
-  def assigned_open_merge_request_count(force: false)
-    Rails.cache.fetch(['users', id, 'assigned_open_merge_request_count'], force: force) do
-      assigned_merge_requests.opened.count
+  def assigned_open_merge_requests_count(force: false)
+    Rails.cache.fetch(['users', id, 'assigned_open_merge_requests_count'], force: force) do
+      MergeRequestsFinder.new(self, assignee_id: self.id, state: 'opened').execute.count
     end
   end
 
   def assigned_open_issues_count(force: false)
     Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force) do
-      assigned_issues.opened.count
+      IssuesFinder.new(self, assignee_id: self.id, state: 'opened').execute.count
     end
   end
 
   def update_cache_counts
-    assigned_open_merge_request_count(force: true)
+    assigned_open_merge_requests_count(force: true)
     assigned_open_issues_count(force: true)
   end