Merge remote-tracking branch 'origin/master' into 22191-delete-dynamic-envs-mr

30 files changed, 450 insertions, 92 deletions

diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb
index b01a244032d..2340453831e 100644
--- a/app/models/abuse_report.rb
+++ b/app/models/abuse_report.rb
@@ -1,4 +1,8 @@
 class AbuseReport < ActiveRecord::Base
+  include CacheMarkdownField
+
+  cache_markdown_field :message, pipeline: :single_line
+
   belongs_to :reporter, class_name: 'User'
   belongs_to :user
 
@@ -7,6 +11,9 @@ class AbuseReport < ActiveRecord::Base
   validates :message, presence: true
   validates :user_id, uniqueness: { message: 'has already been reported' }
 
+  # For CacheMarkdownField
+  alias_method :author, :reporter
+
   def remove_user(deleted_by:)
     user.block
     DeleteUserWorker.perform_async(deleted_by.id, user.id, delete_solo_owned_groups: true)
diff --git a/app/models/appearance.rb b/app/models/appearance.rb
index 4cf8dd9a8ce..e4106e1c2e9 100644
--- a/app/models/appearance.rb
+++ b/app/models/appearance.rb
@@ -1,4 +1,8 @@
 class Appearance < ActiveRecord::Base
+  include CacheMarkdownField
+
+  cache_markdown_field :description
+
   validates :title,       presence: true
   validates :description, presence: true
   validates :logo,        file_size: { maximum: 1.megabyte }
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 55d2e07de08..c99aa7772bb 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -1,5 +1,7 @@
 class ApplicationSetting < ActiveRecord::Base
+  include CacheMarkdownField
   include TokenAuthenticatable
+
   add_authentication_token_field :runners_registration_token
   add_authentication_token_field :health_check_access_token
 
@@ -17,6 +19,11 @@ class ApplicationSetting < ActiveRecord::Base
   serialize :domain_whitelist, Array
   serialize :domain_blacklist, Array
 
+  cache_markdown_field :sign_in_text
+  cache_markdown_field :help_page_text
+  cache_markdown_field :shared_runners_text, pipeline: :plain_markdown
+  cache_markdown_field :after_sign_up_text
+
   attr_accessor :domain_whitelist_raw, :domain_blacklist_raw
 
   validates :session_expire_delay,
diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb
index 61498140f27..cb40f33932a 100644
--- a/app/models/broadcast_message.rb
+++ b/app/models/broadcast_message.rb
@@ -1,6 +1,9 @@
 class BroadcastMessage < ActiveRecord::Base
+  include CacheMarkdownField
   include Sortable
 
+  cache_markdown_field :message, pipeline: :broadcast_message
+
   validates :message,   presence: true
   validates :starts_at, presence: true
   validates :ends_at,   presence: true
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 09c5223c60d..fd762b8c5ce 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -1,6 +1,7 @@
 module Ci
   class Build < CommitStatus
     include TokenAuthenticatable
+    include AfterCommitQueue
 
     belongs_to :runner, class_name: 'Ci::Runner'
     belongs_to :trigger_request, class_name: 'Ci::TriggerRequest'
@@ -75,25 +76,20 @@ module Ci
 
     state_machine :status do
       after_transition pending: :running do |build|
-        build.execute_hooks
+        build.run_after_commit do
+          BuildHooksWorker.perform_async(id)
+        end
       end
 
       after_transition any => [:success, :failed, :canceled] do |build|
-        build.update_coverage
-        build.execute_hooks
+        build.run_after_commit do
+          BuildFinishedWorker.perform_async(id)
+        end
       end
 
       after_transition any => [:success] do |build|
-        if build.environment.present?
-          service = CreateDeploymentService.new(
-            build.project, build.user,
-            environment: build.environment,
-            sha: build.sha,
-            ref: build.ref,
-            tag: build.tag,
-            options: build.options.to_h[:environment],
-            variables: build.variables)
-          service.execute(build)
+        build.run_after_commit do
+          BuildSuccessWorker.perform_async(id)
         end
       end
     end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 2cf9892edc5..4fdb5fef4fb 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -3,6 +3,7 @@ module Ci
     extend Ci::Model
     include HasStatus
     include Importable
+    include AfterCommitQueue
 
     self.table_name = 'ci_commits'
 
@@ -56,6 +57,10 @@ module Ci
         pipeline.finished_at = Time.now
       end
 
+      before_transition do |pipeline|
+        pipeline.update_duration
+      end
+
       after_transition [:created, :pending] => :running do |pipeline|
         MergeRequest::Metrics.where(merge_request_id: pipeline.merge_requests.map(&:id)).
           update_all(latest_build_started_at: pipeline.started_at, latest_build_finished_at: nil)
@@ -66,12 +71,16 @@ module Ci
           update_all(latest_build_finished_at: pipeline.finished_at)
       end
 
-      before_transition do |pipeline|
-        pipeline.update_duration
+      after_transition [:created, :pending, :running] => :success do |pipeline|
+        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(id) }
       end
 
       after_transition do |pipeline, transition|
-        pipeline.execute_hooks unless transition.loopback?
+        next if transition.loopback?
+
+        pipeline.run_after_commit do
+          PipelineHooksWorker.perform_async(id)
+        end
       end
     end
 
@@ -292,11 +301,9 @@ module Ci
     # Merge requests for which the current pipeline is running against
     # the merge request's latest commit.
     def merge_requests
-      @merge_requests ||=
-        begin
-          project.merge_requests.where(source_branch: self.ref).
-            select { |merge_request| merge_request.pipeline.try(:id) == self.id }
-        end
+      @merge_requests ||= project.merge_requests
+        .where(source_branch: self.ref)
+        .select { |merge_request| merge_request.pipeline.try(:id) == self.id }
     end
 
     private
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 9fa8d17e74e..7b554be4f9a 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -1,6 +1,7 @@
 class CommitStatus < ActiveRecord::Base
   include HasStatus
   include Importable
+  include AfterCommitQueue
 
   self.table_name = 'ci_builds'
 
@@ -85,25 +86,24 @@ class CommitStatus < ActiveRecord::Base
     end
 
     after_transition do |commit_status, transition|
-      commit_status.pipeline.try do |pipeline|
-        break if transition.loopback?
-
-        if commit_status.complete?
-          ProcessPipelineWorker.perform_async(pipeline.id)
+      next if transition.loopback?
+
+      commit_status.run_after_commit do
+        pipeline.try do |pipeline|
+          if complete?
+            PipelineProcessWorker.perform_async(pipeline.id)
+          else
+            PipelineUpdateWorker.perform_async(pipeline.id)
+          end
         end
-
-        UpdatePipelineWorker.perform_async(pipeline.id)
       end
-
-      true
-    end
-
-    after_transition [:created, :pending, :running] => :success do |commit_status|
-      MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.pipeline.project, nil).trigger(commit_status)
     end
 
     after_transition any => :failed do |commit_status|
-      MergeRequests::AddTodoWhenBuildFailsService.new(commit_status.pipeline.project, nil).execute(commit_status)
+      commit_status.run_after_commit do
+        MergeRequests::AddTodoWhenBuildFailsService
+          .new(pipeline.project, nil).execute(self)
+      end
     end
   end
 
diff --git a/app/models/compare.rb b/app/models/compare.rb
index 4856510f526..3a8bbcb1acd 100644
--- a/app/models/compare.rb
+++ b/app/models/compare.rb
@@ -11,9 +11,10 @@ class Compare
     end
   end
 
-  def initialize(compare, project)
+  def initialize(compare, project, straight: false)
     @compare = compare
     @project = project
+    @straight = straight
   end
 
   def commits
@@ -45,6 +46,18 @@ class Compare
                    end
   end
 
+  def start_commit_sha
+    start_commit.try(:sha)
+  end
+
+  def base_commit_sha
+    base_commit.try(:sha)
+  end
+
+  def head_commit_sha
+    commit.try(:sha)
+  end
+
   def raw_diffs(*args)
     @compare.diffs(*args)
   end
@@ -58,9 +71,9 @@ class Compare
 
   def diff_refs
     Gitlab::Diff::DiffRefs.new(
-      base_sha:  base_commit.try(:sha),
-      start_sha: start_commit.try(:sha),
-      head_sha: commit.try(:sha)
+      base_sha:  @straight ? start_commit_sha : base_commit_sha,
+      start_sha: start_commit_sha,
+      head_sha: head_commit_sha
     )
   end
 end
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
new file mode 100644
index 00000000000..90bd6490a02
--- /dev/null
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -0,0 +1,131 @@
+# This module takes care of updating cache columns for Markdown-containing
+# fields. Use like this in the body of your class:
+#
+#     include CacheMarkdownField
+#     cache_markdown_field :foo
+#     cache_markdown_field :bar
+#     cache_markdown_field :baz, pipeline: :single_line
+#
+# Corresponding foo_html, bar_html and baz_html fields should exist.
+module CacheMarkdownField
+  # Knows about the relationship between markdown and html field names, and
+  # stores the rendering contexts for the latter
+  class FieldData
+    extend Forwardable
+
+    def initialize
+      @data = {}
+    end
+
+    def_delegators :@data, :[], :[]=
+    def_delegator :@data, :keys, :markdown_fields
+
+    def html_field(markdown_field)
+      "#{markdown_field}_html"
+    end
+
+    def html_fields
+      markdown_fields.map {|field| html_field(field) }
+    end
+  end
+
+  # Dynamic registries don't really work in Rails as it's not guaranteed that
+  # every class will be loaded, so hardcode the list.
+  CACHING_CLASSES = %w[
+    AbuseReport
+    Appearance
+    ApplicationSetting
+    BroadcastMessage
+    Issue
+    Label
+    MergeRequest
+    Milestone
+    Namespace
+    Note
+    Project
+    Release
+    Snippet
+  ]
+
+  def self.caching_classes
+    CACHING_CLASSES.map(&:constantize)
+  end
+
+  extend ActiveSupport::Concern
+
+  included do
+    cattr_reader :cached_markdown_fields do
+      FieldData.new
+    end
+
+    # Returns the default Banzai render context for the cached markdown field.
+    def banzai_render_context(field)
+      raise ArgumentError.new("Unknown field: #{field.inspect}") unless
+        cached_markdown_fields.markdown_fields.include?(field)
+
+      # Always include a project key, or Banzai complains
+      project = self.project if self.respond_to?(:project)
+      context = cached_markdown_fields[field].merge(project: project)
+
+      # Banzai is less strict about authors, so don't always have an author key
+      context[:author] = self.author if self.respond_to?(:author)
+
+      context
+    end
+
+    # Allow callers to look up the cache field name, rather than hardcoding it
+    def markdown_cache_field_for(field)
+      raise ArgumentError.new("Unknown field: #{field}") unless
+        cached_markdown_fields.markdown_fields.include?(field)
+
+      cached_markdown_fields.html_field(field)
+    end
+
+    # Always exclude _html fields from attributes (including serialization).
+    # They contain unredacted HTML, which would be a security issue
+    alias_method :attributes_before_markdown_cache, :attributes
+    def attributes
+      attrs = attributes_before_markdown_cache
+
+      cached_markdown_fields.html_fields.each do |field|
+        attrs.delete(field)
+      end
+
+      attrs
+    end
+  end
+
+  class_methods do
+    private
+
+    # Specify that a field is markdown. Its rendered output will be cached in
+    # a corresponding _html field. Any custom rendering options may be provided
+    # as a context.
+    def cache_markdown_field(markdown_field, context = {})
+      raise "Add #{self} to CacheMarkdownField::CACHING_CLASSES" unless
+        CacheMarkdownField::CACHING_CLASSES.include?(self.to_s)
+
+      cached_markdown_fields[markdown_field] = context
+
+      html_field = cached_markdown_fields.html_field(markdown_field)
+      cache_method = "#{markdown_field}_cache_refresh".to_sym
+      invalidation_method = "#{html_field}_invalidated?".to_sym
+
+      define_method(cache_method) do
+        html = Banzai::Renderer.cacheless_render_field(self, markdown_field)
+        __send__("#{html_field}=", html)
+        true
+      end
+
+      # The HTML becomes invalid if any dependent fields change. For now, assume
+      # author and project invalidate the cache in all circumstances.
+      define_method(invalidation_method) do
+        changed_fields = changed_attributes.keys
+        invalidations = changed_fields & [markdown_field.to_s, "author", "project"]
+        !invalidations.empty?
+      end
+
+      before_save cache_method, if: invalidation_method
+    end
+  end
+end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index ff465d2c745..c4b42ad82c7 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -6,6 +6,7 @@
 #
 module Issuable
   extend ActiveSupport::Concern
+  include CacheMarkdownField
   include Participable
   include Mentionable
   include Subscribable
@@ -13,6 +14,9 @@ module Issuable
   include Awardable
 
   included do
+    cache_markdown_field :title, pipeline: :single_line
+    cache_markdown_field :description
+
     belongs_to :author, class_name: "User"
     belongs_to :assignee, class_name: "User"
     belongs_to :updated_by, class_name: "User"
diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb
index be295487fd2..8ed4a56b19b 100644
--- a/app/models/cycle_analytics.rb
+++ b/app/models/cycle_analytics.rb
@@ -2,6 +2,8 @@ class CycleAnalytics
   include Gitlab::Database::Median
   include Gitlab::Database::DateTime
 
+  DEPLOYMENT_METRIC_STAGES = %i[production staging]
+
   def initialize(project, from:)
     @project = project
     @from = from
@@ -66,7 +68,7 @@ class CycleAnalytics
     # cycle analytics stage.
     interval_query = Arel::Nodes::As.new(
       cte_table,
-      subtract_datetimes(base_query, end_time_attrs, start_time_attrs, name.to_s))
+      subtract_datetimes(base_query_for(name), end_time_attrs, start_time_attrs, name.to_s))
 
     median_datetime(cte_table, interval_query, name)
   end
@@ -75,7 +77,7 @@ class CycleAnalytics
   # closes the given issue) with issue and merge request metrics included. The metrics
   # are loaded with an inner join, so issues / merge requests without metrics are
   # automatically excluded.
-  def base_query
+  def base_query_for(name)
     arel_table = MergeRequestsClosingIssues.arel_table
 
     # Load issues
@@ -91,7 +93,11 @@ class CycleAnalytics
             join(MergeRequest::Metrics.arel_table).
             on(MergeRequest.arel_table[:id].eq(MergeRequest::Metrics.arel_table[:merge_request_id]))
 
-    # Limit to merge requests that have been deployed to production after `@from`
-    query.where(MergeRequest::Metrics.arel_table[:first_deployed_to_production_at].gteq(@from))
+    if DEPLOYMENT_METRIC_STAGES.include?(name)
+      # Limit to merge requests that have been deployed to production after `@from`
+      query.where(MergeRequest::Metrics.arel_table[:first_deployed_to_production_at].gteq(@from))
+    end
+
+    query
   end
 end
diff --git a/app/models/deployment.rb b/app/models/deployment.rb
index 070c76339b1..054d54f124e 100644
--- a/app/models/deployment.rb
+++ b/app/models/deployment.rb
@@ -40,7 +40,14 @@ class Deployment < ActiveRecord::Base
   def includes_commit?(commit)
     return false unless commit
 
-    project.repository.is_ancestor?(commit.id, sha)
+    # Before 8.10, deployments didn't have keep-around refs. Any deployment
+    # created before then could have a `sha` referring to a commit that no
+    # longer exists in the repository, so just ignore those.
+    begin
+      project.repository.is_ancestor?(commit.id, sha)
+    rescue Rugged::OdbError
+      false
+    end
   end
 
   def update_merge_request_metrics!
@@ -90,6 +97,10 @@ class Deployment < ActiveRecord::Base
     close_action.present?
   end
 
+  def formatted_deployment_time
+    created_at.to_time.in_time_zone.to_s(:medium)
+  end
+
   private
 
   def ref_path
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 6ec498ea2b7..07f14a7ad8d 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -66,7 +66,22 @@ class Environment < ActiveRecord::Base
     self.name == "production"
   end
 
+  def first_deployment_for(commit)
+    ref = project.repository.ref_name_for_sha(ref_path, commit.sha)
+
+    return nil unless ref
+
+    deployment_id = ref.split('/').last
+    deployments.find(deployment_id)
+  end
+
   def ref_path
     "refs/environments/#{Shellwords.shellescape(name)}"
   end
+
+  def formatted_external_url
+    return nil unless external_url
+
+    external_url.gsub(/\A.*?:\/\//, '')
+  end
 end
diff --git a/app/models/event.rb b/app/models/event.rb
index 633019fe0af..0764cb8cabd 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -68,8 +68,10 @@ class Event < ActiveRecord::Base
       true
     elsif issue? || issue_note?
       Ability.allowed?(user, :read_issue, note? ? note_target : target)
+    elsif merge_request? || merge_request_note?
+      Ability.allowed?(user, :read_merge_request, note? ? note_target : target)
     else
-      ((merge_request? || note?) && target.present?) || milestone?
+      milestone?
     end
   end
 
@@ -280,6 +282,10 @@ class Event < ActiveRecord::Base
     note? && target && target.for_issue?
   end
 
+  def merge_request_note?
+    note? && target && target.for_merge_request?
+  end
+
   def project_snippet_note?
     target.for_snippet?
   end
@@ -335,7 +341,7 @@ class Event < ActiveRecord::Base
     # update the project. Only one query should actually perform the update,
     # hence we add the extra WHERE clause for last_activity_at.
     Project.unscoped.where(id: project_id).
-      where('last_activity_at > ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago).
+      where('last_activity_at <= ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago).
       update_all(last_activity_at: created_at)
   end
 
diff --git a/app/models/global_label.rb b/app/models/global_label.rb
index ddd4bad5c21..698a7bbd327 100644
--- a/app/models/global_label.rb
+++ b/app/models/global_label.rb
@@ -4,6 +4,10 @@ class GlobalLabel
 
   delegate :color, :description, to: :@first_label
 
+  def for_display
+    @first_label
+  end
+
   def self.build_collection(labels)
     labels = labels.group_by(&:title)
 
diff --git a/app/models/global_milestone.rb b/app/models/global_milestone.rb
index bda2b5c5d5d..cde4a568577 100644
--- a/app/models/global_milestone.rb
+++ b/app/models/global_milestone.rb
@@ -4,6 +4,10 @@ class GlobalMilestone
   attr_accessor :title, :milestones
   alias_attribute :name, :title
 
+  def for_display
+    @first_milestone
+  end
+
   def self.build_collection(milestones)
     milestones = milestones.group_by(&:title)
 
@@ -17,6 +21,7 @@ class GlobalMilestone
     @title = title
     @name = title
     @milestones = milestones
+    @first_milestone = milestones.find {|m| m.description.present? } || milestones.first
   end
 
   def safe_title
diff --git a/app/models/label.rb b/app/models/label.rb
index a23140b7d64..e8e12e2904e 100644
--- a/app/models/label.rb
+++ b/app/models/label.rb
@@ -1,4 +1,5 @@
 class Label < ActiveRecord::Base
+  include CacheMarkdownField
   include Referable
   include Subscribable
 
@@ -8,6 +9,8 @@ class Label < ActiveRecord::Base
   None = LabelStruct.new('No Label', 'No Label')
   Any = LabelStruct.new('Any Label', '')
 
+  cache_markdown_field :description, pipeline: :single_line
+
   DEFAULT_COLOR = '#428BCA'
 
   default_value_for :color, DEFAULT_COLOR
diff --git a/app/models/member.rb b/app/models/member.rb
index 38a278ea559..b89ba8ecbb8 100644
--- a/app/models/member.rb
+++ b/app/models/member.rb
@@ -103,7 +103,12 @@ class Member < ActiveRecord::Base
       }
 
       if member.request?
-        ::Members::ApproveAccessRequestService.new(source, current_user, id: member.id).execute
+        ::Members::ApproveAccessRequestService.new(
+          source,
+          current_user,
+          id: member.id,
+          access_level: access_level
+        ).execute
       else
         member.save
       end
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 071dfe54ef9..5ccfe11a2a2 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -31,7 +31,7 @@ class MergeRequest < ActiveRecord::Base
 
   # Temporary fields to store compare vars
   # when creating new merge request
-  attr_accessor :can_be_created, :compare_commits, :compare
+  attr_accessor :can_be_created, :compare_commits, :diff_options, :compare
 
   state_machine :state, initial: :opened do
     event :close do
@@ -196,7 +196,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def diff_size
-    merge_request_diff.size
+    diffs(diff_options).size
   end
 
   def diff_base_commit
@@ -688,12 +688,15 @@ class MergeRequest < ActiveRecord::Base
   def environments
     return [] unless diff_head_commit
 
-    environments = source_project.environments_for(
-      source_branch, diff_head_commit)
-    environments += target_project.environments_for(
-      target_branch, diff_head_commit, with_tags: true)
+    @environments ||=
+      begin
+        environments = source_project.environments_for(
+          source_branch, diff_head_commit)
+        environments += target_project.environments_for(
+          target_branch, diff_head_commit, with_tags: true)
 
-    environments.uniq
+        environments.uniq
+      end
   end
 
   def state_human_name
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 36b8b70870b..b8a10b7968e 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -6,6 +6,9 @@ class MergeRequestDiff < ActiveRecord::Base
   # Prevent store of diff if commits amount more then 500
   COMMITS_SAFE_SIZE = 100
 
+  # Valid types of serialized diffs allowed by Gitlab::Git::Diff
+  VALID_CLASSES = [Hash, Rugged::Patch, Rugged::Diff::Delta]
+
   belongs_to :merge_request
 
   state_machine :state, initial: :empty do
@@ -164,12 +167,24 @@ class MergeRequestDiff < ActiveRecord::Base
     self == merge_request.merge_request_diff
   end
 
-  def compare_with(sha)
-    CompareService.new.execute(project, head_commit_sha, project, sha)
+  def compare_with(sha, straight: true)
+    # When compare merge request versions we want diff A..B instead of A...B
+    # so we handle cases when user does squash and rebase of the commits between versions.
+    # For this reason we set straight to true by default.
+    CompareService.new.execute(project, head_commit_sha, project, sha, straight: straight)
   end
 
   private
 
+  # Old GitLab implementations may have generated diffs as ["--broken-diff"].
+  # Avoid an error 500 by ignoring bad elements. See:
+  # https://gitlab.com/gitlab-org/gitlab-ce/issues/20776
+  def valid_raw_diff?(raw)
+    return false unless raw.respond_to?(:each)
+
+    raw.any? { |element| VALID_CLASSES.include?(element.class) }
+  end
+
   def dump_commits(commits)
     commits.map(&:to_hash)
   end
@@ -200,7 +215,7 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def load_diffs(raw, options)
-    if raw.respond_to?(:each)
+    if valid_raw_diff?(raw)
       if paths = options[:paths]
         raw = raw.select do |diff|
           paths.include?(diff[:old_path]) || paths.include?(diff[:new_path])
diff --git a/app/models/milestone.rb b/app/models/milestone.rb
index 44c3cbb2c73..23aecbfa3a6 100644
--- a/app/models/milestone.rb
+++ b/app/models/milestone.rb
@@ -6,12 +6,16 @@ class Milestone < ActiveRecord::Base
   Any = MilestoneStruct.new('Any Milestone', '', -1)
   Upcoming = MilestoneStruct.new('Upcoming', '#upcoming', -2)
 
+  include CacheMarkdownField
   include InternalId
   include Sortable
   include Referable
   include StripAttribute
   include Milestoneish
 
+  cache_markdown_field :title, pipeline: :single_line
+  cache_markdown_field :description
+
   belongs_to :project
   has_many :issues
   has_many :labels, -> { distinct.reorder('labels.title') },  through: :issues
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 919b3b1f095..b67049f0f55 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -1,9 +1,12 @@
 class Namespace < ActiveRecord::Base
   acts_as_paranoid
 
+  include CacheMarkdownField
   include Sortable
   include Gitlab::ShellAdapter
 
+  cache_markdown_field :description, pipeline: :description
+
   has_many :projects, dependent: :destroy
   belongs_to :owner, class_name: "User"
 
@@ -58,15 +61,13 @@ class Namespace < ActiveRecord::Base
     def clean_path(path)
       path = path.dup
       # Get the email username by removing everything after an `@` sign.
-      path.gsub!(/@.*\z/,             "")
-      # Usernames can't end in .git, so remove it.
-      path.gsub!(/\.git\z/,           "")
-      # Remove dashes at the start of the username.
-      path.gsub!(/\A-+/,              "")
-      # Remove periods at the end of the username.
-      path.gsub!(/\.+\z/,             "")
+      path.gsub!(/@.*\z/,                "")
       # Remove everything that's not in the list of allowed characters.
-      path.gsub!(/[^a-zA-Z0-9_\-\.]/, "")
+      path.gsub!(/[^a-zA-Z0-9_\-\.]/,    "")
+      # Remove trailing violations ('.atom', '.git', or '.')
+      path.gsub!(/(\.atom|\.git|\.)*\z/, "")
+      # Remove leading violations ('-')
+      path.gsub!(/\A\-+/,                "")
 
       # Users with the great usernames of "." or ".." would end up with a blank username.
       # Work around that by setting their username to "blank", followed by a counter.
diff --git a/app/models/note.rb b/app/models/note.rb
index f2656df028b..2d644b03e4d 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -6,10 +6,13 @@ class Note < ActiveRecord::Base
   include Awardable
   include Importable
   include FasterCacheKeys
+  include CacheMarkdownField
+
+  cache_markdown_field :note, pipeline: :note
 
   # Attribute containing rendered and redacted Markdown as generated by
   # Banzai::ObjectRenderer.
-  attr_accessor :note_html
+  attr_accessor :redacted_note_html
 
   # An Array containing the number of visible references as generated by
   # Banzai::ObjectRenderer
diff --git a/app/models/project.rb b/app/models/project.rb
index 12705f9ae48..2e8073733d4 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -6,6 +6,7 @@ class Project < ActiveRecord::Base
   include Gitlab::VisibilityLevel
   include Gitlab::CurrentSettings
   include AccessRequestable
+  include CacheMarkdownField
   include Referable
   include Sortable
   include AfterCommitQueue
@@ -15,8 +16,13 @@ class Project < ActiveRecord::Base
 
   extend Gitlab::ConfigHelper
 
+  class BoardLimitExceeded < StandardError; end
+
+  NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'
 
+  cache_markdown_field :description, pipeline: :description
+
   delegate :feature_available?, :builds_enabled?, :wiki_enabled?, :merge_requests_enabled?, to: :project_feature, allow_nil: true
 
   default_value_for :archived, false
@@ -62,8 +68,7 @@ class Project < ActiveRecord::Base
   belongs_to :namespace
 
   has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event', foreign_key: 'project_id'
-
-  has_one :board, dependent: :destroy
+  has_many :boards, before_add: :validate_board_limit, dependent: :destroy
 
   # Project services
   has_many :services
@@ -372,19 +377,9 @@ class Project < ActiveRecord::Base
       %r{(?<project>#{name_pattern}/#{name_pattern})}
     end
 
-    def trending(since = 1.month.ago)
-      # By counting in the JOIN we don't expose the GROUP BY to the outer query.
-      # This means that calls such as "any?" and "count" just return a number of
-      # the total count, instead of the counts grouped per project as a Hash.
-      join_body = "INNER JOIN (
-        SELECT project_id, COUNT(*) AS amount
-        FROM notes
-        WHERE created_at >= #{sanitize(since)}
-        AND system IS FALSE
-        GROUP BY project_id
-      ) join_note_counts ON projects.id = join_note_counts.project_id"
-
-      joins(join_body).reorder('join_note_counts.amount DESC')
+    def trending
+      joins('INNER JOIN trending_projects ON projects.id = trending_projects.project_id').
+        reorder('trending_projects.id ASC')
     end
 
     def cached_count
@@ -495,7 +490,7 @@ class Project < ActiveRecord::Base
   end
 
   def import_url
-    if import_data && super
+    if import_data && super.present?
       import_url = Gitlab::UrlSanitizer.new(super, credentials: import_data.credentials)
       import_url.full_url
     else
@@ -834,11 +829,6 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def update_merge_requests(oldrev, newrev, ref, user)
-    MergeRequests::RefreshService.new(self, user).
-      execute(oldrev, newrev, ref)
-  end
-
   def valid_repo?
     repository.exists?
   rescue
@@ -1346,4 +1336,8 @@ class Project < ActiveRecord::Base
 
     shared_projects.any?
   end
+
+  def validate_board_limit(board)
+    raise BoardLimitExceeded, 'Number of permitted boards exceeded' if boards.size >= NUMBER_OF_PERMITTED_BOARDS
+  end
 end
diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb
index 7613cbdea93..db46def11eb 100644
--- a/app/models/project_group_link.rb
+++ b/app/models/project_group_link.rb
@@ -10,7 +10,7 @@ class ProjectGroupLink < ActiveRecord::Base
   belongs_to :group
 
   validates :project_id, presence: true
-  validates :group_id, presence: true
+  validates :group, presence: true
   validates :group_id, uniqueness: { scope: [:project_id], message: "already shared with this group" }
   validates :group_access, presence: true
   validates :group_access, inclusion: { in: Gitlab::Access.values }, presence: true
diff --git a/app/models/release.rb b/app/models/release.rb
index e196b84eb18..c936899799e 100644
--- a/app/models/release.rb
+++ b/app/models/release.rb
@@ -1,4 +1,8 @@
 class Release < ActiveRecord::Base
+  include CacheMarkdownField
+
+  cache_markdown_field :description
+
   belongs_to :project
 
   validates :description, :project, :tag, presence: true
diff --git a/app/models/repository.rb b/app/models/repository.rb
index eb574555df6..72e473871fa 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -111,8 +111,10 @@ class Repository
   def find_commits_by_message(query, ref = nil, path = nil, limit = 1000, offset = 0)
     ref ||= root_ref
 
-    # Limited to 1000 commits for now, could be parameterized?
-    args = %W(#{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset} --max-count #{limit} --grep=#{query})
+    args = %W(
+      #{Gitlab.config.git.bin_path} log #{ref} --pretty=%H --skip #{offset}
+      --max-count #{limit} --grep=#{query} --regexp-ignore-case
+    )
     args = args.concat(%W(-- #{path})) if path.present?
 
     git_log_results = Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:chomp)
@@ -717,6 +719,14 @@ class Repository
     end
   end
 
+  def ref_name_for_sha(ref_path, sha)
+    args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha})
+
+    # Not found -> ["", 0]
+    # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
+    Gitlab::Popen.popen(args, path_to_repo).first.split.last
+  end
+
   def refs_contains_sha(ref_type, sha)
     args = %W(#{Gitlab.config.git.bin_path} #{ref_type} --contains #{sha})
     names = Gitlab::Popen.popen(args, path_to_repo).first
@@ -838,6 +848,52 @@ class Repository
     end
   end
 
+  def multi_action(user:, branch:, message:, actions:, author_email: nil, author_name: nil)
+    update_branch_with_hooks(user, branch) do |ref|
+      index = rugged.index
+      parents = []
+      branch = find_branch(ref)
+
+      if branch
+        last_commit = branch.target
+        index.read_tree(last_commit.raw_commit.tree)
+        parents = [last_commit.sha]
+      end
+
+      actions.each do |action|
+        case action[:action]
+        when :create, :update, :move
+          mode =
+            case action[:action]
+            when :update
+              index.get(action[:file_path])[:mode]
+            when :move
+              index.get(action[:previous_path])[:mode]
+            end
+          mode ||= 0o100644
+
+          index.remove(action[:previous_path]) if action[:action] == :move
+
+          content = action[:encoding] == 'base64' ? Base64.decode64(action[:content]) : action[:content]
+          oid = rugged.write(content, :blob)
+
+          index.add(path: action[:file_path], oid: oid, mode: mode)
+        when :delete
+          index.remove(action[:file_path])
+        end
+      end
+
+      options = {
+        tree: index.write_tree(rugged),
+        message: message,
+        parents: parents
+      }
+      options.merge!(get_committer_and_author(user, email: author_email, name: author_name))
+
+      Rugged::Commit.create(rugged, options)
+    end
+  end
+
   def get_committer_and_author(user, email: nil, name: nil)
     committer = user_to_committer(user)
     author = Gitlab::Git::committer_hash(email: email, name: name) || committer
@@ -968,7 +1024,8 @@ class Repository
     root_ref_commit = commit(root_ref)
 
     if branch_commit
-      is_ancestor?(branch_commit.id, root_ref_commit.id)
+      same_head = branch_commit.id == root_ref_commit.id
+      !same_head && is_ancestor?(branch_commit.id, root_ref_commit.id)
     else
       nil
     end
diff --git a/app/models/snippet.rb b/app/models/snippet.rb
index 8a1730f3f36..2373b445009 100644
--- a/app/models/snippet.rb
+++ b/app/models/snippet.rb
@@ -1,11 +1,21 @@
 class Snippet < ActiveRecord::Base
   include Gitlab::VisibilityLevel
   include Linguist::BlobHelper
+  include CacheMarkdownField
   include Participable
   include Referable
   include Sortable
   include Awardable
 
+  cache_markdown_field :title, pipeline: :single_line
+  cache_markdown_field :content
+
+  # If file_name changes, it invalidates content
+  alias_method :default_content_html_invalidator, :content_html_invalidated?
+  def content_html_invalidated?
+    default_content_html_invalidator || file_name_changed?
+  end
+
   default_value_for :visibility_level, Snippet::PRIVATE
 
   belongs_to :author, class_name: 'User'
diff --git a/app/models/trending_project.rb b/app/models/trending_project.rb
new file mode 100644
index 00000000000..27e3732da17
--- /dev/null
+++ b/app/models/trending_project.rb
@@ -0,0 +1,35 @@
+class TrendingProject < ActiveRecord::Base
+  belongs_to :project
+
+  # The number of months to include in the trending calculation.
+  MONTHS_TO_INCLUDE = 1
+
+  # The maximum number of projects to include in the trending set.
+  PROJECTS_LIMIT = 100
+
+  # Populates the trending projects table with the current list of trending
+  # projects.
+  def self.refresh!
+    # The calculation **must** run in a transaction. If the removal of data and
+    # insertion of new data were to run separately a user might end up with an
+    # empty list of trending projects for a short period of time.
+    transaction do
+      delete_all
+
+      timestamp = connection.quote(MONTHS_TO_INCLUDE.months.ago)
+
+      connection.execute <<-EOF.strip_heredoc
+        INSERT INTO #{table_name} (project_id)
+        SELECT project_id
+        FROM notes
+        INNER JOIN projects ON projects.id = notes.project_id
+        WHERE notes.created_at >= #{timestamp}
+        AND notes.system IS FALSE
+        AND projects.visibility_level = #{Gitlab::VisibilityLevel::PUBLIC}
+        GROUP BY project_id
+        ORDER BY count(*) DESC
+        LIMIT #{PROJECTS_LIMIT};
+      EOF
+    end
+  end
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index 508efd85050..f367f4616fb 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -589,6 +589,11 @@ class User < ActiveRecord::Base
   end
 
   def set_projects_limit
+    # `User.select(:id)` raises
+    # `ActiveModel::MissingAttributeError: missing attribute: projects_limit`
+    # without this safeguard!
+    return unless self.has_attribute?(:projects_limit)
+
     connection_default_value_defined = new_record? && !projects_limit_changed?
     return unless self.projects_limit.nil? || connection_default_value_defined
 
@@ -902,7 +907,7 @@ class User < ActiveRecord::Base
       if domain_matches?(allowed_domains, self.email)
         valid = true
       else
-        error = "is not whitelisted. Email domains valid for registration are: #{allowed_domains.join(', ')}"
+        error = "domain is not authorized for sign-up"
         valid = false
       end
     end