Merge remote-tracking branch 'origin/master' into 2979-personal-access-tokens

37 files changed, 862 insertions, 433 deletions

diff --git a/app/models/ability.rb b/app/models/ability.rb
index 6103a2947e2..44515550d9e 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -23,20 +23,41 @@ class Ability
       end.concat(global_abilities(user))
     end
 
+    # Given a list of users and a project this method returns the users that can
+    # read the given project.
+    def users_that_can_read_project(users, project)
+      if project.public?
+        users
+      else
+        users.select do |user|
+          if user.admin?
+            true
+          elsif project.internal? && !user.external?
+            true
+          elsif project.owner == user
+            true
+          elsif project.team.members.include?(user)
+            true
+          else
+            false
+          end
+        end
+      end
+    end
+
     # List of possible abilities for anonymous user
     def anonymous_abilities(user, subject)
-      case true
-      when subject.is_a?(PersonalSnippet)
+      if subject.is_a?(PersonalSnippet)
         anonymous_personal_snippet_abilities(subject)
-      when subject.is_a?(ProjectSnippet)
+      elsif subject.is_a?(ProjectSnippet)
         anonymous_project_snippet_abilities(subject)
-      when subject.is_a?(CommitStatus)
+      elsif subject.is_a?(CommitStatus)
         anonymous_commit_status_abilities(subject)
-      when subject.is_a?(Project) || subject.respond_to?(:project)
+      elsif subject.is_a?(Project) || subject.respond_to?(:project)
         anonymous_project_abilities(subject)
-      when subject.is_a?(Group) || subject.respond_to?(:group)
+      elsif subject.is_a?(Group) || subject.respond_to?(:group)
         anonymous_group_abilities(subject)
-      when subject.is_a?(User)
+      elsif subject.is_a?(User)
         anonymous_user_abilities
       else
         []
@@ -60,7 +81,9 @@ class Ability
           :read_project_member,
           :read_merge_request,
           :read_note,
+          :read_pipeline,
           :read_commit_status,
+          :read_container_image,
           :download_code
         ]
 
@@ -203,6 +226,8 @@ class Ability
         :admin_label,
         :read_commit_status,
         :read_build,
+        :read_container_image,
+        :read_pipeline,
       ]
     end
 
@@ -214,9 +239,13 @@ class Ability
         :update_commit_status,
         :create_build,
         :update_build,
+        :create_pipeline,
+        :update_pipeline,
         :create_merge_request,
         :create_wiki,
-        :push_code
+        :push_code,
+        :create_container_image,
+        :update_container_image,
       ]
     end
 
@@ -242,7 +271,9 @@ class Ability
         :admin_wiki,
         :admin_project,
         :admin_commit_status,
-        :admin_build
+        :admin_build,
+        :admin_container_image,
+        :admin_pipeline
       ]
     end
 
@@ -285,6 +316,11 @@ class Ability
 
       unless project.builds_enabled
         rules += named_abilities('build')
+        rules += named_abilities('pipeline')
+      end
+
+      unless project.container_registry_enabled
+        rules += named_abilities('container_image')
       end
 
       rules
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 7039db2d41e..42f908aa344 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -1,11 +1,13 @@
 class ApplicationSetting < ActiveRecord::Base
   include TokenAuthenticatable
   add_authentication_token_field :runners_registration_token
+  add_authentication_token_field :health_check_access_token
 
   CACHE_KEY = 'application_setting.last'
 
   serialize :restricted_visibility_levels
   serialize :import_sources
+  serialize :disabled_oauth_sign_in_sources, Array
   serialize :restricted_signup_domains, Array
   attr_accessor :restricted_signup_domains_raw
 
@@ -49,6 +51,10 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
 
+  validates :container_registry_token_expire_delay,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
   validates_each :restricted_visibility_levels do |record, attr, value|
     unless value.nil?
       value.each do |level|
@@ -69,7 +75,18 @@ class ApplicationSetting < ActiveRecord::Base
     end
   end
 
+  validates_each :disabled_oauth_sign_in_sources do |record, attr, value|
+    unless value.nil?
+      value.each do |source|
+        unless Devise.omniauth_providers.include?(source.to_sym)
+          record.errors.add(attr, "'#{source}' is not an OAuth sign-in source")
+        end
+      end
+    end
+  end
+
   before_save :ensure_runners_registration_token
+  before_save :ensure_health_check_access_token
 
   after_commit do
     Rails.cache.write(CACHE_KEY, self)
@@ -85,6 +102,10 @@ class ApplicationSetting < ActiveRecord::Base
     Rails.cache.delete(CACHE_KEY)
   end
 
+  def self.cached
+    Rails.cache.fetch(CACHE_KEY)
+  end
+
   def self.create_from_defaults
     create(
       default_projects_limit: Settings.gitlab['default_projects_limit'],
@@ -107,6 +128,9 @@ class ApplicationSetting < ActiveRecord::Base
       recaptcha_enabled: false,
       akismet_enabled: false,
       repository_checks_enabled: true,
+      disabled_oauth_sign_in_sources: [],
+      send_user_confirmation_email: false,
+      container_registry_token_expire_delay: 5,
     )
   end
 
@@ -133,4 +157,8 @@ class ApplicationSetting < ActiveRecord::Base
   def runners_registration_token
     ensure_runners_registration_token!
   end
+
+  def health_check_access_token
+    ensure_health_check_access_token!
+  end
 end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index c9506dd3057..64723ab6b4b 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -53,6 +53,7 @@ module Ci
         new_build.stage_idx = build.stage_idx
         new_build.trigger_request = build.trigger_request
         new_build.save
+        MergeRequests::AddTodoWhenBuildFailsService.new(build.project, nil).close(new_build)
         new_build
       end
     end
@@ -95,8 +96,12 @@ module Ci
     end
 
     def trace_html
-      html = Ci::Ansi2html::convert(trace) if trace.present?
-      html || ''
+      trace_with_state[:html] || ''
+    end
+
+    def trace_with_state(state = nil)
+      trace_with_state = Ci::Ansi2html::convert(trace, state) if trace.present?
+      trace_with_state || {}
     end
 
     def timeout
@@ -201,7 +206,7 @@ module Ci
     end
 
     def recreate_trace_dir
-      unless Dir.exists?(dir_to_trace)
+      unless Dir.exist?(dir_to_trace)
         FileUtils.mkdir_p(dir_to_trace)
       end
     end
@@ -281,14 +286,20 @@ module Ci
       project.runners_token
     end
 
-    def valid_token? token
+    def valid_token?(token)
       project.valid_runners_token? token
     end
 
     def can_be_served?(runner)
+      return false unless has_tags? || runner.run_untagged?
+
       (tag_list - runner.tag_list).empty?
     end
 
+    def has_tags?
+      tag_list.any?
+    end
+
     def any_runners_online?
       project.any_runners? { |runner| runner.active? && runner.online? && can_be_served?(runner) }
     end
@@ -302,6 +313,7 @@ module Ci
       build_data = Gitlab::BuildDataBuilder.build(self)
       project.execute_hooks(build_data.dup, :build_hooks)
       project.execute_services(build_data.dup, :build_hooks)
+      project.running_or_pending_build_count(force: true)
     end
 
     def artifacts?
diff --git a/app/models/ci/commit.rb b/app/models/ci/commit.rb
index f4b61c75ab6..f22b573a94c 100644
--- a/app/models/ci/commit.rb
+++ b/app/models/ci/commit.rb
@@ -8,8 +8,6 @@ module Ci
     has_many :builds, class_name: 'Ci::Build'
     has_many :trigger_requests, dependent: :destroy, class_name: 'Ci::TriggerRequest'
 
-    delegate :stages, to: :statuses
-
     validates_presence_of :sha
     validates_presence_of :status
     validate :valid_commit_sha
@@ -22,7 +20,8 @@ module Ci
     end
 
     def self.stages
-      CommitStatus.where(commit: all).stages
+      # We use pluck here due to problems with MySQL which doesn't allow LIMIT/OFFSET in queries
+      CommitStatus.where(commit: pluck(:id)).stages
     end
 
     def project_id
@@ -67,6 +66,29 @@ module Ci
       end
     end
 
+    def cancelable?
+      builds.running_or_pending.any?
+    end
+
+    def cancel_running
+      builds.running_or_pending.each(&:cancel)
+    end
+
+    def retry_failed
+      builds.latest.failed.select(&:retryable?).each(&:retry)
+    end
+
+    def latest?
+      return false unless ref
+      commit = project.commit(ref)
+      return false unless commit
+      commit.sha == sha
+    end
+
+    def triggered?
+      trigger_requests.any?
+    end
+
     def create_builds(user, trigger_request = nil)
       return unless config_processor
       config_processor.stages.any? do |stage|
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 0cdb64d6c33..adb65292208 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -3,7 +3,8 @@ module Ci
     extend Ci::Model
 
     LAST_CONTACT_TIME = 5.minutes.ago
-    AVAILABLE_SCOPES = ['specific', 'shared', 'active', 'paused', 'online']
+    AVAILABLE_SCOPES = %w[specific shared active paused online]
+    FORM_EDITABLE = %i[description tag_list active run_untagged]
 
     has_many :builds, class_name: 'Ci::Build'
     has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject'
@@ -25,6 +26,8 @@ module Ci
         .where("ci_runner_projects.gl_project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id)
     end
 
+    validate :tag_constraints
+
     acts_as_taggable
 
     # Searches for runners matching the given query.
@@ -57,7 +60,7 @@ module Ci
     end
 
     def display_name
-      return short_sha unless !description.blank?
+      return short_sha if description.blank?
 
       description
     end
@@ -95,5 +98,18 @@ module Ci
     def short_sha
       token[0...8] if token
     end
+
+    def has_tags?
+      tag_list.any?
+    end
+
+    private
+
+    def tag_constraints
+      unless has_tags? || run_untagged?
+        errors.add(:tags_list,
+          'can not be empty when runner is not allowed to pick untagged jobs')
+      end
+    end
   end
 end
diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb
index 10802f64813..f8d5d4486fd 100644
--- a/app/models/ci/variable.rb
+++ b/app/models/ci/variable.rb
@@ -11,6 +11,9 @@ module Ci
       format: { with: /\A[a-zA-Z0-9_]+\z/,
                 message: "can contain only letters, digits and '_'." }
 
-    attr_encrypted :value, mode: :per_attribute_iv_and_salt, key: Gitlab::Application.secrets.db_key_base
+    attr_encrypted :value, 
+       mode: :per_attribute_iv_and_salt,
+       key: Gitlab::Application.secrets.db_key_base,
+       algorithm: 'aes-256-cbc'
   end
 end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 562c3ed15b2..f96c7cb34d0 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -8,7 +8,10 @@ class Commit
   include StaticModel
 
   attr_mentionable :safe_message, pipeline: :single_line
-  participant :author, :committer, :notes
+
+  participant :author
+  participant :committer
+  participant :notes_with_associations
 
   attr_accessor :project
 
@@ -194,6 +197,10 @@ class Commit
     project.notes.for_commit_id(self.id)
   end
 
+  def notes_with_associations
+    notes.includes(:author, :project)
+  end
+
   def method_missing(m, *args, &block)
     @raw.send(m, *args, &block)
   end
@@ -219,7 +226,7 @@ class Commit
   def revert_branch_name
     "revert-#{short_id}"
   end
-  
+
   def cherry_pick_branch_name
     project.repository.next_branch("cherry-pick-#{short_id}", mild: true)
   end
@@ -251,11 +258,13 @@ class Commit
   end
 
   def has_been_reverted?(current_user = nil, noteable = self)
-    Gitlab::ReferenceExtractor.lazily do
-      noteable.notes.system.flat_map do |note|
-        note.all_references(current_user).commits
-      end
-    end.any? { |commit_ref| commit_ref.reverts_commit?(self) }
+    ext = all_references(current_user)
+
+    noteable.notes_with_associations.system.each do |note|
+      note.all_references(current_user, extractor: ext)
+    end
+
+    ext.commits.any? { |commit_ref| commit_ref.reverts_commit?(self) }
   end
 
   def change_type_title
diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb
index 51673897d98..4066958f67c 100644
--- a/app/models/commit_range.rb
+++ b/app/models/commit_range.rb
@@ -62,7 +62,7 @@ class CommitRange
   def initialize(range_string, project)
     @project = project
 
-    range_string.strip!
+    range_string = range_string.strip
 
     unless range_string =~ /\A#{PATTERN}\z/
       raise ArgumentError, "invalid CommitRange string format: #{range_string}"
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index cacbc13b391..f774b6e0efb 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -14,7 +14,8 @@ class CommitStatus < ActiveRecord::Base
   alias_attribute :author, :user
 
   scope :latest, -> { where(id: unscope(:select).select('max(id)').group(:name, :commit_id)) }
-  scope :ordered, -> { order(:ref, :stage_idx, :name) }
+  scope :retried, -> { where.not(id: latest) }
+  scope :ordered, -> { order(:name) }
   scope :ignored, -> { where(allow_failure: true, status: [:failed, :canceled]) }
 
   state_machine :status, initial: :pending do
@@ -45,6 +46,10 @@ class CommitStatus < ActiveRecord::Base
     after_transition [:pending, :running] => :success do |commit_status|
       MergeRequests::MergeWhenBuildSucceedsService.new(commit_status.commit.project, nil).trigger(commit_status)
     end
+
+    after_transition any => :failed do |commit_status|
+      MergeRequests::AddTodoWhenBuildFailsService.new(commit_status.commit.project, nil).execute(commit_status)
+    end
   end
 
   delegate :sha, :short_sha, to: :commit
@@ -54,13 +59,15 @@ class CommitStatus < ActiveRecord::Base
   end
 
   def self.stages
-    order_by = 'max(stage_idx)'
-    group('stage').order(order_by).pluck(:stage, order_by).map(&:first).compact
+    # We group by stage name, but order stages by theirs' index
+    unscoped.from(all, :sg).group('stage').order('max(stage_idx)', 'stage').pluck('sg.stage')
   end
 
   def self.stages_status
-    all.stages.inject({}) do |h, stage|
-      h[stage] = all.where(stage: stage).status
+    # We execute subquery for each stage to calculate a stage status
+    statuses = unscoped.from(all, :sg).group('stage').pluck('sg.stage', all.where('stage=sg.stage').status_sql)
+    statuses.inject({}) do |h, k|
+      h[k.first] = k.last
       h
     end
   end
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index c1248b53031..e86d5236abb 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -31,18 +31,21 @@ module Issuable
     scope :unassigned, -> { where("assignee_id IS NULL") }
     scope :of_projects, ->(ids) { where(project_id: ids) }
     scope :of_milestones, ->(ids) { where(milestone_id: ids) }
+    scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) }
     scope :opened, -> { with_state(:opened, :reopened) }
     scope :only_opened, -> { with_state(:opened) }
     scope :only_reopened, -> { with_state(:reopened) }
     scope :closed, -> { with_state(:closed) }
-    scope :order_milestone_due_desc, -> { outer_join_milestone.reorder('milestones.due_date IS NULL ASC, milestones.due_date DESC, milestones.id DESC') }
-    scope :order_milestone_due_asc, -> { outer_join_milestone.reorder('milestones.due_date IS NULL ASC, milestones.due_date ASC, milestones.id ASC') }
-    scope :without_label, -> { joins("LEFT OUTER JOIN label_links ON label_links.target_type = '#{name}' AND label_links.target_id = #{table_name}.id").where(label_links: { id: nil }) }
 
+    scope :left_joins_milestones,    -> { joins("LEFT OUTER JOIN milestones ON #{table_name}.milestone_id = milestones.id") }
+    scope :order_milestone_due_desc, -> { left_joins_milestones.reorder('milestones.due_date IS NULL, milestones.id IS NULL, milestones.due_date DESC') }
+    scope :order_milestone_due_asc,  -> { left_joins_milestones.reorder('milestones.due_date IS NULL, milestones.id IS NULL, milestones.due_date ASC') }
+
+    scope :without_label, -> { joins("LEFT OUTER JOIN label_links ON label_links.target_type = '#{name}' AND label_links.target_id = #{table_name}.id").where(label_links: { id: nil }) }
     scope :join_project, -> { joins(:project) }
     scope :references_project, -> { references(:project) }
     scope :non_archived, -> { join_project.where(projects: { archived: false }) }
-    scope :outer_join_milestone, -> { joins("LEFT OUTER JOIN milestones ON milestones.id = #{table_name}.milestone_id") }
+
 
     delegate :name,
              :email,
@@ -56,11 +59,23 @@ module Issuable
              prefix: true
 
     attr_mentionable :title, pipeline: :single_line
-    attr_mentionable :description, cache: true
-    participant :author, :assignee, :notes_with_associations
+    attr_mentionable :description
+
+    participant :author
+    participant :assignee
+    participant :notes_with_associations
+
     strip_attributes :title
 
     acts_as_paranoid
+
+    after_save :update_assignee_cache_counts, if: :assignee_id_changed?
+
+    def update_assignee_cache_counts
+      # make sure we flush the cache for both the old *and* new assignee
+      User.find(assignee_id_was).update_cache_counts if assignee_id_was
+      assignee.update_cache_counts if assignee
+    end
   end
 
   module ClassMethods
@@ -123,13 +138,29 @@ module Issuable
       joins(join_clause).group(issuable_table[:id]).reorder("COUNT(notes.id) DESC")
     end
 
-    def with_label(title)
+    def with_label(title, sort = nil)
       if title.is_a?(Array) && title.size > 1
-        joins(:labels).where(labels: { title: title }).group(arel_table[:id]).having("COUNT(DISTINCT labels.title) = #{title.size}")
+        joins(:labels).where(labels: { title: title }).group(*grouping_columns(sort)).having("COUNT(DISTINCT labels.title) = #{title.size}")
       else
         joins(:labels).where(labels: { title: title })
       end
     end
+
+    # Includes table keys in group by clause when sorting
+    # preventing errors in postgres
+    #
+    # Returns an array of arel columns
+    def grouping_columns(sort)
+      grouping_columns = [arel_table[:id]]
+
+      if ["milestone_due_desc", "milestone_due_asc"].include?(sort)
+        milestone_table = Milestone.arel_table
+        grouping_columns << milestone_table[:id]
+        grouping_columns << milestone_table[:due_date]
+      end
+
+      grouping_columns
+    end
   end
 
   def today?
@@ -182,6 +213,10 @@ module Issuable
     hook_data
   end
 
+  def labels_array
+    labels.to_a
+  end
+
   def label_names
     labels.order('title ASC').pluck(:title)
   end
diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb
index b381d225485..f00b5b8497c 100644
--- a/app/models/concerns/mentionable.rb
+++ b/app/models/concerns/mentionable.rb
@@ -23,7 +23,7 @@ module Mentionable
 
   included do
     if self < Participable
-      participant ->(current_user) { mentioned_users(current_user) }
+      participant -> (user, ext) { all_references(user, extractor: ext) }
     end
   end
 
@@ -43,23 +43,22 @@ module Mentionable
     self
   end
 
-  def all_references(current_user = nil, text = nil)
-    ext = Gitlab::ReferenceExtractor.new(self.project, current_user || self.author, self.author)
+  def all_references(current_user = nil, text = nil, extractor: nil)
+    extractor ||= Gitlab::ReferenceExtractor.
+      new(project, current_user || author)
 
     if text
-      ext.analyze(text)
+      extractor.analyze(text, author: author)
     else
       self.class.mentionable_attrs.each do |attr, options|
-        text = send(attr)
+        text = __send__(attr)
+        options = options.merge(cache_key: [self, attr], author: author)
 
-        context = options.dup
-        context[:cache_key] = [self, attr] if context.delete(:cache) && self.persisted?
-
-        ext.analyze(text, context)
+        extractor.analyze(text, options)
       end
     end
 
-    ext
+    extractor
   end
 
   def mentioned_users(current_user = nil)
diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb
index fc6f83b918b..9056722f45e 100644
--- a/app/models/concerns/participable.rb
+++ b/app/models/concerns/participable.rb
@@ -3,8 +3,6 @@
 # Contains functionality related to objects that can have participants, such as
 # an author, an assignee and people mentioned in its description or comments.
 #
-# Used by Issue, Note, MergeRequest, Snippet and Commit.
-#
 # Usage:
 #
 #     class Issue < ActiveRecord::Base
@@ -12,22 +10,36 @@
 #
 #       # ...
 #
-#       participant :author, :assignee, :notes, ->(current_user) { mentioned_users(current_user) }
+#       participant :author
+#       participant :assignee
+#       participant :notes
+#
+#       participant -> (current_user, ext) do
+#         ext.analyze('...')
+#       end
 #     end
 #
 #     issue = Issue.last
 #     users = issue.participants
-#     # `users` will contain the issue's author, its assignee,
-#     # all users returned by its #mentioned_users method,
-#     # as well as all participants to all of the issue's notes,
-#     # since Note implements Participable as well.
-#
 module Participable
   extend ActiveSupport::Concern
 
   module ClassMethods
-    def participant(*attrs)
-      participant_attrs.concat(attrs)
+    # Adds a list of participant attributes. Attributes can either be symbols or
+    # Procs.
+    #
+    # When using a Proc instead of a Symbol the Proc will be given two
+    # arguments:
+    #
+    # 1. The current user (as an instance of User)
+    # 2. An instance of `Gitlab::ReferenceExtractor`
+    #
+    # It is expected that a Proc populates the given reference extractor
+    # instance with data. The return value of the Proc is ignored.
+    #
+    # attr - The name of the attribute or a Proc
+    def participant(attr)
+      participant_attrs << attr
     end
 
     def participant_attrs
@@ -35,42 +47,42 @@ module Participable
     end
   end
 
-  # Be aware that this method makes a lot of sql queries.
-  # Save result into variable if you are going to reuse it inside same request
-  def participants(current_user = self.author)
-    participants =
-      Gitlab::ReferenceExtractor.lazily do
-        self.class.participant_attrs.flat_map do |attr|
-          value =
-            if attr.respond_to?(:call)
-              instance_exec(current_user, &attr)
-            else
-              send(attr)
-            end
+  # Returns the users participating in a discussion.
+  #
+  # This method processes attributes of objects in breadth-first order.
+  #
+  # Returns an Array of User instances.
+  def participants(current_user = nil)
+    current_user ||= author
+    ext = Gitlab::ReferenceExtractor.new(project, current_user)
+    participants = Set.new
+    process = [self]
 
-          participants_for(value, current_user)
-        end.compact.uniq
-      end
+    until process.empty?
+      source = process.pop
 
-    unless Gitlab::ReferenceExtractor.lazy?
-      participants.select! do |user|
-        user.can?(:read_project, project)
+      case source
+      when User
+        participants << source
+      when Participable
+        source.class.participant_attrs.each do |attr|
+          if attr.respond_to?(:call)
+            source.instance_exec(current_user, ext, &attr)
+          else
+            process << source.__send__(attr)
+          end
+        end
+      when Enumerable, ActiveRecord::Relation
+        # This uses reverse_each so we can use "pop" to get the next value to
+        # process (in order). Using unshift instead of pop would require
+        # moving all Array values one index to the left (which can be
+        # expensive).
+        source.reverse_each { |obj| process << obj }
       end
     end
 
-    participants
-  end
-
-  private
+    participants.merge(ext.users)
 
-  def participants_for(value, current_user = nil)
-    case value
-    when User, Banzai::LazyReference
-      [value]
-    when Enumerable, ActiveRecord::Relation
-      value.flat_map { |v| participants_for(v, current_user) }
-    when Participable
-      value.participants(current_user)
-    end
+    Ability.users_that_can_read_project(participants.to_a, project)
   end
 end
diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb
index d5a881b2445..083257f1005 100644
--- a/app/models/concerns/subscribable.rb
+++ b/app/models/concerns/subscribable.rb
@@ -36,6 +36,12 @@ module Subscribable
       update(subscribed: !subscribed?(user))
   end
 
+  def subscribe(user)
+    subscriptions.
+      find_or_initialize_by(user_id: user.id).
+      update(subscribed: true)
+  end
+
   def unsubscribe(user)
     subscriptions.
       find_or_initialize_by(user_id: user.id).
diff --git a/app/models/event.rb b/app/models/event.rb
index 17ee48b91a8..716039fb54b 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -80,7 +80,7 @@ class Event < ActiveRecord::Base
   end
 
   def target_title
-    target.title if target && target.respond_to?(:title)
+    target.try(:title)
   end
 
   def created?
@@ -266,28 +266,20 @@ class Event < ActiveRecord::Base
     branch? && project.default_branch != branch_name
   end
 
-  def note_commit_id
-    target.commit_id
-  end
-
   def target_iid
     target.respond_to?(:iid) ? target.iid : target_id
   end
 
-  def note_short_commit_id
-    Commit.truncate_sha(note_commit_id)
-  end
-
-  def note_commit?
-    target.noteable_type == "Commit"
+  def commit_note?
+    target.for_commit?
   end
 
   def issue_note?
-    note? && target && target.noteable_type == "Issue"
+    note? && target && target.for_issue?
   end
 
-  def note_project_snippet?
-    target.noteable_type == "Snippet"
+  def project_snippet_note?
+    target.for_snippet?
   end
 
   def note_target
@@ -295,19 +287,22 @@ class Event < ActiveRecord::Base
   end
 
   def note_target_id
-    if note_commit?
+    if commit_note?
       target.commit_id
     else
       target.noteable_id.to_s
     end
   end
 
-  def note_target_iid
-    if note_target.respond_to?(:iid)
-      note_target.iid
+  def note_target_reference
+    return unless note_target
+
+    # Commit#to_reference returns the full SHA, but we want the short one here
+    if commit_note?
+      note_target.short_id
     else
-      note_target_id
-    end.to_s
+      note_target.to_reference
+    end
   end
 
   def note_target_type
diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb
index fde05f729dc..8b87b6c3d64 100644
--- a/app/models/hooks/web_hook.rb
+++ b/app/models/hooks/web_hook.rb
@@ -38,7 +38,7 @@ class WebHook < ActiveRecord::Base
                               basic_auth: auth)
     end
 
-    [(response.code >= 200 && response.code < 300), ActionView::Base.full_sanitizer.sanitize(response.to_s)]
+    [response.code, response.to_s]
   rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Net::OpenTimeout => e
     logger.error("WebHook Error => #{e}")
     [false, e.to_s]
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 2d4a9b9f19a..bd0fbc96d18 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -95,14 +95,13 @@ class Issue < ActiveRecord::Base
   end
 
   def referenced_merge_requests(current_user = nil)
-    @referenced_merge_requests ||= {}
-    @referenced_merge_requests[current_user] ||= begin
-      Gitlab::ReferenceExtractor.lazily do
-        [self, *notes].flat_map do |note|
-          note.all_references(current_user).merge_requests
-        end
-      end.sort_by(&:iid).uniq
+    ext = all_references(current_user)
+
+    notes_with_associations.each do |object|
+      object.all_references(current_user, extractor: ext)
     end
+
+    ext.merge_requests.sort_by(&:iid)
   end
 
   # All branches containing the current issue's ID, except for
@@ -139,9 +138,13 @@ class Issue < ActiveRecord::Base
   def closed_by_merge_requests(current_user = nil)
     return [] unless open?
 
-    notes.system.flat_map do |note|
-      note.all_references(current_user).merge_requests
-    end.uniq.select { |mr| mr.open? && mr.closes_issue?(self) }
+    ext = all_references(current_user)
+
+    notes.system.each do |note|
+      note.all_references(current_user, extractor: ext)
+    end
+
+    ext.merge_requests.select { |mr| mr.open? && mr.closes_issue?(self) }
   end
 
   def moved?
diff --git a/app/models/key.rb b/app/models/key.rb
index d52afda67d1..0532e84f47d 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -26,7 +26,7 @@ class Key < ActiveRecord::Base
   end
 
   def publishable_key
-    #Removes anything beyond the keytype and key itself
+    # Removes anything beyond the keytype and key itself
     self.key.split[0..1].join(' ')
   end
 
diff --git a/app/models/legacy_diff_note.rb b/app/models/legacy_diff_note.rb
new file mode 100644
index 00000000000..bbefc911b29
--- /dev/null
+++ b/app/models/legacy_diff_note.rb
@@ -0,0 +1,157 @@
+class LegacyDiffNote < Note
+  serialize :st_diff
+
+  validates :line_code, presence: true, line_code: true
+
+  before_create :set_diff
+
+  class << self
+    def build_discussion_id(noteable_type, noteable_id, line_code, active = true)
+      [super(noteable_type, noteable_id), line_code, active].join("-")
+    end
+  end
+
+  def diff_note?
+    true
+  end
+
+  def legacy_diff_note?
+    true
+  end
+
+  def discussion_id
+    @discussion_id ||= self.class.build_discussion_id(noteable_type, noteable_id || commit_id, line_code, active?)
+  end
+
+  def diff_file_hash
+    line_code.split('_')[0] if line_code
+  end
+
+  def diff_old_line
+    line_code.split('_')[1].to_i if line_code
+  end
+
+  def diff_new_line
+    line_code.split('_')[2].to_i if line_code
+  end
+
+  def diff
+    @diff ||= Gitlab::Git::Diff.new(st_diff) if st_diff.respond_to?(:map)
+  end
+
+  def diff_file_path
+    diff.new_path.presence || diff.old_path
+  end
+
+  def diff_lines
+    @diff_lines ||= Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
+  end
+
+  def diff_line
+    @diff_line ||= diff_lines.find { |line| generate_line_code(line) == self.line_code }
+  end
+
+  def diff_line_text
+    diff_line.try(:text)
+  end
+
+  def diff_line_type
+    diff_line.try(:type)
+  end
+
+  def highlighted_diff_lines
+    Gitlab::Diff::Highlight.new(diff_lines).highlight
+  end
+
+  def truncated_diff_lines
+    max_number_of_lines = 16
+    prev_match_line = nil
+    prev_lines = []
+
+    highlighted_diff_lines.each do |line|
+      if line.type == "match"
+        prev_lines.clear
+        prev_match_line = line
+      else
+        prev_lines << line
+
+        break if generate_line_code(line) == self.line_code
+
+        prev_lines.shift if prev_lines.length >= max_number_of_lines
+      end
+    end
+
+    prev_lines
+  end
+
+  # Check if this note is part of an "active" discussion
+  #
+  # This will always return true for anything except MergeRequest noteables,
+  # which have special logic.
+  #
+  # If the note's current diff cannot be matched in the MergeRequest's current
+  # diff, it's considered inactive.
+  def active?
+    return @active if defined?(@active)
+    return true if for_commit?
+    return true unless self.diff
+    return false unless noteable
+
+    noteable_diff = find_noteable_diff
+
+    if noteable_diff
+      parsed_lines = Gitlab::Diff::Parser.new.parse(noteable_diff.diff.each_line)
+
+      @active = parsed_lines.any? { |line_obj| line_obj.text == diff_line_text }
+    else
+      @active = false
+    end
+
+    @active
+  end
+
+  private
+
+  def find_diff
+    return nil unless noteable
+    return @diff if defined?(@diff)
+
+    @diff = noteable.diffs(Commit.max_diff_options).find do |d|
+      d.new_path && Digest::SHA1.hexdigest(d.new_path) == diff_file_hash
+    end
+  end
+
+  def set_diff
+    # First lets find notes with same diff
+    # before iterating over all mr diffs
+    diff = diff_for_line_code unless for_merge_request?
+    diff ||= find_diff
+
+    self.st_diff = diff.to_hash if diff
+  end
+
+  def diff_for_line_code
+    attributes = {
+      noteable_type: noteable_type,
+      line_code: line_code
+    }
+
+    if for_commit?
+      attributes[:commit_id] = commit_id
+    else
+      attributes[:noteable_id] = noteable_id
+    end
+
+    self.class.where(attributes).last.try(:diff)
+  end
+
+  def generate_line_code(line)
+    Gitlab::Diff::LineCode.generate(diff_file_path, line.new_pos, line.old_pos)
+  end
+
+  # Find the diff on noteable that matches our own
+  def find_noteable_diff
+    diffs = noteable.diffs(Commit.max_diff_options)
+    diffs.find { |d| d.new_path == self.diff.new_path }
+  end
+end
diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb
index 8dae3bb2ef2..46955b430f3 100644
--- a/app/models/members/project_member.rb
+++ b/app/models/members/project_member.rb
@@ -5,7 +5,6 @@ class ProjectMember < Member
 
   belongs_to :project, class_name: 'Project', foreign_key: 'source_id'
 
-
   # Make sure project member points only to project as it source
   default_value_for :source_type, SOURCE_TYPE
   validates_format_of :source_type, with: /\AProject\z/
@@ -15,6 +14,8 @@ class ProjectMember < Member
   scope :in_projects, ->(projects) { where(source_id: projects.pluck(:id)) }
   scope :with_user, ->(user) { where(user_id: user.id) }
 
+  before_destroy :delete_member_todos
+
   class << self
 
     # Add users to project teams with passed access option
@@ -102,6 +103,10 @@ class ProjectMember < Member
 
   private
 
+  def delete_member_todos
+    user.todos.where(project_id: source_id).destroy_all if user
+  end
+
   def send_invite
     notification_service.invite_project_member(self, @raw_invite_token)
 
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 5460935366c..722c258244c 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -26,6 +26,10 @@ class MergeRequest < ActiveRecord::Base
   # when creating new merge request
   attr_accessor :can_be_created, :compare_commits, :compare
 
+  # Temporary fields to store target_sha, and base_sha to
+  # compare when importing pull requests from GitHub
+  attr_accessor :base_target_sha, :head_source_sha
+
   state_machine :state, initial: :opened do
     event :close do
       transition [:reopened, :opened] => :closed
@@ -282,6 +286,18 @@ class MergeRequest < ActiveRecord::Base
       last_commit == source_project.commit(source_branch)
   end
 
+  def should_remove_source_branch?
+    merge_params['should_remove_source_branch'].present?
+  end
+
+  def force_remove_source_branch?
+    merge_params['force_remove_source_branch'].present?
+  end
+
+  def remove_source_branch?
+    should_remove_source_branch? || force_remove_source_branch?
+  end
+
   def mr_and_commit_notes
     # Fetch comments only from last 100 commits
     commits_for_notes_limit = 100
@@ -422,7 +438,10 @@ class MergeRequest < ActiveRecord::Base
 
     self.merge_when_build_succeeds = false
     self.merge_user = nil
-    self.merge_params = nil
+    if merge_params
+      merge_params.delete('should_remove_source_branch')
+      merge_params.delete('commit_message')
+    end
 
     self.save
   end
@@ -490,10 +509,14 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def target_sha
-    @target_sha ||= target_project.repository.commit(target_branch).try(:sha)
+    return @base_target_sha if defined?(@base_target_sha)
+
+    target_project.repository.commit(target_branch).try(:sha)
   end
 
   def source_sha
+    return @head_source_sha if defined?(@head_source_sha)
+
     last_commit.try(:sha) || source_tip.try(:sha)
   end
 
@@ -514,7 +537,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def ref_is_fetched?
-    File.exists?(File.join(project.repository.path_to_repo, ref_path))
+    File.exist?(File.join(project.repository.path_to_repo, ref_path))
   end
 
   def ensure_ref_fetched
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index eb42c07b9b9..7d5103748f5 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -6,7 +6,7 @@ class MergeRequestDiff < ActiveRecord::Base
 
   belongs_to :merge_request
 
-  delegate :target_branch, :source_branch, to: :merge_request, prefix: nil
+  delegate :head_source_sha, :target_branch, :source_branch, to: :merge_request, prefix: nil
 
   state_machine :state, initial: :empty do
     state :collected
@@ -38,8 +38,8 @@ class MergeRequestDiff < ActiveRecord::Base
       @diffs_no_whitespace ||= begin
         compare = Gitlab::Git::Compare.new(
           self.repository.raw_repository,
-          self.target_branch,
-          self.source_sha,
+          self.base,
+          self.head,
         )
         compare.diffs(options)
       end
@@ -98,9 +98,7 @@ class MergeRequestDiff < ActiveRecord::Base
     commits = compare.commits
 
     if commits.present?
-      commits = Commit.decorate(commits, merge_request.source_project).
-        sort_by(&:created_at).
-        reverse
+      commits = Commit.decorate(commits, merge_request.source_project).reverse
     end
 
     commits
@@ -144,7 +142,7 @@ class MergeRequestDiff < ActiveRecord::Base
 
     self.st_diffs = new_diffs
 
-    self.base_commit_sha = self.repository.merge_base(self.source_sha, self.target_branch)
+    self.base_commit_sha = self.repository.merge_base(self.head, self.base)
 
     self.save
   end
@@ -160,10 +158,24 @@ class MergeRequestDiff < ActiveRecord::Base
   end
 
   def source_sha
+    return head_source_sha if head_source_sha.present?
+
     source_commit = merge_request.source_project.commit(source_branch)
     source_commit.try(:sha)
   end
 
+  def target_sha
+    merge_request.target_sha
+  end
+
+  def base
+    self.target_sha || self.target_branch
+  end
+
+  def head
+    self.source_sha
+  end
+
   def compare
     @compare ||=
       begin
@@ -172,8 +184,8 @@ class MergeRequestDiff < ActiveRecord::Base
 
         Gitlab::Git::Compare.new(
           self.repository.raw_repository,
-          self.target_branch,
-          self.source_sha
+          self.base,
+          self.head
         )
       end
   end
diff --git a/app/models/milestone.rb b/app/models/milestone.rb
index e4fdd23badb..e0c8454a998 100644
--- a/app/models/milestone.rb
+++ b/app/models/milestone.rb
@@ -59,25 +59,67 @@ class Milestone < ActiveRecord::Base
     end
   end
 
+  def self.reference_prefix
+    '%'
+  end
+
   def self.reference_pattern
-    nil
+    # NOTE: The iid pattern only matches when all characters on the expression
+    # are digits, so it will match %2 but not %2.1 because that's probably a
+    # milestone name and we want it to be matched as such.
+    @reference_pattern ||= %r{
+      (#{Project.reference_pattern})?
+      #{Regexp.escape(reference_prefix)}
+      (?:
+        (?<milestone_iid>
+          \d+(?!\S\w)\b # Integer-based milestone iid, or
+        ) |
+        (?<milestone_name>
+          [^"\s]+\b |  # String-based single-word milestone title, or
+          "[^"]+"      # String-based multi-word milestone surrounded in quotes
+        )
+      )
+    }x
   end
 
   def self.link_reference_pattern
     @link_reference_pattern ||= super("milestones", /(?<milestone>\d+)/)
   end
 
-  def self.upcoming
-    self.where('due_date > ?', Time.now).reorder(due_date: :asc).first
-  end
+  def self.upcoming_ids_by_projects(projects)
+    rel = unscoped.of_projects(projects).active.where('due_date > ?', Time.now)
 
-  def to_reference(from_project = nil)
-    escaped_title = self.title.gsub("]", "\\]")
+    if Gitlab::Database.postgresql?
+      rel.order(:project_id, :due_date).select('DISTINCT ON (project_id) id')
+    else
+      rel.
+        group(:project_id).
+        having('due_date = MIN(due_date)').
+        pluck(:id, :project_id, :due_date).
+        map(&:first)
+    end
+  end
 
-    h = Gitlab::Routing.url_helpers
-    url = h.namespace_project_milestone_url(self.project.namespace, self.project, self)
+  ##
+  # Returns the String necessary to reference this Milestone in Markdown
+  #
+  # format - Symbol format to use (default: :iid, optional: :name)
+  #
+  # Examples:
+  #
+  #   Milestone.first.to_reference                # => "%1"
+  #   Milestone.first.to_reference(format: :name) # => "%\"goal\""
+  #   Milestone.first.to_reference(project)       # => "gitlab-org/gitlab-ce%1"
+  #
+  def to_reference(from_project = nil, format: :iid)
+    format_reference = milestone_format_reference(format)
+    reference = "#{self.class.reference_prefix}#{format_reference}"
 
-    "[#{escaped_title}](#{url})"
+    if cross_project_reference?(from_project)
+      project.to_reference + reference
+    else
+      reference
+    end
   end
 
   def reference_link_text(from_project = nil)
@@ -149,4 +191,16 @@ class Milestone < ActiveRecord::Base
     issues.where(id: ids).
       update_all(["position = CASE #{conditions} ELSE position END", *pairs])
   end
+
+  private
+
+  def milestone_format_reference(format = :iid)
+    raise ArgumentError, 'Unknown format' unless [:iid, :name].include?(format)
+
+    if format == :name && !name.include?('"')
+      %("#{name}")
+    else
+      iid
+    end
+  end
 end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 9c942a8f4e3..da19462f265 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -110,6 +110,10 @@ class Namespace < ActiveRecord::Base
     # Ensure old directory exists before moving it
     gitlab_shell.add_namespace(path_was)
 
+    if any_project_has_container_registry_tags?
+      raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
+    end
+
     if gitlab_shell.mv_namespace(path_was, path)
       Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
 
@@ -131,6 +135,10 @@ class Namespace < ActiveRecord::Base
     end
   end
 
+  def any_project_has_container_registry_tags?
+    projects.any?(&:has_container_registry_tags?)
+  end
+
   def send_update_instructions
     projects.each do |project|
       project.send_move_instructions("#{path_was}/#{project.path}")
diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb
index f4e90125373..a2aee2f925b 100644
--- a/app/models/network/graph.rb
+++ b/app/models/network/graph.rb
@@ -22,9 +22,16 @@ module Network
 
     def collect_notes
       h = Hash.new(0)
-      @project.notes.where('noteable_type = ?' ,"Commit").group('notes.commit_id').select('notes.commit_id, count(notes.id) as note_count').each do |item|
-        h[item.commit_id] = item.note_count.to_i
-      end
+
+      @project
+        .notes
+        .where('noteable_type = ?', 'Commit')
+        .group('notes.commit_id')
+        .select('notes.commit_id, count(notes.id) as note_count')
+        .each do |item|
+          h[item.commit_id] = item.note_count.to_i
+        end
+
       h
     end
 
@@ -89,7 +96,7 @@ module Network
         end
       end
 
-      if self.class.max_count / 2 < offset then
+      if self.class.max_count / 2 < offset
         # get max index that commit is displayed in the center.
         offset - self.class.max_count / 2
       else
@@ -130,7 +137,7 @@ module Network
       commit.parents(@map).each do |parent|
         range = commit.time..parent.time
 
-        space = if commit.space >= parent.space then
+        space = if commit.space >= parent.space
                   find_free_parent_space(range, parent.space, -1, commit.space)
                 else
                   find_free_parent_space(range, commit.space, -1, parent.space)
@@ -144,7 +151,7 @@ module Network
     end
 
     def find_free_parent_space(range, space_base, space_step, space_default)
-      if is_overlap?(range, space_default) then
+      if is_overlap?(range, space_default)
         find_free_space(range, space_step, space_base, space_default)
       else
         space_default
@@ -155,9 +162,9 @@ module Network
       range.each do |i|
         if i != range.first &&
           i != range.last &&
-          @commits[i].spaces.include?(overlap_space) then
+          @commits[i].spaces.include?(overlap_space)
 
-          return true;
+          return true
         end
       end
 
@@ -198,7 +205,7 @@ module Network
       # Visit branching chains
       leaves.each do |l|
         parents = l.parents(@map).select{|p| p.space.zero?}
-        for p in parents
+        parents.each do |p|
           place_chain(p, l.time)
         end
       end
@@ -216,7 +223,7 @@ module Network
     end
 
     def mark_reserved(time_range, space)
-      for day in time_range
+      time_range.each do |day|
         @reserved[day].push(space)
       end
     end
@@ -225,15 +232,15 @@ module Network
       space_default ||= space_base
 
       reserved = []
-      for day in time_range
+      time_range.each do |day|
         reserved.push(*@reserved[day])
       end
       reserved.uniq!
 
       space = space_default
-      while reserved.include?(space) do
+      while reserved.include?(space)
         space += space_step
-        if space < space_base then
+        if space < space_base
           space_step *= -1
           space = space_base + space_step
         end
@@ -253,7 +260,7 @@ module Network
       leaves = []
       leaves.push(commit) if commit.space.zero?
 
-      while true
+      loop do
         return leaves if commit.parents(@map).count.zero?
 
         commit = commit.parents(@map).first
diff --git a/app/models/note.rb b/app/models/note.rb
index f26aa1bf63f..c21981ead84 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -1,13 +1,12 @@
-require 'carrierwave/orm/activerecord'
-
 class Note < ActiveRecord::Base
+  extend ActiveModel::Naming
   include Gitlab::CurrentSettings
   include Participable
   include Mentionable
 
   default_value_for :system, false
 
-  attr_mentionable :note, cache: true, pipeline: :note
+  attr_mentionable :note, pipeline: :note
   participant :author
 
   belongs_to :project
@@ -20,29 +19,33 @@ class Note < ActiveRecord::Base
   delegate :gfm_reference, :local_reference, to: :noteable
   delegate :name, to: :project, prefix: true
   delegate :name, :email, to: :author, prefix: true
+  delegate :title, to: :noteable, allow_nil: true
 
   before_validation :set_award!
-  before_validation :clear_blank_line_code!
 
   validates :note, :project, presence: true
   validates :note, uniqueness: { scope: [:author, :noteable_type, :noteable_id] }, if: ->(n) { n.is_award }
   validates :note, inclusion: { in: Emoji.emojis_names }, if: ->(n) { n.is_award }
-  validates :line_code, line_code: true, allow_blank: true
   # Attachments are deprecated and are handled by Markdown uploader
   validates :attachment, file_size: { maximum: :max_attachment_size }
 
-  validates :noteable_id, presence: true, if: ->(n) { n.noteable_type.present? && n.noteable_type != 'Commit' }
-  validates :commit_id, presence: true, if: ->(n) { n.noteable_type == 'Commit' }
+  validates :noteable_type, presence: true
+  validates :noteable_id, presence: true, unless: :for_commit?
+  validates :commit_id, presence: true, if: :for_commit?
   validates :author, presence: true
 
+  validate unless: :for_commit? do |note|
+    unless note.noteable.try(:project) == note.project
+      errors.add(:invalid_project, 'Note and noteable project mismatch')
+    end
+  end
+
   mount_uploader :attachment, AttachmentUploader
 
   # Scopes
   scope :awards, ->{ where(is_award: true) }
   scope :nonawards, ->{ where(is_award: false) }
   scope :for_commit_id, ->(commit_id) { where(noteable_type: "Commit", commit_id: commit_id) }
-  scope :inline, ->{ where("line_code IS NOT NULL") }
-  scope :not_inline, ->{ where(line_code: nil) }
   scope :system, ->{ where(system: true) }
   scope :user, ->{ where(system: false) }
   scope :common, ->{ where(noteable_type: ["", nil]) }
@@ -50,52 +53,61 @@ class Note < ActiveRecord::Base
   scope :inc_author_project, ->{ includes(:project, :author) }
   scope :inc_author, ->{ includes(:author) }
 
+  scope :legacy_diff_notes, ->{ where(type: 'LegacyDiffNote') }
+  scope :non_diff_notes, ->{ where(type: ['Note', nil]) }
+
   scope :with_associations, -> do
     includes(:author, :noteable, :updated_by,
              project: [:project_members, { group: [:group_members] }])
   end
 
-  serialize :st_diff
-  before_create :set_diff, if: ->(n) { n.line_code.present? }
+  before_validation :clear_blank_line_code!
 
   class << self
-    def discussions_from_notes(notes)
-      discussion_ids = []
-      discussions = []
-
-      notes.each do |note|
-        next if discussion_ids.include?(note.discussion_id)
-
-        # don't group notes for the main target
-        if !note.for_diff_line? && note.for_merge_request?
-          discussions << [note]
-        else
-          discussions << notes.select do |other_note|
-            note.discussion_id == other_note.discussion_id
-          end
-          discussion_ids << note.discussion_id
-        end
-      end
+    def model_name
+      ActiveModel::Name.new(self, nil, 'note')
+    end
+
+    def build_discussion_id(noteable_type, noteable_id)
+      [:discussion, noteable_type.try(:underscore), noteable_id].join("-")
+    end
 
-      discussions
+    def discussions
+      all.group_by(&:discussion_id).values
     end
 
-    def build_discussion_id(type, id, line_code)
-      [:discussion, type.try(:underscore), id, line_code].join("-").to_sym
+    def grouped_diff_notes
+      legacy_diff_notes.select(&:active?).sort_by(&:created_at).group_by(&:line_code)
     end
 
     # Searches for notes matching the given query.
     #
     # This method uses ILIKE on PostgreSQL and LIKE on MySQL.
     #
-    # query - The search query as a String.
+    # query   - The search query as a String.
+    # as_user - Limit results to those viewable by a specific user
     #
     # Returns an ActiveRecord::Relation.
-    def search(query)
+    def search(query, as_user: nil)
       table   = arel_table
       pattern = "%#{query}%"
 
-      where(table[:note].matches(pattern))
+      found_notes = joins('LEFT JOIN issues ON issues.id = noteable_id').
+        where(table[:note].matches(pattern))
+
+      if as_user
+        found_notes.where('
+          issues.confidential IS NULL
+          OR issues.confidential IS FALSE
+          OR (issues.confidential IS TRUE
+            AND (issues.author_id = :user_id
+            OR issues.assignee_id = :user_id
+            OR issues.project_id IN(:project_ids)))',
+          user_id: as_user.id,
+          project_ids: as_user.authorized_projects.select(:id))
+      else
+        found_notes.where('issues.confidential IS NULL OR issues.confidential IS FALSE')
+      end
     end
 
     def grouped_awards
@@ -116,167 +128,39 @@ class Note < ActiveRecord::Base
     system && SystemNoteService.cross_reference?(note)
   end
 
-  def max_attachment_size
-    current_application_settings.max_attachment_size.megabytes.to_i
-  end
-
-  def find_diff
-    return nil unless noteable
-    return @diff if defined?(@diff)
-
-    # Don't use ||= because nil is a valid value for @diff
-    @diff = noteable.diffs(Commit.max_diff_options).find do |d|
-      Digest::SHA1.hexdigest(d.new_path) == diff_file_index if d.new_path
-    end
+  def diff_note?
+    false
   end
 
-  def hook_attrs
-    attributes
+  def legacy_diff_note?
+    false
   end
 
-  def set_diff
-    # First lets find notes with same diff
-    # before iterating over all mr diffs
-    diff = diff_for_line_code unless for_merge_request?
-    diff ||= find_diff
-
-    self.st_diff = diff.to_hash if diff
-  end
-
-  def diff
-    @diff ||= Gitlab::Git::Diff.new(st_diff) if st_diff.respond_to?(:map)
-  end
-
-  def diff_for_line_code
-    Note.where(noteable_id: noteable_id, noteable_type: noteable_type, line_code: line_code).last.try(:diff)
-  end
-
-  # Check if this note is part of an "active" discussion
-  #
-  # This will always return true for anything except MergeRequest noteables,
-  # which have special logic.
-  #
-  # If the note's current diff cannot be matched in the MergeRequest's current
-  # diff, it's considered inactive.
   def active?
-    return true unless self.diff
-    return false unless noteable
-    return @active if defined?(@active)
-
-    noteable_diff = find_noteable_diff
-
-    if noteable_diff
-      parsed_lines = Gitlab::Diff::Parser.new.parse(noteable_diff.diff.each_line)
-
-      @active = parsed_lines.any? { |line_obj| line_obj.text == diff_line }
-    else
-      @active = false
-    end
-
-    @active
-  end
-
-  def diff_file_index
-    line_code.split('_')[0] if line_code
-  end
-
-  def diff_file_name
-    diff.new_path if diff
-  end
-
-  def file_path
-    if diff.new_path.present?
-      diff.new_path
-    elsif diff.old_path.present?
-      diff.old_path
-    end
-  end
-
-  def diff_old_line
-    line_code.split('_')[1].to_i if line_code
-  end
-
-  def diff_new_line
-    line_code.split('_')[2].to_i if line_code
+    true
   end
 
-  def generate_line_code(line)
-    Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos)
-  end
-
-  def diff_line
-    return @diff_line if @diff_line
-
-    if diff
-      diff_lines.each do |line|
-        if generate_line_code(line) == self.line_code
-          @diff_line = line.text
-        end
-      end
-    end
-
-    @diff_line
-  end
-
-  def diff_line_type
-    return @diff_line_type if @diff_line_type
-
-    if diff
-      diff_lines.each do |line|
-        if generate_line_code(line) == self.line_code
-          @diff_line_type = line.type
-        end
-      end
-    end
-
-    @diff_line_type
-  end
-
-  def truncated_diff_lines
-    max_number_of_lines = 16
-    prev_match_line = nil
-    prev_lines = []
-
-    highlighted_diff_lines.each do |line|
-      if line.type == "match"
-        prev_lines.clear
-        prev_match_line = line
+  def discussion_id
+    @discussion_id ||=
+      if for_merge_request?
+        [:discussion, :note, id].join("-")
       else
-        prev_lines << line
-
-        break if generate_line_code(line) == self.line_code
-
-        prev_lines.shift if prev_lines.length >= max_number_of_lines
+        self.class.build_discussion_id(noteable_type, noteable_id || commit_id)
       end
-    end
-
-    prev_lines
   end
 
-  def diff_lines
-    @diff_lines ||= Gitlab::Diff::Parser.new.parse(diff.diff.each_line)
-  end
-
-  def highlighted_diff_lines
-    Gitlab::Diff::Highlight.new(diff_lines).highlight
+  def max_attachment_size
+    current_application_settings.max_attachment_size.megabytes.to_i
   end
 
-  def discussion_id
-    @discussion_id ||= Note.build_discussion_id(noteable_type, noteable_id || commit_id, line_code)
+  def hook_attrs
+    attributes
   end
 
   def for_commit?
     noteable_type == "Commit"
   end
 
-  def for_commit_diff_line?
-    for_commit? && for_diff_line?
-  end
-
-  def for_diff_line?
-    line_code.present?
-  end
-
   def for_issue?
     noteable_type == "Issue"
   end
@@ -285,10 +169,6 @@ class Note < ActiveRecord::Base
     noteable_type == "MergeRequest"
   end
 
-  def for_merge_request_diff_line?
-    for_merge_request? && for_diff_line?
-  end
-
   def for_snippet?
     noteable_type == "Snippet"
   end
@@ -361,14 +241,8 @@ class Note < ActiveRecord::Base
     self.line_code = nil if self.line_code.blank?
   end
 
-  # Find the diff on noteable that matches our own
-  def find_noteable_diff
-    diffs = noteable.diffs(Commit.max_diff_options)
-    diffs.find { |d| d.new_path == self.diff.new_path }
-  end
-
   def awards_supported?
-    (for_issue? || for_merge_request?) && !for_diff_line?
+    (for_issue? || for_merge_request?) && !diff_note?
   end
 
   def contains_emoji_only?
diff --git a/app/models/project.rb b/app/models/project.rb
index 418b85e028a..e4a9d17a20c 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -22,6 +22,7 @@ class Project < ActiveRecord::Base
   default_value_for :builds_enabled, gitlab_config_features.builds
   default_value_for :wiki_enabled, gitlab_config_features.wiki
   default_value_for :snippets_enabled, gitlab_config_features.snippets
+  default_value_for :container_registry_enabled, gitlab_config_features.container_registry
   default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
 
   # set last_activity_at to the same as created_at
@@ -49,6 +50,8 @@ class Project < ActiveRecord::Base
   attr_accessor :new_default_branch
   attr_accessor :old_path_with_namespace
 
+  alias_attribute :title, :name
+
   # Relations
   belongs_to :creator, foreign_key: 'creator_id', class_name: 'User'
   belongs_to :group, -> { where(type: Group) }, foreign_key: 'namespace_id'
@@ -168,17 +171,17 @@ class Project < ActiveRecord::Base
 
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
-  scope :sorted_by_names, -> { joins(:namespace).reorder('namespaces.name ASC, projects.name ASC') }
 
-  scope :without_user, ->(user)  { where('projects.id NOT IN (:ids)', ids: user.authorized_projects.map(&:id) ) }
-  scope :without_team, ->(team) { team.projects.present? ? where('projects.id NOT IN (:ids)', ids: team.projects.map(&:id)) : scoped  }
-  scope :not_in_group, ->(group) { where('projects.id NOT IN (:ids)', ids: group.project_ids ) }
   scope :in_namespace, ->(namespace_ids) { where(namespace_id: namespace_ids) }
-  scope :in_group_namespace, -> { joins(:group) }
   scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
   scope :joined, ->(user) { where('namespace_id != ?', user.namespace_id) }
+  scope :visible_to_user, ->(user) { where(id: user.authorized_projects.select(:id).reorder(nil)) }
   scope :non_archived, -> { where(archived: false) }
   scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct }
+  scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
+
+  scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') }
+  scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) }
 
   state_machine :import_status, initial: :none do
     event :import_start do
@@ -201,23 +204,10 @@ class Project < ActiveRecord::Base
     state :finished
     state :failed
 
-    after_transition any => :started, do: :schedule_add_import_job
-    after_transition any => :finished, do: :clear_import_data
+    after_transition any => :finished, do: :reset_cache_and_import_attrs
   end
 
   class << self
-    def abandoned
-      where('projects.last_activity_at < ?', 6.months.ago)
-    end
-
-    def with_push
-      joins(:events).where('events.action = ?', Event::PUSHED)
-    end
-
-    def active
-      joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC')
-    end
-
     # Searches for a list of projects based on the query given in `query`.
     #
     # On PostgreSQL this method uses "ILIKE" to perform a case-insensitive
@@ -279,10 +269,6 @@ class Project < ActiveRecord::Base
         projects.iwhere('projects.path' => project_path).take
     end
 
-    def find_by_ci_id(id)
-      find_by(ci_id: id.to_i)
-    end
-
     def visibility_levels
       Gitlab::VisibilityLevel.options
     end
@@ -313,10 +299,6 @@ class Project < ActiveRecord::Base
 
       joins(join_body).reorder('join_note_counts.amount DESC')
     end
-
-    def visible_to_user(user)
-      where(id: user.authorized_projects.select(:id).reorder(nil))
-    end
   end
 
   def team
@@ -327,6 +309,34 @@ class Project < ActiveRecord::Base
     @repository ||= Repository.new(path_with_namespace, self)
   end
 
+  def container_registry_path_with_namespace
+    path_with_namespace.downcase
+  end
+
+  def container_registry_repository
+    return unless Gitlab.config.registry.enabled
+
+    @container_registry_repository ||= begin
+      token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_path_with_namespace)
+      url = Gitlab.config.registry.api_url
+      host_port = Gitlab.config.registry.host_port
+      registry = ContainerRegistry::Registry.new(url, token: token, path: host_port)
+      registry.repository(container_registry_path_with_namespace)
+    end
+  end
+
+  def container_registry_repository_url
+    if Gitlab.config.registry.enabled
+      "#{Gitlab.config.registry.host_port}/#{container_registry_path_with_namespace}"
+    end
+  end
+
+  def has_container_registry_tags?
+    return unless container_registry_repository
+
+    container_registry_repository.tags.any?
+  end
+
   def commit(id = 'HEAD')
     repository.commit(id)
   end
@@ -340,10 +350,6 @@ class Project < ActiveRecord::Base
     id && persisted?
   end
 
-  def schedule_add_import_job
-    run_after_commit(:add_import_job)
-  end
-
   def add_import_job
     if forked?
       job_id = RepositoryForkWorker.perform_async(self.id, forked_from_project.path_with_namespace, self.namespace.path)
@@ -358,7 +364,7 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def clear_import_data
+  def reset_cache_and_import_attrs
     update(import_error: nil)
 
     ProjectCacheWorker.perform_async(self.id)
@@ -367,14 +373,14 @@ class Project < ActiveRecord::Base
   end
 
   def import_url=(value)
-    import_url = Gitlab::ImportUrl.new(value)
+    import_url = Gitlab::UrlSanitizer.new(value)
     create_or_update_import_data(credentials: import_url.credentials)
     super(import_url.sanitized_url)
   end
 
   def import_url
     if import_data && super
-      import_url = Gitlab::ImportUrl.new(super, credentials: import_data.credentials)
+      import_url = Gitlab::UrlSanitizer.new(super, credentials: import_data.credentials)
       import_url.full_url
     else
       super
@@ -424,17 +430,18 @@ class Project < ActiveRecord::Base
   end
 
   def safe_import_url
-    result = URI.parse(self.import_url)
-    result.password = '*****' unless result.password.nil?
-    result.user = '*****' unless result.user.nil? || result.user == "git" #tokens or other data may be saved as user
-    result.to_s
-  rescue
-    self.import_url
+    Gitlab::UrlSanitizer.new(import_url).masked_url
   end
 
   def check_limit
     unless creator.can_create_project? or namespace.kind == 'group'
-      self.errors.add(:limit_reached, "Your project limit is #{creator.projects_limit} projects! Please contact your administrator to increase it")
+      projects_limit = creator.projects_limit
+
+      if projects_limit == 0
+        self.errors.add(:limit_reached, "Personal project creation is not allowed. Please contact your administrator with questions")
+      else
+        self.errors.add(:limit_reached, "Your project limit is #{projects_limit} projects! Please contact your administrator to increase it")
+      end
     end
   rescue
     self.errors.add(:base, "Can't check your ability to create project")
@@ -742,6 +749,11 @@ class Project < ActiveRecord::Base
 
     expire_caches_before_rename(old_path_with_namespace)
 
+    if has_container_registry_tags?
+      # we currently doesn't support renaming repository if it contains tags in container registry
+      raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
+    end
+
     if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
       # If repository moved successfully we need to send update instructions to users.
       # However we cannot allow rollback since we moved repository
@@ -938,13 +950,13 @@ class Project < ActiveRecord::Base
     shared_runners_enabled? && Ci::Runner.shared.active.any?(&block)
   end
 
-  def valid_runners_token? token
+  def valid_runners_token?(token)
     self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token)
   end
 
   # TODO (ayufan): For now we use runners_token (backward compatibility)
   # In 8.4 every build will have its own individual token valid for time of build
-  def valid_build_token? token
+  def valid_build_token?(token)
     self.builds_enabled? && self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token)
   end
 
@@ -999,4 +1011,22 @@ class Project < ActiveRecord::Base
 
     update_attribute(:pending_delete, true)
   end
+
+  def running_or_pending_build_count(force: false)
+    Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
+      builds.running_or_pending.count(:all)
+    end
+  end
+
+  def mark_import_as_failed(error_message)
+    original_errors = errors.dup
+    sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message)
+
+    import_fail
+    update_column(:import_error, sanitized_message)
+  rescue ActiveRecord::ActiveRecordError => e
+    Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}")
+  ensure
+    @errors = original_errors
+  end
 end
diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb
index e2f9ffb69ac..ca8a9b4217b 100644
--- a/app/models/project_import_data.rb
+++ b/app/models/project_import_data.rb
@@ -6,7 +6,8 @@ class ProjectImportData < ActiveRecord::Base
                  key: Gitlab::Application.secrets.db_key_base,
                  marshal: true,
                  encode: true,
-                 mode: :per_attribute_iv_and_salt
+                 mode: :per_attribute_iv_and_salt,
+                 algorithm: 'aes-256-cbc'
 
   serialize :data, JSON
 
diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb
index f042bfc5210..d7b6e505191 100644
--- a/app/models/project_services/external_wiki_service.rb
+++ b/app/models/project_services/external_wiki_service.rb
@@ -25,7 +25,7 @@ class ExternalWikiService < Service
 
   def execute(_data)
     @response = HTTParty.get(properties['external_wiki_url'], verify: true) rescue nil
-    if @response !=200
+    if @response != 200
       nil
     end
   end
diff --git a/app/models/project_services/irker_service.rb b/app/models/project_services/irker_service.rb
index 91015e6c9b1..2e5e854fc5e 100644
--- a/app/models/project_services/irker_service.rb
+++ b/app/models/project_services/irker_service.rb
@@ -70,7 +70,7 @@ class IrkerService < Service
   private
 
   def get_channels
-    return true unless :activated?
+    return true unless activated?
     return true if recipients.nil? || recipients.empty?
 
     map_recipients
diff --git a/app/models/project_services/slack_service/build_message.rb b/app/models/project_services/slack_service/build_message.rb
index c124cad4afd..69c21b3fc38 100644
--- a/app/models/project_services/slack_service/build_message.rb
+++ b/app/models/project_services/slack_service/build_message.rb
@@ -35,8 +35,8 @@ class SlackService
     private
 
     def message
-      "#{project_link}: Commit #{commit_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} second(s)"
-    end
+      "#{project_link}: Commit #{commit_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{'second'.pluralize(duration)}"
+    end   
 
     def format(string)
       Slack::Notifier::LinkFormatter.format(string)
diff --git a/app/models/project_services/slack_service/issue_message.rb b/app/models/project_services/slack_service/issue_message.rb
index 438ff33fdff..88e053ec192 100644
--- a/app/models/project_services/slack_service/issue_message.rb
+++ b/app/models/project_services/slack_service/issue_message.rb
@@ -34,7 +34,12 @@ class SlackService
     private
 
     def message
-      "#{user_name} #{state} #{issue_link} in #{project_link}: *#{title}*"
+      case state
+      when "opened"
+        "[#{project_link}] Issue #{state} by #{user_name}"
+      else
+        "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}"
+      end
     end
 
     def opened_issue?
@@ -42,7 +47,11 @@ class SlackService
     end
 
     def description_message
-      [{ text: format(description), color: attachment_color }]
+      [{
+        title: issue_title,
+        title_link: issue_url,
+        text: format(description),
+        color: "#C95823" }]
     end
 
     def project_link
@@ -50,7 +59,11 @@ class SlackService
     end
 
     def issue_link
-      "[issue ##{issue_iid}](#{issue_url})"
+      "[#{issue_title}](#{issue_url})"
+    end
+
+    def issue_title
+      "##{issue_iid} #{title}"
     end
   end
 end
diff --git a/app/models/project_snippet.rb b/app/models/project_snippet.rb
index 5fba6baa204..25b5d777641 100644
--- a/app/models/project_snippet.rb
+++ b/app/models/project_snippet.rb
@@ -7,5 +7,6 @@ class ProjectSnippet < Snippet
   # Scopes
   scope :fresh, -> { order("created_at DESC") }
 
-  participant :author, :notes
+  participant :author
+  participant :notes_with_associations
 end
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 7c1a61bb0bf..25d82929c0b 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -27,6 +27,10 @@ class ProjectWiki
     @project.path_with_namespace + ".wiki"
   end
 
+  def web_url
+    Gitlab::Routing.url_helpers.namespace_project_wiki_url(@project.namespace, @project, :home)
+  end
+
   def url_to_repo
     gitlab_shell.url_to_repo(path_with_namespace)
   end
@@ -40,7 +44,7 @@ class ProjectWiki
   end
 
   def wiki_base_path
-    ["/", @project.path_with_namespace, "/wikis"].join('')
+    [Gitlab.config.gitlab.relative_url_root, "/", @project.path_with_namespace, "/wikis"].join('')
   end
 
   # Returns the Gollum::Wiki object.
@@ -113,7 +117,7 @@ class ProjectWiki
   end
 
   def page_title_and_dir(title)
-    title_array =  title.split("/")
+    title_array = title.split("/")
     title = title_array.pop
     [title, title_array.join("/")]
   end
@@ -142,6 +146,16 @@ class ProjectWiki
     wiki
   end
 
+  def hook_attrs
+    {
+      web_url: web_url,
+      git_ssh_url: ssh_url_to_repo,
+      git_http_url: http_url_to_repo,
+      path_with_namespace: path_with_namespace,
+      default_branch: default_branch
+    }
+  end
+
   private
 
   def init_repo(path_with_namespace)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index a4b42d7226d..1ab163510bf 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -195,6 +195,10 @@ class Repository
     cache.fetch(:branch_names) { branches.map(&:name) }
   end
 
+  def branch_exists?(branch_name)
+    branch_names.include?(branch_name)
+  end
+
   def tag_names
     cache.fetch(:tag_names) { raw_repository.tag_names }
   end
@@ -241,7 +245,7 @@ class Repository
   def cache_keys
     %i(size branch_names tag_names commit_count
        readme version contribution_guide changelog
-       license_blob license_key)
+       license_blob license_key gitignore)
   end
 
   def build_cache
@@ -252,6 +256,10 @@ class Repository
     end
   end
 
+  def expire_gitignore
+    cache.expire(:gitignore)
+  end
+
   def expire_tags_cache
     cache.expire(:tag_names)
     @tags = nil
@@ -453,7 +461,7 @@ class Repository
   def version
     cache.fetch(:version) do
       tree(:head).blobs.find do |file|
-        file.name.downcase == 'version'
+        file.name.casecmp('version').zero?
       end
     end
   end
@@ -468,33 +476,37 @@ class Repository
 
   def changelog
     cache.fetch(:changelog) do
-      tree(:head).blobs.find do |file|
-        file.name =~ /\A(changelog|history|changes|news)/i
-      end
+      file_on_head(/\A(changelog|history|changes|news)/i)
     end
   end
 
   def license_blob
-    return nil if !exists? || empty?
+    return nil unless head_exists?
 
     cache.fetch(:license_blob) do
-      tree(:head).blobs.find do |file|
-        file.name =~ /\A(licen[sc]e|copying)(\..+|\z)/i
-      end
+      file_on_head(/\A(licen[sc]e|copying)(\..+|\z)/i)
     end
   end
 
   def license_key
-    return nil if !exists? || empty?
+    return nil unless head_exists?
 
     cache.fetch(:license_key) do
       Licensee.license(path).try(:key)
     end
   end
 
-  def gitlab_ci_yml
+  def gitignore
     return nil if !exists? || empty?
 
+    cache.fetch(:gitignore) do
+      file_on_head(/\A\.gitignore\z/)
+    end
+  end
+
+  def gitlab_ci_yml
+    return nil unless head_exists?
+
     @gitlab_ci_yml ||= tree(:head).blobs.find do |file|
       file.name == '.gitlab-ci.yml'
     end
@@ -795,7 +807,7 @@ class Repository
   def check_revert_content(commit, base_branch)
     source_sha = find_branch(base_branch).target
     args       = [commit.id, source_sha]
-    args       << { mainline: 1 } if commit.merge_commit?
+    args << { mainline: 1 } if commit.merge_commit?
 
     revert_index = rugged.revert_commit(*args)
     return false if revert_index.conflicts?
@@ -809,7 +821,7 @@ class Repository
   def check_cherry_pick_content(commit, base_branch)
     source_sha = find_branch(base_branch).target
     args       = [commit.id, source_sha]
-    args       << 1 if commit.merge_commit?
+    args << 1 if commit.merge_commit?
 
     cherry_pick_index = rugged.cherrypick_commit(*args)
     return false if cherry_pick_index.conflicts?
@@ -850,7 +862,7 @@ class Repository
 
   def search_files(query, ref)
     offset = 2
-    args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -e #{Regexp.escape(query)} #{ref || root_ref})
+    args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
     Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
   end
 
@@ -960,12 +972,6 @@ class Repository
     end
   end
 
-  def main_language
-    return if empty? || rugged.head_unborn?
-
-    Linguist::Repository.new(rugged, rugged.head.target_id).language
-  end
-
   def avatar
     return nil unless exists?
 
@@ -981,4 +987,12 @@ class Repository
   def cache
     @cache ||= RepositoryCache.new(path_with_namespace)
   end
+
+  def head_exists?
+    exists? && !empty? && !rugged.head_unborn?
+  end
+
+  def file_on_head(regex)
+    tree(:head).blobs.find { |file| file.name =~ regex }
+  end
 end
diff --git a/app/models/snippet.rb b/app/models/snippet.rb
index 0a3c3b57669..407697b745c 100644
--- a/app/models/snippet.rb
+++ b/app/models/snippet.rb
@@ -30,7 +30,8 @@ class Snippet < ActiveRecord::Base
   scope :public_and_internal, -> { where(visibility_level: [Snippet::PUBLIC, Snippet::INTERNAL]) }
   scope :fresh,   -> { order("created_at DESC") }
 
-  participant :author, :notes
+  participant :author
+  participant :notes_with_associations
 
   def self.reference_prefix
     '$'
@@ -100,6 +101,10 @@ class Snippet < ActiveRecord::Base
     content.lines.count > 1000
   end
 
+  def notes_with_associations
+    notes.includes(:author, :project)
+  end
+
   class << self
     # Searches for snippets with a matching title or file name.
     #
diff --git a/app/models/todo.rb b/app/models/todo.rb
index f8b59fe4126..3a091373329 100644
--- a/app/models/todo.rb
+++ b/app/models/todo.rb
@@ -1,6 +1,7 @@
 class Todo < ActiveRecord::Base
-  ASSIGNED  = 1
-  MENTIONED = 2
+  ASSIGNED     = 1
+  MENTIONED    = 2
+  BUILD_FAILED = 3
 
   belongs_to :author, class_name: "User"
   belongs_to :note
@@ -28,6 +29,10 @@ class Todo < ActiveRecord::Base
     state :done
   end
 
+  def build_failed?
+    action == BUILD_FAILED
+  end
+
   def body
     if note.present?
       note.note
diff --git a/app/models/user.rb b/app/models/user.rb
index 138bb5a6bf2..ccd5f61640c 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -20,6 +20,11 @@ class User < ActiveRecord::Base
   default_value_for :hide_no_password, false
   default_value_for :theme_id, gitlab_config.default_theme
 
+  attr_encrypted :otp_secret,
+    key:       Gitlab::Application.config.secret_key_base,
+    mode:      :per_attribute_iv_and_salt,
+    algorithm: 'aes-256-cbc'
+
   devise :two_factor_authenticatable,
          otp_secret_encryption_key: Gitlab::Application.config.secret_key_base
   alias_attribute :two_factor_enabled, :otp_required_for_login
@@ -27,7 +32,7 @@ class User < ActiveRecord::Base
   devise :two_factor_backupable, otp_number_of_backup_codes: 10
   serialize :otp_backup_codes, JSON
 
-  devise :lockable, :async, :recoverable, :rememberable, :trackable,
+  devise :lockable, :recoverable, :rememberable, :trackable,
     :validatable, :omniauthable, :confirmable, :registerable
 
   attr_accessor :force_random_password
@@ -113,6 +118,7 @@ class User < ActiveRecord::Base
   before_save :ensure_external_user_rights
   after_save :ensure_namespace_correct
   after_initialize :set_projects_limit
+  before_create :check_confirmation_email
   after_create :post_create_hook
   after_destroy :post_destroy_hook
 
@@ -308,6 +314,10 @@ class User < ActiveRecord::Base
     @reset_token
   end
 
+  def check_confirmation_email
+    skip_confirmation! unless current_application_settings.send_user_confirmation_email
+  end
+
   def recently_sent_password_reset?
     reset_password_sent_at.present? && reset_password_sent_at >= 1.minute.ago
   end
@@ -382,17 +392,17 @@ class User < ActiveRecord::Base
     Project.where("projects.id IN (#{projects_union.to_sql})")
   end
 
+  def viewable_starred_projects
+    starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (#{projects_union.to_sql})",
+                           [Project::PUBLIC, Project::INTERNAL])
+  end
+
   def owned_projects
     @owned_projects ||=
       Project.where('namespace_id IN (?) OR namespace_id = ?',
                     owned_groups.select(:id), namespace.id).joins(:namespace)
   end
 
-  # Team membership in authorized projects
-  def tm_in_authorized_projects
-    ProjectMember.where(source_id: authorized_projects.map(&:id), user_id: self.id)
-  end
-
   def is_admin?
     admin
   end
@@ -482,10 +492,6 @@ class User < ActiveRecord::Base
     "#{name} (#{username})"
   end
 
-  def tm_of(project)
-    project.project_member_by_id(self.id)
-  end
-
   def already_forked?(project)
     !!fork_of(project)
   end
@@ -771,6 +777,23 @@ class User < ActiveRecord::Base
     notification_settings.find_or_initialize_by(source: source)
   end
 
+  def assigned_open_merge_request_count(force: false)
+    Rails.cache.fetch(['users', id, 'assigned_open_merge_request_count'], force: force) do
+      assigned_merge_requests.opened.count
+    end
+  end
+
+  def assigned_open_issues_count(force: false)
+    Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force) do
+      assigned_issues.opened.count
+    end
+  end
+
+  def update_cache_counts
+    assigned_open_merge_request_count(force: true)
+    assigned_open_issues_count(force: true)
+  end
+
   private
 
   def projects_union