Merge remote-tracking branch 'upstream/master' into feature/1376-allow-write-access-deploy-keys

* upstream/master: (236 commits)
  Mention Git strategy none
  Remove ToC since it's now supported in the docs portal itself
  Add 8.14 to versions with further additions to review apps
  Add Limitations sections to environments and review apps docs
  Add link to environments docs
  Fix URL to review apps docs
  Add a prerequisites section, add some links
  Link to NGINX example project for the time being
  Get rid most of the irrelevant sections
  Add note about current limitation in $CI_BUILD_REF_NAME
  Add an intro and an Overview section for Review Apps
  WIP review apps
  Add Review apps link to CI README
  Add stop environment permissions and remove delete
  Add note about auto-stopping of environments
  Update CHANGELOG.md for 8.13.6
  Finish "Stopping envs" and "Grouping similar envs" sections
  Highlight first user autocomplete option
  Remove ToC and fix headings in Markdown docs
  Revert "Merge branch '22680-unlabel-limit-autocomplete-to-selected-items' into 'master'"
  ...

16 files changed, 127 insertions, 67 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index bb60cc8736c..bf463a3b6bb 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -19,6 +19,7 @@ class ApplicationSetting < ActiveRecord::Base
   serialize :domain_whitelist, Array
   serialize :domain_blacklist, Array
   serialize :repository_storages
+  serialize :sidekiq_throttling_queues, Array
 
   cache_markdown_field :sign_in_text
   cache_markdown_field :help_page_text
@@ -85,6 +86,15 @@ class ApplicationSetting < ActiveRecord::Base
             presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' },
             if: :domain_blacklist_enabled?
 
+  validates :sidekiq_throttling_factor,
+            numericality: { greater_than: 0, less_than: 1 },
+            presence: { message: 'Throttling factor cannot be empty if Sidekiq Throttling is enabled.' },
+            if: :sidekiq_throttling_enabled?
+
+  validates :sidekiq_throttling_queues,
+            presence: { message: 'Queues to throttle cannot be empty if Sidekiq Throttling is enabled.' },
+            if: :sidekiq_throttling_enabled?
+
   validates :housekeeping_incremental_repack_period,
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }
@@ -180,6 +190,7 @@ class ApplicationSetting < ActiveRecord::Base
       container_registry_token_expire_delay: 5,
       repository_storages: ['default'],
       user_default_external: false,
+      sidekiq_throttling_enabled: false,
       housekeeping_enabled: true,
       housekeeping_bitmaps_enabled: true,
       housekeeping_incremental_repack_period: 10,
@@ -192,6 +203,10 @@ class ApplicationSetting < ActiveRecord::Base
     ActiveRecord::Base.connection.column_exists?(:application_settings, :home_page_url)
   end
 
+  def sidekiq_throttling_column_exists?
+    ActiveRecord::Base.connection.column_exists?(:application_settings, :sidekiq_throttling_enabled)
+  end
+
   def domain_whitelist_raw
     self.domain_whitelist.join("\n") unless self.domain_whitelist.nil?
   end
@@ -245,6 +260,12 @@ class ApplicationSetting < ActiveRecord::Base
     ensure_health_check_access_token!
   end
 
+  def sidekiq_throttling_enabled?
+    return false unless sidekiq_throttling_column_exists?
+
+    sidekiq_throttling_enabled
+  end
+
   private
 
   def check_repository_storages
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index bf5f92f8462..33612256540 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -271,6 +271,7 @@ module Ci
 
     def append_trace(trace_part, offset)
       recreate_trace_dir
+      touch if needs_touch?
 
       trace_part = hide_secrets(trace_part)
 
@@ -280,6 +281,10 @@ module Ci
       end
     end
 
+    def needs_touch?
+      Time.now - updated_at > 15.minutes.to_i
+    end
+
     def trace_file_path
       if has_old_trace_file?
         old_path_to_trace
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 73f415c0ef0..5278efd71d2 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -37,6 +37,10 @@ class Environment < ActiveRecord::Base
     state :stopped
   end
 
+  def recently_updated_on_branch?(ref)
+    ref.to_s == last_deployment.try(:ref)
+  end
+
   def last_deployment
     deployments.last
   end
@@ -92,6 +96,7 @@ class Environment < ActiveRecord::Base
   def stop!(current_user)
     return unless stoppable?
 
+    stop
     stop_action.play(current_user)
   end
 end
diff --git a/app/models/event.rb b/app/models/event.rb
index c76d88b1c7b..21eaca917b8 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -62,7 +62,7 @@ class Event < ActiveRecord::Base
   end
 
   def visible_to_user?(user = nil)
-    if push?
+    if push? || commit_note?
       Ability.allowed?(user, :download_code, project)
     elsif membership_changed?
       true
@@ -283,7 +283,7 @@ class Event < ActiveRecord::Base
   end
 
   def commit_note?
-    target.for_commit?
+    note? && target && target.for_commit?
   end
 
   def issue_note?
@@ -295,7 +295,7 @@ class Event < ActiveRecord::Base
   end
 
   def project_snippet_note?
-    target.for_snippet?
+    note? && target && target.for_snippet?
   end
 
   def note_target
diff --git a/app/models/issue.rb b/app/models/issue.rb
index adbca510ef7..4a4017003d8 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -266,7 +266,7 @@ class Issue < ActiveRecord::Base
 
   def as_json(options = {})
     super(options).tap do |json|
-      json[:subscribed] = subscribed?(options[:user]) if options.has_key?(:user)
+      json[:subscribed] = subscribed?(options[:user]) if options.has_key?(:user) && options[:user]
 
       if options.has_key?(:labels)
         json[:labels] = labels.as_json(
diff --git a/app/models/key.rb b/app/models/key.rb
index 568a60b8af3..ff8dda2dc89 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -6,7 +6,7 @@ class Key < ActiveRecord::Base
 
   belongs_to :user
 
-  before_validation :strip_white_space, :generate_fingerprint
+  before_validation :generate_fingerprint
 
   validates :title, presence: true, length: { within: 0..255 }
   validates :key, presence: true, length: { within: 0..5000 }, format: { with: /\A(ssh|ecdsa)-.*\Z/ }
@@ -21,8 +21,9 @@ class Key < ActiveRecord::Base
   after_destroy :remove_from_shell
   after_destroy :post_destroy_hook
 
-  def strip_white_space
-    self.key = key.strip unless key.blank?
+  def key=(value)
+    value.strip! unless value.blank?
+    write_attribute(:key, value)
   end
 
   def publishable_key
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index d76feb9680e..9d3eab52189 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -692,12 +692,15 @@ class MergeRequest < ActiveRecord::Base
   def environments
     return [] unless diff_head_commit
 
-    @environments ||=
-      begin
-        envs = target_project.environments_for(target_branch, diff_head_commit, with_tags: true)
-        envs.concat(source_project.environments_for(source_branch, diff_head_commit)) if source_project
-        envs.uniq
-      end
+    @environments ||= begin
+      target_envs = target_project.environments_for(
+        target_branch, commit: diff_head_commit, with_tags: true)
+
+      source_envs = source_project.environments_for(
+        source_branch, commit: diff_head_commit) if source_project
+
+      (target_envs.to_a + source_envs.to_a).uniq
+    end
   end
 
   def state_human_name
diff --git a/app/models/note.rb b/app/models/note.rb
index 2d644b03e4d..9ff5e308ed2 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -7,6 +7,7 @@ class Note < ActiveRecord::Base
   include Importable
   include FasterCacheKeys
   include CacheMarkdownField
+  include AfterCommitQueue
 
   cache_markdown_field :note, pipeline: :note
 
diff --git a/app/models/project.rb b/app/models/project.rb
index bbe590b5a8a..f9bcc547c36 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -35,6 +35,7 @@ class Project < ActiveRecord::Base
   default_value_for :builds_enabled, gitlab_config_features.builds
   default_value_for :wiki_enabled, gitlab_config_features.wiki
   default_value_for :snippets_enabled, gitlab_config_features.snippets
+  default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
 
   after_create :ensure_dir_exist
   after_create :create_project_feature, unless: :project_feature
@@ -76,7 +77,6 @@ class Project < ActiveRecord::Base
   has_many :boards, before_add: :validate_board_limit, dependent: :destroy
 
   # Project services
-  has_many :services
   has_one :campfire_service, dependent: :destroy
   has_one :drone_ci_service, dependent: :destroy
   has_one :emails_on_push_service, dependent: :destroy
@@ -748,27 +748,32 @@ class Project < ActiveRecord::Base
     update_column(:has_external_wiki, services.external_wikis.any?)
   end
 
-  def build_missing_services
+  def find_or_initialize_services
     services_templates = Service.where(template: true)
 
-    Service.available_services_names.each do |service_name|
+    Service.available_services_names.map do |service_name|
       service = find_service(services, service_name)
 
-      # If service is available but missing in db
-      if service.nil?
+      if service
+        service
+      else
         # We should check if template for the service exists
         template = find_service(services_templates, service_name)
 
         if template.nil?
-          # If no template, we should create an instance. Ex `create_gitlab_ci_service`
-          public_send("create_#{service_name}_service")
+          # If no template, we should create an instance. Ex `build_gitlab_ci_service`
+          public_send("build_#{service_name}_service")
         else
-          Service.create_from_template(self.id, template)
+          Service.build_from_template(id, template)
         end
       end
     end
   end
 
+  def find_or_initialize_service(name)
+    find_or_initialize_services.find { |service| service.to_param == name }
+  end
+
   def create_labels
     Label.templates.each do |label|
       params = label.attributes.except('id', 'template', 'created_at', 'updated_at')
@@ -878,7 +883,7 @@ class Project < ActiveRecord::Base
   end
 
   def empty_repo?
-    !repository.exists? || !repository.has_visible_content?
+    repository.empty_repo?
   end
 
   def repo
@@ -1318,22 +1323,30 @@ class Project < ActiveRecord::Base
     Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) }
   end
 
-  def environments_for(ref, commit, with_tags: false)
-    environment_ids = deployments.group(:environment_id).
-      select(:environment_id)
+  def environments_for(ref, commit: nil, with_tags: false)
+    deployments_query = with_tags ? 'ref = ? OR tag IS TRUE' : 'ref = ?'
 
-    environment_ids =
-      if with_tags
-        environment_ids.where('ref=? OR tag IS TRUE', ref)
-      else
-        environment_ids.where(ref: ref)
-      end
+    environment_ids = deployments
+      .where(deployments_query, ref.to_s)
+      .group(:environment_id)
+      .select(:environment_id)
 
-    environments.available.where(id: environment_ids).select do |environment|
+    environments_found = environments.available
+      .where(id: environment_ids).to_a
+
+    return environments_found unless commit
+
+    environments_found.select do |environment|
       environment.includes_commit?(commit)
     end
   end
 
+  def environments_recently_updated_on_branch(branch)
+    environments_for(branch).select do |environment|
+      environment.recently_updated_on_branch?(branch)
+    end
+  end
+
   private
 
   def pushes_since_gc_redis_key
diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb
index 34fd5a57b5e..5c53c8f1ee5 100644
--- a/app/models/project_feature.rb
+++ b/app/models/project_feature.rb
@@ -49,20 +49,14 @@ class ProjectFeature < ActiveRecord::Base
   end
 
   def builds_enabled?
-    return true unless builds_access_level
-
     builds_access_level > DISABLED
   end
 
   def wiki_enabled?
-    return true unless wiki_access_level
-
     wiki_access_level > DISABLED
   end
 
   def merge_requests_enabled?
-    return true unless merge_requests_access_level
-
     merge_requests_access_level > DISABLED
   end
 
diff --git a/app/models/project_services/slack_service/note_message.rb b/app/models/project_services/slack_service/note_message.rb
index 9e84e90f38c..797c5937f09 100644
--- a/app/models/project_services/slack_service/note_message.rb
+++ b/app/models/project_services/slack_service/note_message.rb
@@ -46,25 +46,25 @@ class SlackService
       commit_sha = commit[:id]
       commit_sha = Commit.truncate_sha(commit_sha)
       commented_on_message(
-        "[commit #{commit_sha}](#{@note_url})",
+        "commit #{commit_sha}",
         format_title(commit[:message]))
     end
 
     def create_issue_note(issue)
       commented_on_message(
-        "[issue ##{issue[:iid]}](#{@note_url})",
+        "issue ##{issue[:iid]}",
         format_title(issue[:title]))
     end
 
     def create_merge_note(merge_request)
       commented_on_message(
-        "[merge request !#{merge_request[:iid]}](#{@note_url})",
+        "merge request !#{merge_request[:iid]}",
         format_title(merge_request[:title]))
     end
 
     def create_snippet_note(snippet)
       commented_on_message(
-        "[snippet ##{snippet[:id]}](#{@note_url})",
+        "snippet ##{snippet[:id]}",
         format_title(snippet[:title]))
     end
 
@@ -76,8 +76,8 @@ class SlackService
       "[#{@project_name}](#{@project_url})"
     end
 
-    def commented_on_message(target_link, title)
-      @message = "#{@user_name} commented on #{target_link} in #{project_link}: *#{title}*"
+    def commented_on_message(target, title)
+      @message = "#{@user_name} [commented on #{target}](#{@note_url}) in #{project_link}: *#{title}*"
     end
   end
 end
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 46f70da2452..9db96347322 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -127,7 +127,7 @@ class ProjectWiki
   end
 
   def search_files(query)
-    repository.search_files(query, default_branch)
+    repository.search_files_by_content(query, default_branch)
   end
 
   def repository
diff --git a/app/models/repository.rb b/app/models/repository.rb
index fe991904601..146424d2b1c 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -15,16 +15,6 @@ class Repository
     Gitlab.config.repositories.storages
   end
 
-  def self.remove_storage_from_path(repo_path)
-    storages.find do |_, storage_path|
-      if repo_path.start_with?(storage_path)
-        return repo_path.sub(storage_path, '')
-      end
-    end
-
-    repo_path
-  end
-
   def initialize(path_with_namespace, project)
     @path_with_namespace = path_with_namespace
     @project = project
@@ -243,7 +233,7 @@ class Repository
     # offer 'compare and swap' ref updates. Without compare-and-swap we can
     # (and have!) accidentally reset the ref to an earlier state, clobbering
     # commits. See also https://github.com/libgit2/libgit2/issues/1534.
-    command = %w[git update-ref --stdin -z]
+    command = %W(#{Gitlab.config.git.bin_path} update-ref --stdin -z)
     _, status = Gitlab::Popen.popen(command, path_to_repo) do |stdin|
       stdin.write("update #{name}\x00#{newrev}\x00#{oldrev}\x00")
     end
@@ -631,7 +621,7 @@ class Repository
     @head_tree ||= Tree.new(self, head_commit.sha, nil)
   end
 
-  def tree(sha = :head, path = nil)
+  def tree(sha = :head, path = nil, recursive: false)
     if sha == :head
       if path.nil?
         return head_tree
@@ -640,7 +630,7 @@ class Repository
       end
     end
 
-    Tree.new(self, sha, path)
+    Tree.new(self, sha, path, recursive: recursive)
   end
 
   def blob_at_branch(branch_name, path)
@@ -1063,16 +1053,25 @@ class Repository
     merge_base(ancestor_id, descendant_id) == ancestor_id
   end
 
-  def search_files(query, ref)
-    unless exists? && has_visible_content? && query.present?
-      return []
-    end
+  def empty_repo?
+    !exists? || !has_visible_content?
+  end
+
+  def search_files_by_content(query, ref)
+    return [] if empty_repo? || query.blank?
 
     offset = 2
     args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
     Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
   end
 
+  def search_files_by_name(query, ref)
+    return [] if empty_repo? || query.blank?
+
+    args = %W(#{Gitlab.config.git.bin_path} ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)})
+    Gitlab::Popen.popen(args, path_to_repo).first.lines.map(&:strip)
+  end
+
   def fetch_ref(source_path, source_ref, target_ref)
     args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
     Gitlab::Popen.popen(args, path_to_repo)
diff --git a/app/models/service.rb b/app/models/service.rb
index 625fbc48302..9d6ff190cdf 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -222,11 +222,11 @@ class Service < ActiveRecord::Base
     ]
   end
 
-  def self.create_from_template(project_id, template)
+  def self.build_from_template(project_id, template)
     service = template.dup
     service.template = false
     service.project_id = project_id
-    service if service.save
+    service
   end
 
   private
diff --git a/app/models/tree.rb b/app/models/tree.rb
index 7c4ed6e393b..2d1d68dbd81 100644
--- a/app/models/tree.rb
+++ b/app/models/tree.rb
@@ -3,15 +3,16 @@ class Tree
 
   attr_accessor :repository, :sha, :path, :entries
 
-  def initialize(repository, sha, path = '/')
+  def initialize(repository, sha, path = '/', recursive: false)
     path = '/' if path.blank?
 
     @repository = repository
     @sha = sha
     @path = path
+    @recursive = recursive
 
     git_repo = @repository.raw_repository
-    @entries = Gitlab::Git::Tree.where(git_repo, @sha, @path)
+    @entries = get_entries(git_repo, @sha, @path, recursive: @recursive)
   end
 
   def readme
@@ -58,4 +59,21 @@ class Tree
   def sorted_entries
     trees + blobs + submodules
   end
+
+  private
+
+  def get_entries(git_repo, sha, path, recursive: false)
+    current_path_entries = Gitlab::Git::Tree.where(git_repo, sha, path)
+    ordered_entries = []
+
+    current_path_entries.each do |entry|
+      ordered_entries << entry
+
+      if recursive && entry.dir?
+        ordered_entries.concat(get_entries(git_repo, sha, entry.path, recursive: true))
+      end
+    end
+
+    ordered_entries
+  end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index 0e96ad88638..40130b8b25c 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -173,7 +173,7 @@ class User < ActiveRecord::Base
   scope :external, -> { where(external: true) }
   scope :active, -> { with_state(:active) }
   scope :not_in_project, ->(project) { project.users.present? ? where("id not in (:ids)", ids: project.users.map(&:id) ) : all }
-  scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members)') }
+  scope :without_projects, -> { where('id NOT IN (SELECT DISTINCT(user_id) FROM members WHERE user_id IS NOT NULL AND requested_at IS NULL)') }
   scope :todo_authors, ->(user_id, state) { where(id: Todo.where(user_id: user_id, state: state).select(:author_id)) }
 
   def self.with_two_factor