Merge remote-tracking branch 'upstream/master' into pipeline-notifications

* upstream/master: (70 commits)
  Fix routing spec for group controller
  Add small improvements to constrainers and specs
  Faster search
  Fix broken commits search
  Changed helper method to check for none on params Moved if statements around in view
  API: Return 400 when creating a systemhook fails
  Update non-exist group spinach test to match routing
  Bump omniauth-gitlab to 1.0.2 to fix incompatibility with omniauth-oauth2
  Replace trigger with the new ID of the docs project
  Refactor method name
  17492 Update link color for more accessible contrast
  Fixed todos empty state when filtering
  Refactor namespace regex
  implements reset incoming email token on issues modal and account page, reactivates all tests and writes more tests for it
  Use separate email-friendly token for incoming email and let incoming email token be reset
  Use the Gitlab Workhorse HTTP header in the admin dashboard
  Refactor project routing
  Fix 404 when visit /projects page
  Rewritten spinach git_blame tests to rspec feature tests
  Add tests for project#index routing
  ...

9 files changed, 126 insertions, 15 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 6e7a90e7d9c..bb60cc8736c 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -85,6 +85,18 @@ class ApplicationSetting < ActiveRecord::Base
             presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' },
             if: :domain_blacklist_enabled?
 
+  validates :housekeeping_incremental_repack_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :housekeeping_full_repack_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: :housekeeping_incremental_repack_period }
+
+  validates :housekeeping_gc_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: :housekeeping_full_repack_period }
+
   validates_each :restricted_visibility_levels do |record, attr, value|
     unless value.nil?
       value.each do |level|
@@ -168,6 +180,11 @@ class ApplicationSetting < ActiveRecord::Base
       container_registry_token_expire_delay: 5,
       repository_storages: ['default'],
       user_default_external: false,
+      housekeeping_enabled: true,
+      housekeeping_bitmaps_enabled: true,
+      housekeeping_incremental_repack_period: 10,
+      housekeeping_full_repack_period: 50,
+      housekeeping_gc_period: 200,
     )
   end
 
@@ -202,11 +219,7 @@ class ApplicationSetting < ActiveRecord::Base
   end
 
   def repository_storages
-    value = read_attribute(:repository_storages)
-    value = [value] if value.is_a?(String)
-    value = [] if value.nil?
-
-    value
+    Array(read_attribute(:repository_storages))
   end
 
   # repository_storage is still required in the API. Remove in 9.0
diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb
index 613444e0d70..93a6b3122e0 100644
--- a/app/models/concerns/issuable.rb
+++ b/app/models/concerns/issuable.rb
@@ -286,6 +286,11 @@ module Issuable
     false
   end
 
+  def assignee_or_author?(user)
+    # We're comparing IDs here so we don't need to load any associations.
+    author_id == user.id || assignee_id == user.id
+  end
+
   def record_metrics
     metrics = self.metrics || create_metrics
     metrics.record!
diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb
index 24c7b26d223..04d30f46210 100644
--- a/app/models/concerns/token_authenticatable.rb
+++ b/app/models/concerns/token_authenticatable.rb
@@ -4,17 +4,21 @@ module TokenAuthenticatable
   private
 
   def write_new_token(token_field)
-    new_token = generate_token(token_field)
+    new_token = generate_available_token(token_field)
     write_attribute(token_field, new_token)
   end
 
-  def generate_token(token_field)
+  def generate_available_token(token_field)
     loop do
-      token = Devise.friendly_token
+      token = generate_token(token_field)
       break token unless self.class.unscoped.find_by(token_field => token)
     end
   end
 
+  def generate_token(token_field)
+    Devise.friendly_token
+  end
+
   class_methods do
     def authentication_token_fields
       @token_fields || []
diff --git a/app/models/external_issue.rb b/app/models/external_issue.rb
index fd9a8c1b8b7..91b508eb325 100644
--- a/app/models/external_issue.rb
+++ b/app/models/external_issue.rb
@@ -29,6 +29,15 @@ class ExternalIssue
     @project
   end
 
+  def project_id
+    @project.id
+  end
+
+  # Pattern used to extract `JIRA-123` issue references from text
+  def self.reference_pattern
+    @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)}
+  end
+
   def to_reference(_from_project = nil)
     id
   end
diff --git a/app/models/issue_collection.rb b/app/models/issue_collection.rb
new file mode 100644
index 00000000000..f0b7d9914c8
--- /dev/null
+++ b/app/models/issue_collection.rb
@@ -0,0 +1,42 @@
+# IssueCollection can be used to reduce a list of issues down to a subset.
+#
+# IssueCollection is not meant to be some sort of Enumerable, instead it's meant
+# to take a list of issues and return a new list of issues based on some
+# criteria. For example, given a list of issues you may want to return a list of
+# issues that can be read or updated by a given user.
+class IssueCollection
+  attr_reader :collection
+
+  def initialize(collection)
+    @collection = collection
+  end
+
+  # Returns all the issues that can be updated by the user.
+  def updatable_by_user(user)
+    return collection if user.admin?
+
+    # Given all the issue projects we get a list of projects that the current
+    # user has at least reporter access to.
+    projects_with_reporter_access = user.
+      projects_with_reporter_access_limited_to(project_ids).
+      pluck(:id)
+
+    collection.select do |issue|
+      if projects_with_reporter_access.include?(issue.project_id)
+        true
+      elsif issue.is_a?(Issue)
+        issue.assignee_or_author?(user)
+      else
+        false
+      end
+    end
+  end
+
+  alias_method :visible_to, :updatable_by_user
+
+  private
+
+  def project_ids
+    @project_ids ||= collection.map(&:project_id).uniq
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 686d285410b..4c9c7c001dd 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -624,13 +624,12 @@ class Project < ActiveRecord::Base
   end
 
   def new_issue_address(author)
-    # This feature is disabled for the time being.
-    return nil
+    return unless Gitlab::IncomingEmail.supports_issue_creation? && author
 
-    if Gitlab::IncomingEmail.enabled? && author # rubocop:disable Lint/UnreachableCode
-      Gitlab::IncomingEmail.reply_address(
-        "#{path_with_namespace}+#{author.authentication_token}")
-    end
+    author.ensure_incoming_email_token!
+
+    Gitlab::IncomingEmail.reply_address(
+      "#{path_with_namespace}+#{author.incoming_email_token}")
   end
 
   def build_commit_note(commit)
diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb
index 0a493b7a12b..2dbe0075465 100644
--- a/app/models/project_services/jira_service.rb
+++ b/app/models/project_services/jira_service.rb
@@ -163,6 +163,21 @@ class JiraService < IssueTrackerService
     add_comment(data, issue_key)
   end
 
+  # reason why service cannot be tested
+  def disabled_title
+    "Please fill in Password and Username."
+  end
+
+  def can_test?
+    username.present? && password.present?
+  end
+
+  # JIRA does not need test data.
+  # We are requesting the project that belongs to the project key.
+  def test_data(user = nil, project = nil)
+    nil
+  end
+
   def test_settings
     return unless url.present?
     # Test settings by getting the project
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 30be7262438..7d06ce1e85b 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -1064,6 +1064,10 @@ class Repository
   end
 
   def search_files(query, ref)
+    unless exists? && has_visible_content? && query.present?
+      return []
+    end
+
     offset = 2
     args = %W(#{Gitlab.config.git.bin_path} grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
     Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
diff --git a/app/models/user.rb b/app/models/user.rb
index 65e96ee6b2e..3813df6684e 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -13,6 +13,7 @@ class User < ActiveRecord::Base
   DEFAULT_NOTIFICATION_LEVEL = :participating
 
   add_authentication_token_field :authentication_token
+  add_authentication_token_field :incoming_email_token
 
   default_value_for :admin, false
   default_value_for(:external) { current_application_settings.user_default_external }
@@ -119,7 +120,7 @@ class User < ActiveRecord::Base
   before_validation :set_public_email, if: ->(user) { user.public_email_changed? }
 
   after_update :update_emails_with_primary_email, if: ->(user) { user.email_changed? }
-  before_save :ensure_authentication_token
+  before_save :ensure_authentication_token, :ensure_incoming_email_token
   before_save :ensure_external_user_rights
   after_save :ensure_namespace_correct
   after_initialize :set_projects_limit
@@ -444,6 +445,16 @@ class User < ActiveRecord::Base
     Project.where("projects.id IN (#{projects_union(min_access_level).to_sql})")
   end
 
+  # Returns the projects this user has reporter (or greater) access to, limited
+  # to at most the given projects.
+  #
+  # This method is useful when you have a list of projects and want to
+  # efficiently check to which of these projects the user has at least reporter
+  # access.
+  def projects_with_reporter_access_limited_to(projects)
+    authorized_projects(Gitlab::Access::REPORTER).where(id: projects)
+  end
+
   def viewable_starred_projects
     starred_projects.where("projects.visibility_level IN (?) OR projects.id IN (#{projects_union.to_sql})",
                            [Project::PUBLIC, Project::INTERNAL])
@@ -946,4 +957,13 @@ class User < ActiveRecord::Base
       signup_domain =~ regexp
     end
   end
+
+  def generate_token(token_field)
+    if token_field == :incoming_email_token
+      # Needs to be all lowercase and alphanumeric because it's gonna be used in an email address.
+      SecureRandom.hex.to_i(16).to_s(36)
+    else
+      super
+    end
+  end
 end