Merge branch 'master' into ci/api-builds

* master: (76 commits)
  Add Changelog entry for build traces data integrity fix
  Update doc_styleguide.md [ci skip]
  Added hint that you can search users by name, username, or email.
  Add changelog
  Randomize metrics sample intervals
  Make the metrics sampler interval configurable
  Don't automatically require awesome_print
  Disable colorization if STDOUT is not a tty
  Block the reported user before destroying the record
  changes `$quote-gray` to `$secondary-text`
  makes message plural for multiple MRs and removes from loop. Duh.
  Prepare Installation and Update docs for 8.4 RC1
  Mention channel/key bug in irkerd docs
  Revert "Remove the `:coffee` and `:coffeescript` Haml filters"
  gets merge request discussion working again
  adds back in discussion.haml.html for issues commenting and closing/reopening properly.
  removing last chunk of MR ajax changes, rest will be in another MR
  reverting more MR ajax files, will appear in different commit
  reverting MR ajax changes, which will be in a different MR
  reverting _mr_title.html.haml
  ...

14 files changed, 101 insertions, 63 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index f0816a46529..b403d3c5032 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -71,6 +71,7 @@ module API
       expose :avatar_url
       expose :star_count, :forks_count
       expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? }
+      expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
     end
 
     class ProjectMember < UserBasic
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 8b1390e3289..71bb342f844 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -69,7 +69,8 @@ module API
       # Example Request:
       #   GET /projects/:id
       get ":id" do
-        present user_project, with: Entities::ProjectWithAccess, user: current_user
+        present user_project, with: Entities::ProjectWithAccess, user: current_user,
+                              user_can_admin_project: can?(current_user, :admin_project, user_project)
       end
 
       # Get events for a single project
@@ -118,7 +119,8 @@ module API
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(current_user, attrs).execute
         if @project.saved?
-          present @project, with: Entities::Project
+          present @project, with: Entities::Project,
+                            user_can_admin_project: can?(current_user, :admin_project, @project)
         else
           if @project.errors[:limit_reached].present?
             error!(@project.errors[:limit_reached], 403)
@@ -163,7 +165,8 @@ module API
         attrs = map_public_to_visibility_level(attrs)
         @project = ::Projects::CreateService.new(user, attrs).execute
         if @project.saved?
-          present @project, with: Entities::Project
+          present @project, with: Entities::Project,
+                            user_can_admin_project: can?(current_user, :admin_project, @project)
         else
           render_validation_error!(@project)
         end
@@ -182,8 +185,9 @@ module API
         if @forked_project.errors.any?
           conflict!(@forked_project.errors.messages)
         else
-          present @forked_project, with: Entities::Project
-        end
+          present @forked_project, with: Entities::Project,
+                                   user_can_admin_project: can?(current_user, :admin_project, @forked_project)
+         end
       end
 
       # Update an existing project
@@ -229,7 +233,8 @@ module API
         if user_project.errors.any?
           render_validation_error!(user_project)
         else
-          present user_project, with: Entities::Project
+          present user_project, with: Entities::Project,
+                                user_can_admin_project: can?(current_user, :admin_project, user_project)
         end
       end
 
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 7198a8b03e2..5dd6d2fe3c7 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -133,7 +133,7 @@ module Banzai
 
           next unless link && text
 
-          link = URI.decode(link)
+          link = CGI.unescape(link)
           # Ignore ending punctionation like periods or commas
           next unless link == text && text =~ /\A#{pattern}/
 
@@ -170,7 +170,7 @@ module Banzai
           text = node.text
 
           next unless link && text
-          link = URI.decode(link)
+          link = CGI.unescape(link)
           next unless link && link =~ /\A#{pattern}\z/
 
           html = yield link, text
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index e24b94d6159..59926084d07 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -12,7 +12,7 @@ module Gitlab
       end
 
       def execute
-        project_identifier = URI.encode(project.import_source, '/')
+        project_identifier = CGI.escape(project.import_source, '/')
 
         #Issues && Comments
         issues = client.issues(project_identifier)
diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb
index c438a3d167b..b2bdbc10d7f 100644
--- a/lib/gitlab/ldap/access.rb
+++ b/lib/gitlab/ldap/access.rb
@@ -5,7 +5,7 @@
 module Gitlab
   module LDAP
     class Access
-      attr_reader :adapter, :provider, :user
+      attr_reader :provider, :user
 
       def self.open(user, &block)
         Gitlab::LDAP::Adapter.open(user.ldap_identity.provider) do |adapter|
@@ -32,7 +32,7 @@ module Gitlab
       end
 
       def allowed?
-        if Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter)
+        if ldap_user
           return true unless ldap_config.active_directory
 
           # Block user in GitLab if he/she was blocked in AD
@@ -59,6 +59,10 @@ module Gitlab
       def ldap_config
         Gitlab::LDAP::Config.new(provider)
       end
+
+      def ldap_user
+        @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter)
+      end
     end
   end
 end
diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb
index 577a890a7d9..df65179bfea 100644
--- a/lib/gitlab/ldap/adapter.rb
+++ b/lib/gitlab/ldap/adapter.rb
@@ -70,19 +70,25 @@ module Gitlab
       end
 
       def ldap_search(*args)
-        results = ldap.search(*args)
+        # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead.
+        Timeout.timeout(config.timeout) do
+          results = ldap.search(*args)
 
-        if results.nil?
-          response = ldap.get_operation_result
+          if results.nil?
+            response = ldap.get_operation_result
 
-          unless response.code.zero?
-            Rails.logger.warn("LDAP search error: #{response.message}")
-          end
+            unless response.code.zero?
+              Rails.logger.warn("LDAP search error: #{response.message}")
+            end
 
-          []
-        else
-          results
+            []
+          else
+            results
+          end
         end
+      rescue Timeout::Error
+        Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds")
+        []
       end
     end
   end
diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb
index 101a3285f4b..aff7ccb157f 100644
--- a/lib/gitlab/ldap/config.rb
+++ b/lib/gitlab/ldap/config.rb
@@ -88,6 +88,10 @@ module Gitlab
         options['attributes']
       end
 
+      def timeout
+        options['timeout'].to_i
+      end
+
       protected
       def base_config
         Gitlab.config.ldap
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 44356a0e42c..88a265c6af2 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -13,7 +13,8 @@ module Gitlab
         timeout:               current_application_settings[:metrics_timeout],
         method_call_threshold: current_application_settings[:metrics_method_call_threshold],
         host:                  current_application_settings[:metrics_host],
-        port:                  current_application_settings[:metrics_port]
+        port:                  current_application_settings[:metrics_port],
+        sample_interval:       current_application_settings[:metrics_sample_interval] || 15
       }
     end
 
@@ -36,20 +37,6 @@ module Gitlab
       @pool
     end
 
-    # Returns a relative path and line number based on the last application call
-    # frame.
-    def self.last_relative_application_frame
-      frame = caller_locations.find do |l|
-        l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT)
-      end
-
-      if frame
-        return frame.path.sub(PATH_REGEX, ''), frame.lineno
-      else
-        return nil, nil
-      end
-    end
-
     def self.submit_metrics(metrics)
       prepared = prepare_metrics(metrics)
 
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index e7a2f26d48b..6f179789d3e 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -39,10 +39,8 @@ module Gitlab
       end
 
       def tag_controller(trans, env)
-        controller = env[CONTROLLER_KEY]
-        label      = "#{controller.class.name}##{controller.action_name}"
-
-        trans.add_tag(:action, label)
+        controller   = env[CONTROLLER_KEY]
+        trans.action = "#{controller.class.name}##{controller.action_name}"
       end
     end
   end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 1ea425bc904..fc709222a9b 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -7,9 +7,14 @@ module Gitlab
     # statistics, etc.
     class Sampler
       # interval - The sampling interval in seconds.
-      def initialize(interval = 15)
-        @interval = interval
-        @metrics  = []
+      def initialize(interval = Metrics.settings[:sample_interval])
+        interval_half = interval.to_f / 2
+
+        @interval       = interval
+        @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+        @last_step      = nil
+
+        @metrics = []
 
         @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
         @last_major_gc = Delta.new(GC.stat[:major_gc_count])
@@ -26,7 +31,7 @@ module Gitlab
           Thread.current.abort_on_exception = true
 
           loop do
-            sleep(@interval)
+            sleep(sleep_interval)
 
             sample
           end
@@ -102,6 +107,23 @@ module Gitlab
       def sidekiq?
         Sidekiq.server?
       end
+
+      # Returns the sleep interval with a random adjustment.
+      #
+      # The random adjustment is put in place to ensure we:
+      #
+      # 1. Don't generate samples at the exact same interval every time (thus
+      #    potentially missing anything that happens in between samples).
+      # 2. Don't sample data at the same interval two times in a row.
+      def sleep_interval
+        while step = @interval_steps.sample
+          if step != @last_step
+            @last_step = step
+
+            return @interval + @last_step
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index ad441decfa2..fd98aa3412e 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -5,19 +5,14 @@ module Gitlab
     # This middleware is intended to be used as a server-side middleware.
     class SidekiqMiddleware
       def call(worker, message, queue)
-        trans = Transaction.new
+        trans = Transaction.new("#{worker.class.name}#perform")
 
         begin
           trans.run { yield }
         ensure
-          tag_worker(trans, worker)
           trans.finish
         end
       end
-
-      def tag_worker(trans, worker)
-        trans.add_tag(:action, "#{worker.class.name}#perform")
-      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 7c0105d543a..2e9dd4645e3 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -33,16 +33,8 @@ module Gitlab
 
         def tags_for(event)
           path = relative_path(event.payload[:identifier])
-          tags = { view: path }
 
-          file, line = Metrics.last_relative_application_frame
-
-          if file and line
-            tags[:file] = file
-            tags[:line] = line
-          end
-
-          tags
+          { view: path }
         end
 
         def current_transaction
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 73131cc6ef2..2578ddc49f4 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -6,11 +6,15 @@ module Gitlab
 
       attr_reader :tags, :values
 
+      attr_accessor :action
+
       def self.current
         Thread.current[THREAD_KEY]
       end
 
-      def initialize
+      # action - A String describing the action performed, usually the class
+      #          plus method name.
+      def initialize(action = nil)
         @metrics = []
 
         @started_at  = nil
@@ -18,20 +22,30 @@ module Gitlab
 
         @values = Hash.new(0)
         @tags   = {}
+        @action = action
+
+        @memory_before = 0
+        @memory_after  = 0
       end
 
       def duration
         @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0
       end
 
+      def allocated_memory
+        @memory_after - @memory_before
+      end
+
       def run
         Thread.current[THREAD_KEY] = self
 
-        @started_at = Time.now
+        @memory_before = System.memory_usage
+        @started_at    = Time.now
 
         yield
       ensure
-        @finished_at = Time.now
+        @memory_after = System.memory_usage
+        @finished_at  = Time.now
 
         Thread.current[THREAD_KEY] = nil
       end
@@ -60,7 +74,7 @@ module Gitlab
       end
 
       def track_self
-        values = { duration: duration }
+        values = { duration: duration, allocated_memory: allocated_memory }
 
         @values.each do |name, value|
           values[name] = value
@@ -70,7 +84,15 @@ module Gitlab
       end
 
       def submit
-        Metrics.submit_metrics(@metrics.map(&:to_hash))
+        metrics = @metrics.map do |metric|
+          hash = metric.to_hash
+
+          hash[:tags][:action] ||= @action if @action
+
+          hash
+        end
+
+        Metrics.submit_metrics(metrics)
       end
 
       def sidekiq?
diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake
index ebe516ec879..8c63877e51c 100644
--- a/lib/tasks/gitlab/task_helpers.rake
+++ b/lib/tasks/gitlab/task_helpers.rake
@@ -2,6 +2,8 @@ module Gitlab
   class TaskAbortedByUserError < StandardError; end
 end
 
+String.disable_colorization = true unless STDOUT.isatty
+
 namespace :gitlab do
 
   # Ask if the user wants to continue