From 141e946c3da97c7af02aaca5324c6e4ce7362a04 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 9 Dec 2015 16:45:51 +0100
Subject: Storing of application metrics in InfluxDB

This adds the ability to write application metrics (e.g. SQL timings) to
InfluxDB. These metrics can in turn be visualized using Grafana, or
really anything else that can read from InfluxDB. These metrics can be
used to track application performance over time, between different Ruby
versions, different GitLab versions, etc.

== Transaction Metrics

Currently the following is tracked on a per transaction basis (a
transaction is a Rails request or a single Sidekiq job):

* Timings per query along with the raw (obfuscated) SQL and information
  about what file the query originated from.
* Timings per view along with the path of the view and information about
  what file triggered the rendering process.
* The duration of a request itself along with the controller/worker
  class and method name.
* The duration of any instrumented method calls (more below).

== Sampled Metrics

Certain metrics can't be directly associated with a transaction. For
example, a process' total memory usage is unrelated to any running
transactions. While a transaction can result in the memory usage going
up there's no accurate way to determine what transaction is to blame,
this becomes especially problematic in multi-threaded environments.

To solve this problem there's a separate thread that takes samples at a
fixed interval. This thread (using the class Gitlab::Metrics::Sampler)
currently tracks the following:

* The process' total memory usage.
* The number of file descriptors opened by the process.
* The amount of Ruby objects (using ObjectSpace.count_objects).
* GC statistics such as timings, heap slots, etc.

The default/current interval is 15 seconds, any smaller interval might
put too much pressure on InfluxDB (especially when running dozens of
processes).

== Method Instrumentation

While currently not yet used methods can be instrumented to track how
long they take to run. Unlike the likes of New Relic this doesn't
require modifying the source code (e.g. including modules), it all
happens from the outside. For example, to track `User.by_login` we'd add
the following code somewhere in an initializer:

    Gitlab::Metrics::Instrumentation.
      instrument_method(User, :by_login)

to instead instrument an instance method:

    Gitlab::Metrics::Instrumentation.
      instrument_instance_method(User, :save)

Instrumentation for either all public model methods or a few crucial
ones will be added in the near future, I simply haven't gotten to doing
so just yet.

== Configuration

By default metrics are disabled. This means users don't have to bother
setting anything up if they don't want to. Metrics can be enabled by
editing one's gitlab.yml configuration file (see
config/gitlab.yml.example for example settings).

== Writing Data To InfluxDB

Because InfluxDB is still a fairly young product I expect the worse.
Data loss, unexpected reboots, the database not responding, you name it.
Because of this data is _not_ written to InfluxDB directly, instead it's
queued and processed by Sidekiq. This ensures that users won't notice
anything when InfluxDB is giving trouble.

The metrics worker can be started in a standalone manner as following:

    bundle exec sidekiq -q metrics

The corresponding class is called MetricsWorker.
---
 lib/gitlab/metrics.rb                           | 52 +++++++++++++++++
 lib/gitlab/metrics/delta.rb                     | 32 ++++++++++
 lib/gitlab/metrics/instrumentation.rb           | 47 +++++++++++++++
 lib/gitlab/metrics/metric.rb                    | 34 +++++++++++
 lib/gitlab/metrics/obfuscated_sql.rb            | 39 +++++++++++++
 lib/gitlab/metrics/rack_middleware.rb           | 49 ++++++++++++++++
 lib/gitlab/metrics/sampler.rb                   | 77 +++++++++++++++++++++++++
 lib/gitlab/metrics/sidekiq_middleware.rb        | 30 ++++++++++
 lib/gitlab/metrics/subscribers/action_view.rb   | 48 +++++++++++++++
 lib/gitlab/metrics/subscribers/active_record.rb | 43 ++++++++++++++
 lib/gitlab/metrics/subscribers/method_call.rb   | 42 ++++++++++++++
 lib/gitlab/metrics/system.rb                    | 35 +++++++++++
 lib/gitlab/metrics/transaction.rb               | 66 +++++++++++++++++++++
 13 files changed, 594 insertions(+)
 create mode 100644 lib/gitlab/metrics.rb
 create mode 100644 lib/gitlab/metrics/delta.rb
 create mode 100644 lib/gitlab/metrics/instrumentation.rb
 create mode 100644 lib/gitlab/metrics/metric.rb
 create mode 100644 lib/gitlab/metrics/obfuscated_sql.rb
 create mode 100644 lib/gitlab/metrics/rack_middleware.rb
 create mode 100644 lib/gitlab/metrics/sampler.rb
 create mode 100644 lib/gitlab/metrics/sidekiq_middleware.rb
 create mode 100644 lib/gitlab/metrics/subscribers/action_view.rb
 create mode 100644 lib/gitlab/metrics/subscribers/active_record.rb
 create mode 100644 lib/gitlab/metrics/subscribers/method_call.rb
 create mode 100644 lib/gitlab/metrics/system.rb
 create mode 100644 lib/gitlab/metrics/transaction.rb

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
new file mode 100644
index 00000000000..08393995165
--- /dev/null
+++ b/lib/gitlab/metrics.rb
@@ -0,0 +1,52 @@
+module Gitlab
+  module Metrics
+    def self.pool_size
+      Settings.metrics['pool_size'] || 16
+    end
+
+    def self.timeout
+      Settings.metrics['timeout'] || 10
+    end
+
+    def self.enabled?
+      !!Settings.metrics['enabled']
+    end
+
+    def self.pool
+      @pool
+    end
+
+    def self.hostname
+      @hostname
+    end
+
+    def self.last_relative_application_frame
+      root    = Rails.root.to_s
+      metrics = Rails.root.join('lib', 'gitlab', 'metrics').to_s
+
+      frame = caller_locations.find do |l|
+        l.path.start_with?(root) && !l.path.start_with?(metrics)
+      end
+
+      if frame
+        return frame.path.gsub(/^#{Rails.root.to_s}\/?/, ''), frame.lineno
+      else
+        return nil, nil
+      end
+    end
+
+    @hostname = Socket.gethostname
+
+    # When enabled this should be set before being used as the usual pattern
+    # "@foo ||= bar" is _not_ thread-safe.
+    if enabled?
+      @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
+        db   = Settings.metrics['database']
+        user = Settings.metrics['username']
+        pw   = Settings.metrics['password']
+
+        InfluxDB::Client.new(db, username: user, password: pw)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/delta.rb b/lib/gitlab/metrics/delta.rb
new file mode 100644
index 00000000000..bcf28eed84d
--- /dev/null
+++ b/lib/gitlab/metrics/delta.rb
@@ -0,0 +1,32 @@
+module Gitlab
+  module Metrics
+    # Class for calculating the difference between two numeric values.
+    #
+    # Every call to `compared_with` updates the internal value. This makes it
+    # possible to use a single Delta instance to calculate the delta over time
+    # of an ever increasing number.
+    #
+    # Example usage:
+    #
+    #     delta = Delta.new(0)
+    #
+    #     delta.compared_with(10) # => 10
+    #     delta.compared_with(15) # => 5
+    #     delta.compared_with(20) # => 5
+    class Delta
+      def initialize(value = 0)
+        @value = value
+      end
+
+      # new_value - The value to compare with as a Numeric.
+      #
+      # Returns a new Numeric (depending on the type of `new_value`).
+      def compared_with(new_value)
+        delta  = new_value - @value
+        @value = new_value
+
+        delta
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
new file mode 100644
index 00000000000..1c2f84fb09a
--- /dev/null
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module Metrics
+    # Module for instrumenting methods.
+    #
+    # This module allows instrumenting of methods without having to actually
+    # alter the target code (e.g. by including modules).
+    #
+    # Example usage:
+    #
+    #     Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login)
+    module Instrumentation
+      # Instruments a class method.
+      #
+      # mod  - The module to instrument as a Module/Class.
+      # name - The name of the method to instrument.
+      def self.instrument_method(mod, name)
+        instrument(:class, mod, name)
+      end
+
+      # Instruments an instance method.
+      #
+      # mod  - The module to instrument as a Module/Class.
+      # name - The name of the method to instrument.
+      def self.instrument_instance_method(mod, name)
+        instrument(:instance, mod, name)
+      end
+
+      def self.instrument(type, mod, name)
+        return unless Metrics.enabled?
+
+        alias_name = "_original_#{name}"
+        target     = type == :instance ? mod : mod.singleton_class
+
+        target.class_eval do
+          alias_method(alias_name, name)
+
+          define_method(name) do |*args, &block|
+            ActiveSupport::Notifications.
+              instrument("#{type}_method.method_call", module: mod, name: name) do
+                __send__(alias_name, *args, &block)
+              end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
new file mode 100644
index 00000000000..f592f4e571f
--- /dev/null
+++ b/lib/gitlab/metrics/metric.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Metrics
+    # Class for storing details of a single metric (label, value, etc).
+    class Metric
+      attr_reader :series, :values, :tags, :created_at
+
+      # series - The name of the series (as a String) to store the metric in.
+      # values - A Hash containing the values to store.
+      # tags   - A Hash containing extra tags to add to the metrics.
+      def initialize(series, values, tags = {})
+        @values     = values
+        @series     = series
+        @tags       = tags
+        @created_at = Time.now.utc
+      end
+
+      # Returns a Hash in a format that can be directly written to InfluxDB.
+      def to_hash
+        {
+          series: @series,
+          tags:   @tags.merge(
+            hostname:       Metrics.hostname,
+            ruby_engine:    RUBY_ENGINE,
+            ruby_version:   RUBY_VERSION,
+            gitlab_version: Gitlab::VERSION,
+            process_type:   Sidekiq.server? ? 'sidekiq' : 'rails'
+          ),
+          values:    @values,
+          timestamp: @created_at.to_i
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
new file mode 100644
index 00000000000..45f2e2bc62a
--- /dev/null
+++ b/lib/gitlab/metrics/obfuscated_sql.rb
@@ -0,0 +1,39 @@
+module Gitlab
+  module Metrics
+    # Class for producing SQL queries with sensitive data stripped out.
+    class ObfuscatedSQL
+      REPLACEMENT = /
+        \d+(\.\d+)?      # integers, floats
+        | '.+?'          # single quoted strings
+        | \/.+?(?<!\\)\/ # regexps (including escaped slashes)
+      /x
+
+      MYSQL_REPLACEMENTS = /
+        ".+?" # double quoted strings
+      /x
+
+      # Regex to replace consecutive placeholders with a single one indicating
+      # the length. This can be useful when a "IN" statement uses thousands of
+      # IDs (storing this would just be a waste of space).
+      CONSECUTIVE = /(\?(\s*,\s*)?){2,}/
+
+      # sql - The raw SQL query as a String.
+      def initialize(sql)
+        @sql = sql
+      end
+
+      # Returns a new, obfuscated SQL query.
+      def to_s
+        regex = REPLACEMENT
+
+        if Gitlab::Database.mysql?
+          regex = Regexp.union(regex, MYSQL_REPLACEMENTS)
+        end
+
+        @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match|
+          "#{match.count(',') + 1} values"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
new file mode 100644
index 00000000000..5c0587c4c51
--- /dev/null
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module Metrics
+    # Rack middleware for tracking Rails requests.
+    class RackMiddleware
+      CONTROLLER_KEY = 'action_controller.instance'
+
+      def initialize(app)
+        @app = app
+      end
+
+      # env - A Hash containing Rack environment details.
+      def call(env)
+        trans  = transaction_from_env(env)
+        retval = nil
+
+        begin
+          retval = trans.run { @app.call(env) }
+
+        # Even in the event of an error we want to submit any metrics we
+        # might've gathered up to this point.
+        ensure
+          if env[CONTROLLER_KEY]
+            tag_controller(trans, env)
+          end
+
+          trans.finish
+        end
+
+        retval
+      end
+
+      def transaction_from_env(env)
+        trans = Transaction.new
+
+        trans.add_tag(:request_method, env['REQUEST_METHOD'])
+        trans.add_tag(:request_uri, env['REQUEST_URI'])
+
+        trans
+      end
+
+      def tag_controller(trans, env)
+        controller = env[CONTROLLER_KEY]
+        label      = "#{controller.class.name}##{controller.action_name}"
+
+        trans.add_tag(:action, label)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
new file mode 100644
index 00000000000..141953dc985
--- /dev/null
+++ b/lib/gitlab/metrics/sampler.rb
@@ -0,0 +1,77 @@
+module Gitlab
+  module Metrics
+    # Class that sends certain metrics to InfluxDB at a specific interval.
+    #
+    # This class is used to gather statistics that can't be directly associated
+    # with a transaction such as system memory usage, garbage collection
+    # statistics, etc.
+    class Sampler
+      # interval - The sampling interval in seconds.
+      def initialize(interval = 15)
+        @interval = interval
+        @metrics  = []
+
+        @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
+        @last_major_gc = Delta.new(GC.stat[:major_gc_count])
+      end
+
+      def start
+        Thread.new do
+          Thread.current.abort_on_exception = true
+
+          loop do
+            sleep(@interval)
+
+            sample
+          end
+        end
+      end
+
+      def sample
+        sample_memory_usage
+        sample_file_descriptors
+        sample_objects
+        sample_gc
+
+        flush
+      ensure
+        GC::Profiler.clear
+        @metrics.clear
+      end
+
+      def flush
+        MetricsWorker.perform_async(@metrics.map(&:to_hash))
+      end
+
+      def sample_memory_usage
+        @metrics << Metric.new('memory_usage', value: System.memory_usage)
+      end
+
+      def sample_file_descriptors
+        @metrics << Metric.
+          new('file_descriptors', value: System.file_descriptor_count)
+      end
+
+      def sample_objects
+        @metrics << Metric.new('object_counts', ObjectSpace.count_objects)
+      end
+
+      def sample_gc
+        time  = GC::Profiler.total_time * 1000.0
+        stats = GC.stat.merge(total_time: time)
+
+        # We want the difference of GC runs compared to the last sample, not the
+        # total amount since the process started.
+        stats[:minor_gc_count] =
+          @last_minor_gc.compared_with(stats[:minor_gc_count])
+
+        stats[:major_gc_count] =
+          @last_major_gc.compared_with(stats[:major_gc_count])
+
+        stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count]
+
+        @metrics << Metric.new('gc_statistics', stats)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
new file mode 100644
index 00000000000..ec10707d1fb
--- /dev/null
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Metrics
+    # Sidekiq middleware for tracking jobs.
+    #
+    # This middleware is intended to be used as a server-side middleware.
+    class SidekiqMiddleware
+      def call(worker, message, queue)
+        # We don't want to track the MetricsWorker itself as otherwise we'll end
+        # up in an infinite loop.
+        if worker.class == MetricsWorker
+          yield
+          return
+        end
+
+        trans = Transaction.new
+
+        begin
+          trans.run { yield }
+        ensure
+          tag_worker(trans, worker)
+          trans.finish
+        end
+      end
+
+      def tag_worker(trans, worker)
+        trans.add_tag(:action, "#{worker.class.name}#perform")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
new file mode 100644
index 00000000000..2e88e4bea6a
--- /dev/null
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -0,0 +1,48 @@
+module Gitlab
+  module Metrics
+    module Subscribers
+      # Class for tracking the rendering timings of views.
+      class ActionView < ActiveSupport::Subscriber
+        attach_to :action_view
+
+        SERIES = 'views'
+
+        def render_template(event)
+          track(event) if current_transaction
+        end
+
+        alias_method :render_view, :render_template
+
+        private
+
+        def track(event)
+          path   = relative_path(event.payload[:identifier])
+          values = values_for(event)
+
+          current_transaction.add_metric(SERIES, values, path: path)
+        end
+
+        def relative_path(path)
+          path.gsub(/^#{Rails.root.to_s}\/?/, '')
+        end
+
+        def values_for(event)
+          values = { duration: event.duration }
+
+          file, line = Metrics.last_relative_application_frame
+
+          if file and line
+            values[:file] = file
+            values[:line] = line
+          end
+
+          values
+        end
+
+        def current_transaction
+          Transaction.current
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
new file mode 100644
index 00000000000..3cc9b1addf6
--- /dev/null
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module Metrics
+    module Subscribers
+      # Class for tracking raw SQL queries.
+      #
+      # Queries are obfuscated before being logged to ensure no private data is
+      # exposed via InfluxDB/Grafana.
+      class ActiveRecord < ActiveSupport::Subscriber
+        attach_to :active_record
+
+        SERIES = 'sql_queries'
+
+        def sql(event)
+          return unless current_transaction
+
+          sql    = ObfuscatedSQL.new(event.payload[:sql]).to_s
+          values = values_for(event)
+
+          current_transaction.add_metric(SERIES, values, sql: sql)
+        end
+
+        private
+
+        def values_for(event)
+          values = { duration: event.duration }
+
+          file, line = Metrics.last_relative_application_frame
+
+          if file and line
+            values[:file] = file
+            values[:line] = line
+          end
+
+          values
+        end
+
+        def current_transaction
+          Transaction.current
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/subscribers/method_call.rb b/lib/gitlab/metrics/subscribers/method_call.rb
new file mode 100644
index 00000000000..1606134b7e5
--- /dev/null
+++ b/lib/gitlab/metrics/subscribers/method_call.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module Metrics
+    module Subscribers
+      # Class for tracking method call timings.
+      class MethodCall < ActiveSupport::Subscriber
+        attach_to :method_call
+
+        SERIES = 'method_calls'
+
+        def instance_method(event)
+          return unless current_transaction
+
+          label = "#{event.payload[:module].name}##{event.payload[:name]}"
+
+          add_metric(label, event.duration)
+        end
+
+        def class_method(event)
+          return unless current_transaction
+
+          label = "#{event.payload[:module].name}.#{event.payload[:name]}"
+
+          add_metric(label, event.duration)
+        end
+
+        private
+
+        def add_metric(label, duration)
+          file, line = Metrics.last_relative_application_frame
+
+          values = { duration: duration, file: file, line: line }
+
+          current_transaction.add_metric(SERIES, values, method: label)
+        end
+
+        def current_transaction
+          Transaction.current
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
new file mode 100644
index 00000000000..83371265278
--- /dev/null
+++ b/lib/gitlab/metrics/system.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Metrics
+    # Module for gathering system/process statistics such as the memory usage.
+    #
+    # This module relies on the /proc filesystem being available. If /proc is
+    # not available the methods of this module will be stubbed.
+    module System
+      if File.exist?('/proc')
+        # Returns the current process' memory usage in bytes.
+        def self.memory_usage
+          mem   = 0
+          match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/)
+
+          if match and match[1]
+            mem = match[1].to_f * 1024
+          end
+
+          mem
+        end
+
+        def self.file_descriptor_count
+          Dir.glob('/proc/self/fd/*').length
+        end
+      else
+        def self.memory_usage
+          0.0
+        end
+
+        def self.file_descriptor_count
+          0
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
new file mode 100644
index 00000000000..568f9d6ae0c
--- /dev/null
+++ b/lib/gitlab/metrics/transaction.rb
@@ -0,0 +1,66 @@
+module Gitlab
+  module Metrics
+    # Class for storing metrics information of a single transaction.
+    class Transaction
+      THREAD_KEY = :_gitlab_metrics_transaction
+
+      SERIES = 'transactions'
+
+      attr_reader :uuid, :tags
+
+      def self.current
+        Thread.current[THREAD_KEY]
+      end
+
+      # name - The name of this transaction as a String.
+      def initialize
+        @metrics = []
+        @uuid    = SecureRandom.uuid
+
+        @started_at  = nil
+        @finished_at = nil
+
+        @tags = {}
+      end
+
+      def duration
+        @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0
+      end
+
+      def run
+        Thread.current[THREAD_KEY] = self
+
+        @started_at = Time.now
+
+        yield
+      ensure
+        @finished_at = Time.now
+
+        Thread.current[THREAD_KEY] = nil
+      end
+
+      def add_metric(series, values, tags = {})
+        tags = tags.merge(transaction_id: @uuid)
+
+        @metrics << Metric.new(series, values, tags)
+      end
+
+      def add_tag(key, value)
+        @tags[key] = value
+      end
+
+      def finish
+        track_self
+        submit
+      end
+
+      def track_self
+        add_metric(SERIES, { duration: duration }, @tags)
+      end
+
+      def submit
+        MetricsWorker.perform_async(@metrics.map(&:to_hash))
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 60a6a240ea21cbfa564b9373b1c3bb57316aae46 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 13:25:16 +0100
Subject: Improved last_relative_application_frame timings

The previous setup wasn't exactly fast, resulting in instrumented method
calls taking about 600 times longer than non instrumented calls
(including any ActiveSupport code involved). With this commit this
slowdown has been reduced to around 185 times.
---
 lib/gitlab/metrics.rb | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 08393995165..007429fa194 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,5 +1,9 @@
 module Gitlab
   module Metrics
+    RAILS_ROOT   = Rails.root.to_s
+    METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
+    PATH_REGEX   = /^#{RAILS_ROOT}\/?/
+
     def self.pool_size
       Settings.metrics['pool_size'] || 16
     end
@@ -20,16 +24,15 @@ module Gitlab
       @hostname
     end
 
+    # Returns a relative path and line number based on the last application call
+    # frame.
     def self.last_relative_application_frame
-      root    = Rails.root.to_s
-      metrics = Rails.root.join('lib', 'gitlab', 'metrics').to_s
-
       frame = caller_locations.find do |l|
-        l.path.start_with?(root) && !l.path.start_with?(metrics)
+        l.path.start_with?(RAILS_ROOT) && !l.path.start_with?(METRICS_ROOT)
       end
 
       if frame
-        return frame.path.gsub(/^#{Rails.root.to_s}\/?/, ''), frame.lineno
+        return frame.path.sub(PATH_REGEX, ''), frame.lineno
       else
         return nil, nil
       end
-- 
cgit v1.2.1


From b66a16c8384b64eabeb04f3f32017581e4711eb8 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 13:38:45 +0100
Subject: Use string evaluation for method instrumentation

This is faster than using define_method since we don't have to keep
block bindings around.
---
 lib/gitlab/metrics/instrumentation.rb         | 16 +++++++++-------
 lib/gitlab/metrics/subscribers/method_call.rb |  4 ++--
 2 files changed, 11 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 1c2f84fb09a..982e35adfc9 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -31,16 +31,18 @@ module Gitlab
         alias_name = "_original_#{name}"
         target     = type == :instance ? mod : mod.singleton_class
 
-        target.class_eval do
-          alias_method(alias_name, name)
+        target.class_eval <<-EOF, __FILE__, __LINE__ + 1
+          alias_method :#{alias_name}, :#{name}
 
-          define_method(name) do |*args, &block|
-            ActiveSupport::Notifications.
-              instrument("#{type}_method.method_call", module: mod, name: name) do
-                __send__(alias_name, *args, &block)
+          def #{name}(*args, &block)
+            ActiveSupport::Notifications
+              .instrument("#{type}_method.method_call",
+                          module: #{mod.name.inspect},
+                          name: #{name.inspect}) do
+                #{alias_name}(*args, &block)
               end
           end
-        end
+        EOF
       end
     end
   end
diff --git a/lib/gitlab/metrics/subscribers/method_call.rb b/lib/gitlab/metrics/subscribers/method_call.rb
index 1606134b7e5..0094ed0dc6a 100644
--- a/lib/gitlab/metrics/subscribers/method_call.rb
+++ b/lib/gitlab/metrics/subscribers/method_call.rb
@@ -10,7 +10,7 @@ module Gitlab
         def instance_method(event)
           return unless current_transaction
 
-          label = "#{event.payload[:module].name}##{event.payload[:name]}"
+          label = "#{event.payload[:module]}##{event.payload[:name]}"
 
           add_metric(label, event.duration)
         end
@@ -18,7 +18,7 @@ module Gitlab
         def class_method(event)
           return unless current_transaction
 
-          label = "#{event.payload[:module].name}.#{event.payload[:name]}"
+          label = "#{event.payload[:module]}.#{event.payload[:name]}"
 
           add_metric(label, event.duration)
         end
-- 
cgit v1.2.1


From 1b077d2d81bd25fe37492ea56c8bd884f944ce52 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 15:16:56 +0100
Subject: Use custom code for instrumenting method calls

The use of ActiveSupport would slow down instrumented method calls by
about 180x due to:

1. ActiveSupport itself not being the fastest thing on the planet
2. caller_locations() having quite some overhead

The use of caller_locations() has been removed because it's not _that_
useful since we already know the full namespace of receivers and the
names of the called methods.

The use of ActiveSupport has been replaced with some custom code that's
generated using eval() (which can be quite a bit faster than using
define_method).

This new setup results in instrumented methods only being about 35-40x
slower (compared to non instrumented methods).
---
 lib/gitlab/metrics/instrumentation.rb         | 37 +++++++++++++++++++----
 lib/gitlab/metrics/subscribers/method_call.rb | 42 ---------------------------
 2 files changed, 31 insertions(+), 48 deletions(-)
 delete mode 100644 lib/gitlab/metrics/subscribers/method_call.rb

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 982e35adfc9..5b56f2e8513 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -9,6 +9,8 @@ module Gitlab
     #
     #     Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login)
     module Instrumentation
+      SERIES = 'method_calls'
+
       # Instruments a class method.
       #
       # mod  - The module to instrument as a Module/Class.
@@ -31,19 +33,42 @@ module Gitlab
         alias_name = "_original_#{name}"
         target     = type == :instance ? mod : mod.singleton_class
 
+        if type == :instance
+          target = mod
+          label  = "#{mod.name}##{name}"
+        else
+          target = mod.singleton_class
+          label  = "#{mod.name}.#{name}"
+        end
+
         target.class_eval <<-EOF, __FILE__, __LINE__ + 1
           alias_method :#{alias_name}, :#{name}
 
           def #{name}(*args, &block)
-            ActiveSupport::Notifications
-              .instrument("#{type}_method.method_call",
-                          module: #{mod.name.inspect},
-                          name: #{name.inspect}) do
-                #{alias_name}(*args, &block)
-              end
+            trans = Gitlab::Metrics::Instrumentation.transaction
+
+            if trans
+              start    = Time.now
+              retval   = #{alias_name}(*args, &block)
+              duration = (Time.now - start) * 1000.0
+
+              trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
+                               { duration: duration },
+                               method: #{label.inspect})
+
+              retval
+            else
+              #{alias_name}(*args, &block)
+            end
           end
         EOF
       end
+
+      # Small layer of indirection to make it easier to stub out the current
+      # transaction.
+      def self.transaction
+        Transaction.current
+      end
     end
   end
 end
diff --git a/lib/gitlab/metrics/subscribers/method_call.rb b/lib/gitlab/metrics/subscribers/method_call.rb
deleted file mode 100644
index 0094ed0dc6a..00000000000
--- a/lib/gitlab/metrics/subscribers/method_call.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-module Gitlab
-  module Metrics
-    module Subscribers
-      # Class for tracking method call timings.
-      class MethodCall < ActiveSupport::Subscriber
-        attach_to :method_call
-
-        SERIES = 'method_calls'
-
-        def instance_method(event)
-          return unless current_transaction
-
-          label = "#{event.payload[:module]}##{event.payload[:name]}"
-
-          add_metric(label, event.duration)
-        end
-
-        def class_method(event)
-          return unless current_transaction
-
-          label = "#{event.payload[:module]}.#{event.payload[:name]}"
-
-          add_metric(label, event.duration)
-        end
-
-        private
-
-        def add_metric(label, duration)
-          file, line = Metrics.last_relative_application_frame
-
-          values = { duration: duration, file: file, line: line }
-
-          current_transaction.add_metric(SERIES, values, method: label)
-        end
-
-        def current_transaction
-          Transaction.current
-        end
-      end
-    end
-  end
-end
-- 
cgit v1.2.1


From ad69ba57d6e7d52b2a44a20393c072538c299653 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 16:15:02 +0100
Subject: Proper method instrumentation for special symbols

This ensures that methods such as "==" can be instrumented without
producing syntax errors.
---
 lib/gitlab/metrics/instrumentation.rb | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 5b56f2e8513..2ed75495650 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -30,7 +30,8 @@ module Gitlab
       def self.instrument(type, mod, name)
         return unless Metrics.enabled?
 
-        alias_name = "_original_#{name}"
+        name       = name.to_sym
+        alias_name = :"_original_#{name}"
         target     = type == :instance ? mod : mod.singleton_class
 
         if type == :instance
@@ -42,14 +43,14 @@ module Gitlab
         end
 
         target.class_eval <<-EOF, __FILE__, __LINE__ + 1
-          alias_method :#{alias_name}, :#{name}
+          alias_method #{alias_name.inspect}, #{name.inspect}
 
           def #{name}(*args, &block)
             trans = Gitlab::Metrics::Instrumentation.transaction
 
             if trans
               start    = Time.now
-              retval   = #{alias_name}(*args, &block)
+              retval   =
               duration = (Time.now - start) * 1000.0
 
               trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
@@ -58,7 +59,7 @@ module Gitlab
 
               retval
             else
-              #{alias_name}(*args, &block)
+              __send__(#{alias_name.inspect}, *args, &block)
             end
           end
         EOF
-- 
cgit v1.2.1


From 5dbcb635a17aff6543150a66b597c75b819801e2 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 16:15:46 +0100
Subject: Methods for instrumenting multiple methods

The methods Instrumentation.instrument_methods and
Instrumentation.instrument_instance_methods can be used to instrument
all methods of a module at once.
---
 lib/gitlab/metrics/instrumentation.rb | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 2ed75495650..12d5ede4be3 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -27,6 +27,29 @@ module Gitlab
         instrument(:instance, mod, name)
       end
 
+      # Instruments all public methods of a module.
+      #
+      # mod - The module to instrument.
+      def self.instrument_methods(mod)
+        mod.public_methods(false).each do |name|
+          instrument_method(mod, name)
+        end
+      end
+
+      # Instruments all public instance methods of a module.
+      #
+      # mod - The module to instrument.
+      def self.instrument_instance_methods(mod)
+        mod.public_instance_methods(false).each do |name|
+          instrument_instance_method(mod, name)
+        end
+      end
+
+      # Instruments a method.
+      #
+      # type - The type (:class or :instance) of method to instrument.
+      # mod  - The module containing the method.
+      # name - The name of the method to instrument.
       def self.instrument(type, mod, name)
         return unless Metrics.enabled?
 
-- 
cgit v1.2.1


From f43f3b89a633b5ceee4e71acba0c83ed5cb28963 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 16:16:48 +0100
Subject: Added Instrumentation.configure

This makes it easier to instrument multiple modules without having to
type the full namespace over and over again.
---
 lib/gitlab/metrics/instrumentation.rb | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 12d5ede4be3..8822a907967 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -11,6 +11,10 @@ module Gitlab
     module Instrumentation
       SERIES = 'method_calls'
 
+      def self.configure
+        yield self
+      end
+
       # Instruments a class method.
       #
       # mod  - The module to instrument as a Module/Class.
-- 
cgit v1.2.1


From 641761f1d6af5a94c0007e8d7463ee86fc047229 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 16:25:28 +0100
Subject: Only instrument methods defined directly

When using instrument_methods/instrument_instance_methods we only want
to instrument methods defined directly in a class, not those included
via mixins (e.g. whatever RSpec throws in during development).

In case an externally included method _has_ to be instrumented we can
still use the regular instrument_method/instrument_instance_method
methods.
---
 lib/gitlab/metrics/instrumentation.rb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 8822a907967..cf22aa93cdd 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -36,7 +36,9 @@ module Gitlab
       # mod - The module to instrument.
       def self.instrument_methods(mod)
         mod.public_methods(false).each do |name|
-          instrument_method(mod, name)
+          method = mod.method(name)
+
+          instrument_method(mod, name) if method.owner == mod.singleton_class
         end
       end
 
@@ -45,7 +47,9 @@ module Gitlab
       # mod - The module to instrument.
       def self.instrument_instance_methods(mod)
         mod.public_instance_methods(false).each do |name|
-          instrument_instance_method(mod, name)
+          method = mod.instance_method(name)
+
+          instrument_instance_method(mod, name) if method.owner == mod
         end
       end
 
@@ -77,7 +81,7 @@ module Gitlab
 
             if trans
               start    = Time.now
-              retval   =
+              retval   = __send__(#{alias_name.inspect}, *args, &block)
               duration = (Time.now - start) * 1000.0
 
               trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
-- 
cgit v1.2.1


From 09a311568abee739fae0c2577a9cf6aa01516977 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 10 Dec 2015 17:48:14 +0100
Subject: Track object count types as tags

---
 lib/gitlab/metrics/sampler.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 141953dc985..03afa6324dd 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -53,7 +53,9 @@ module Gitlab
       end
 
       def sample_objects
-        @metrics << Metric.new('object_counts', ObjectSpace.count_objects)
+        ObjectSpace.count_objects.each do |type, count|
+          @metrics << Metric.new('object_counts', { count: count }, type: type)
+        end
       end
 
       def sample_gc
-- 
cgit v1.2.1


From f932b781a79d9b829001c39bc214372b7efd8610 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 14 Dec 2015 12:31:56 +0100
Subject: Replace double quotes when obfuscating SQL

InfluxDB escapes double quotes upon output which makes it a pain to deal
with. This ensures that if we're using PostgreSQL we don't store any
queries containing double quotes in InfluxDB, solving the escaping
problem.
---
 lib/gitlab/metrics/obfuscated_sql.rb | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
index 45f2e2bc62a..7b15670aa6b 100644
--- a/lib/gitlab/metrics/obfuscated_sql.rb
+++ b/lib/gitlab/metrics/obfuscated_sql.rb
@@ -30,9 +30,17 @@ module Gitlab
           regex = Regexp.union(regex, MYSQL_REPLACEMENTS)
         end
 
-        @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match|
+        sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match|
           "#{match.count(',') + 1} values"
         end
+
+        # InfluxDB escapes double quotes upon output, so lets get rid of them
+        # whenever we can.
+        if Gitlab::Database.postgresql?
+          sql = sql.gsub('"', '')
+        end
+
+        sql
       end
     end
   end
-- 
cgit v1.2.1


From 9f95ff0d90802467a04816f1d38e30770a026820 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 14 Dec 2015 16:51:38 +0100
Subject: Track location information as tags

This allows the information to be displayed when using certain functions
(e.g. top()) as well as making it easier to aggregate on a per file
basis.
---
 lib/gitlab/metrics/subscribers/action_view.rb   | 17 +++++++++++------
 lib/gitlab/metrics/subscribers/active_record.rb | 17 +++++++++++------
 2 files changed, 22 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 2e88e4bea6a..7e0dcf99d92 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -16,10 +16,10 @@ module Gitlab
         private
 
         def track(event)
-          path   = relative_path(event.payload[:identifier])
           values = values_for(event)
+          tags   = tags_for(event)
 
-          current_transaction.add_metric(SERIES, values, path: path)
+          current_transaction.add_metric(SERIES, values, tags)
         end
 
         def relative_path(path)
@@ -27,16 +27,21 @@ module Gitlab
         end
 
         def values_for(event)
-          values = { duration: event.duration }
+          { duration: event.duration }
+        end
+
+        def tags_for(event)
+          path = relative_path(event.payload[:identifier])
+          tags = { view: path }
 
           file, line = Metrics.last_relative_application_frame
 
           if file and line
-            values[:file] = file
-            values[:line] = line
+            tags[:file] = file
+            tags[:line] = line
           end
 
-          values
+          tags
         end
 
         def current_transaction
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 3cc9b1addf6..d947c128ce2 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -13,25 +13,30 @@ module Gitlab
         def sql(event)
           return unless current_transaction
 
-          sql    = ObfuscatedSQL.new(event.payload[:sql]).to_s
           values = values_for(event)
+          tags   = tags_for(event)
 
-          current_transaction.add_metric(SERIES, values, sql: sql)
+          current_transaction.add_metric(SERIES, values, tags)
         end
 
         private
 
         def values_for(event)
-          values = { duration: event.duration }
+          { duration: event.duration }
+        end
+
+        def tags_for(event)
+          sql  = ObfuscatedSQL.new(event.payload[:sql]).to_s
+          tags = { sql: sql }
 
           file, line = Metrics.last_relative_application_frame
 
           if file and line
-            values[:file] = file
-            values[:line] = line
+            tags[:file] = file
+            tags[:line] = line
           end
 
-          values
+          tags
         end
 
         def current_transaction
-- 
cgit v1.2.1


From 13dbd663acbbe91ddac77b650a90377cd12c54b8 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Tue, 15 Dec 2015 16:31:24 +0100
Subject: Allow filtering of what methods to instrument

This makes it possible to determine if a method should be instrumented
or not using a block.
---
 lib/gitlab/metrics/instrumentation.rb | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index cf22aa93cdd..91e09694cd8 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -33,23 +33,38 @@ module Gitlab
 
       # Instruments all public methods of a module.
       #
+      # This method optionally takes a block that can be used to determine if a
+      # method should be instrumented or not. The block is passed the receiving
+      # module and an UnboundMethod. If the block returns a non truthy value the
+      # method is not instrumented.
+      #
       # mod - The module to instrument.
       def self.instrument_methods(mod)
         mod.public_methods(false).each do |name|
           method = mod.method(name)
 
-          instrument_method(mod, name) if method.owner == mod.singleton_class
+          if method.owner == mod.singleton_class
+            if !block_given? || block_given? && yield(mod, method)
+              instrument_method(mod, name)
+            end
+          end
         end
       end
 
       # Instruments all public instance methods of a module.
       #
+      # See `instrument_methods` for more information.
+      #
       # mod - The module to instrument.
       def self.instrument_instance_methods(mod)
         mod.public_instance_methods(false).each do |name|
           method = mod.instance_method(name)
 
-          instrument_instance_method(mod, name) if method.owner == mod
+          if method.owner == mod
+            if !block_given? || block_given? && yield(mod, method)
+              instrument_instance_method(mod, name)
+            end
+          end
         end
       end
 
-- 
cgit v1.2.1


From a41287d8989d7d49b405fd8f658d6c6e4edfd307 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Tue, 15 Dec 2015 16:38:25 +0100
Subject: Only track method calls above a certain threshold

This ensures we don't end up wasting resources by tracking method calls
that only take a few microseconds. By default the threshold is 10
milliseconds but this can be changed using the gitlab.yml configuration
file.
---
 lib/gitlab/metrics.rb                 | 4 ++++
 lib/gitlab/metrics/instrumentation.rb | 8 +++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 007429fa194..4b92c3244fa 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -16,6 +16,10 @@ module Gitlab
       !!Settings.metrics['enabled']
     end
 
+    def self.method_call_threshold
+      Settings.metrics['method_call_threshold'] || 10
+    end
+
     def self.pool
       @pool
     end
diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 91e09694cd8..ca2dffbc46a 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -99,9 +99,11 @@ module Gitlab
               retval   = __send__(#{alias_name.inspect}, *args, &block)
               duration = (Time.now - start) * 1000.0
 
-              trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
-                               { duration: duration },
-                               method: #{label.inspect})
+              if duration >= Gitlab::Metrics.method_call_threshold
+                trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
+                                 { duration: duration },
+                                 method: #{label.inspect})
+              end
 
               retval
             else
-- 
cgit v1.2.1


From a93a32a290c8e134763188ebd2b62935f5698e6c Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Tue, 15 Dec 2015 17:22:46 +0100
Subject: Support for instrumenting class hierarchies

This will be used to (for example) instrument all ActiveRecord models.
---
 lib/gitlab/metrics/instrumentation.rb | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index ca2dffbc46a..06fc2f25948 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -31,6 +31,29 @@ module Gitlab
         instrument(:instance, mod, name)
       end
 
+      # Recursively instruments all subclasses of the given root module.
+      #
+      # This can be used to for example instrument all ActiveRecord models (as
+      # these all inherit from ActiveRecord::Base).
+      #
+      # This method can optionally take a block to pass to `instrument_methods`
+      # and `instrument_instance_methods`.
+      #
+      # root - The root module for which to instrument subclasses. The root
+      #        module itself is not instrumented.
+      def self.instrument_class_hierarchy(root, &block)
+        visit = root.subclasses
+
+        until visit.empty?
+          klass = visit.pop
+
+          instrument_methods(klass, &block)
+          instrument_instance_methods(klass, &block)
+
+          klass.subclasses.each { |c| visit << c }
+        end
+      end
+
       # Instruments all public methods of a module.
       #
       # This method optionally takes a block that can be used to determine if a
-- 
cgit v1.2.1


From f181f05e8abd7b1066c11578193f6d7170764bf5 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 17 Dec 2015 17:17:18 +0100
Subject: Track object counts using the "allocations" Gem

This allows us to track the counts of actual classes instead of "T_XXX"
nodes. This is only enabled on CRuby as it uses CRuby specific APIs.
---
 lib/gitlab/metrics.rb         |  4 ++++
 lib/gitlab/metrics/sampler.rb | 25 ++++++++++++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 4b92c3244fa..ce89be636d3 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -16,6 +16,10 @@ module Gitlab
       !!Settings.metrics['enabled']
     end
 
+    def self.mri?
+      RUBY_ENGINE == 'ruby'
+    end
+
     def self.method_call_threshold
       Settings.metrics['method_call_threshold'] || 10
     end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 03afa6324dd..828ee1f8c62 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -13,6 +13,12 @@ module Gitlab
 
         @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
         @last_major_gc = Delta.new(GC.stat[:major_gc_count])
+
+        if Gitlab::Metrics.mri?
+          require 'allocations'
+
+          Allocations.start
+        end
       end
 
       def start
@@ -52,9 +58,22 @@ module Gitlab
           new('file_descriptors', value: System.file_descriptor_count)
       end
 
-      def sample_objects
-        ObjectSpace.count_objects.each do |type, count|
-          @metrics << Metric.new('object_counts', { count: count }, type: type)
+      if Metrics.mri?
+        def sample_objects
+          sample = Allocations.to_hash
+          counts = sample.each_with_object({}) do |(klass, count), hash|
+            hash[klass.name] = count
+          end
+
+          # Symbols aren't allocated so we'll need to add those manually.
+          counts['Symbol'] = Symbol.all_symbols.length
+
+          counts.each do |name, count|
+            @metrics << Metric.new('object_counts', { count: count }, type: name)
+          end
+        end
+      else
+        def sample_objects
         end
       end
 
-- 
cgit v1.2.1


From 672cbbff959c66ba10740ed17671d93060410d93 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 24 Dec 2015 15:33:51 +0100
Subject: Only allow group/project members to mention `@all`

---
 lib/banzai/filter/redactor_filter.rb           |  6 +++---
 lib/banzai/filter/reference_filter.rb          |  6 +++++-
 lib/banzai/filter/reference_gatherer_filter.rb |  8 +++++++-
 lib/banzai/filter/user_reference_filter.rb     | 14 +++++++++++++-
 lib/gitlab/reference_extractor.rb              | 19 ++++++++++++-------
 5 files changed, 40 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index 89e7a79789a..f01a32b5ae5 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -11,7 +11,7 @@ module Banzai
     class RedactorFilter < HTML::Pipeline::Filter
       def call
         doc.css('a.gfm').each do |node|
-          unless user_can_reference?(node)
+          unless user_can_see_reference?(node)
             # The reference should be replaced by the original text,
             # which is not always the same as the rendered text.
             text = node.attr('data-original') || node.text
@@ -24,12 +24,12 @@ module Banzai
 
       private
 
-      def user_can_reference?(node)
+      def user_can_see_reference?(node)
         if node.has_attribute?('data-reference-filter')
           reference_type = node.attr('data-reference-filter')
           reference_filter = Banzai::Filter.const_get(reference_type)
 
-          reference_filter.user_can_reference?(current_user, node, context)
+          reference_filter.user_can_see_reference?(current_user, node, context)
         else
           true
         end
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 33457a3f361..5275ac8bf95 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -12,7 +12,7 @@ module Banzai
     #   :project (required) - Current project, ignored if reference is cross-project.
     #   :only_path          - Generate path-only links.
     class ReferenceFilter < HTML::Pipeline::Filter
-      def self.user_can_reference?(user, node, context)
+      def self.user_can_see_reference?(user, node, context)
         if node.has_attribute?('data-project')
           project_id = node.attr('data-project').to_i
           return true if project_id == context[:project].try(:id)
@@ -24,6 +24,10 @@ module Banzai
         end
       end
 
+      def self.user_can_reference?(user, node, context)
+        true
+      end
+
       def self.referenced_by(node)
         raise NotImplementedError, "#{self} does not implement #{__method__}"
       end
diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb
index 855f238ac1e..12412ff7ea9 100644
--- a/lib/banzai/filter/reference_gatherer_filter.rb
+++ b/lib/banzai/filter/reference_gatherer_filter.rb
@@ -35,7 +35,9 @@ module Banzai
 
         return if context[:reference_filter] && reference_filter != context[:reference_filter]
 
-        return unless reference_filter.user_can_reference?(current_user, node, context)
+        return if author && !reference_filter.user_can_reference?(author, node, context)
+
+        return unless reference_filter.user_can_see_reference?(current_user, node, context)
 
         references = reference_filter.referenced_by(node)
         return unless references
@@ -57,6 +59,10 @@ module Banzai
       def current_user
         context[:current_user]
       end
+
+      def author
+        context[:author]
+      end
     end
   end
 end
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index 67c24faf991..7ec771266ed 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -39,7 +39,7 @@ module Banzai
         end
       end
 
-      def self.user_can_reference?(user, node, context)
+      def self.user_can_see_reference?(user, node, context)
         if node.has_attribute?('data-group')
           group = Group.find(node.attr('data-group')) rescue nil
           Ability.abilities.allowed?(user, :read_group, group)
@@ -48,6 +48,18 @@ module Banzai
         end
       end
 
+      def self.user_can_reference?(user, node, context)
+        # Only team members can reference `@all`
+        if node.has_attribute?('data-project')
+          project = Project.find(node.attr('data-project')) rescue nil
+          return false unless project
+
+          user && project.team.member?(user)
+        else
+          super
+        end
+      end
+
       def call
         replace_text_nodes_matching(User.reference_pattern) do |content|
           user_link_filter(content)
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index 0a70d21b1ce..be795649e59 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -3,11 +3,12 @@ require 'banzai'
 module Gitlab
   # Extract possible GFM references from an arbitrary String for further processing.
   class ReferenceExtractor < Banzai::ReferenceExtractor
-    attr_accessor :project, :current_user
+    attr_accessor :project, :current_user, :author
 
-    def initialize(project, current_user = nil)
+    def initialize(project, current_user = nil, author = nil)
       @project = project
       @current_user = current_user
+      @author = author
 
       @references = {}
 
@@ -20,18 +21,22 @@ module Gitlab
 
     %i(user label merge_request snippet commit commit_range).each do |type|
       define_method("#{type}s") do
-        @references[type] ||= references(type, project: project, current_user: current_user)
+        @references[type] ||= references(type, reference_context)
       end
     end
 
     def issues
-      options = { project: project, current_user: current_user }
-
       if project && project.jira_tracker?
-        @references[:external_issue] ||= references(:external_issue, options)
+        @references[:external_issue] ||= references(:external_issue, reference_context)
       else
-        @references[:issue] ||= references(:issue, options)
+        @references[:issue] ||= references(:issue, reference_context)
       end
     end
+
+    private
+
+    def reference_context
+      { project: project, current_user: current_user, author: author }
+    end
   end
 end
-- 
cgit v1.2.1


From 83d42c1518274dc0af0f49fda3a10e846569cbcc Mon Sep 17 00:00:00 2001
From: Valery Sizov <vsv2711@gmail.com>
Date: Fri, 25 Dec 2015 18:13:55 +0200
Subject: Revert upvotes and downvotes params to MR API

---
 lib/api/entities.rb | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index f8511ac5f5c..26e7c956e8f 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -166,7 +166,6 @@ module API
 
     class MergeRequest < ProjectEntity
       expose :target_branch, :source_branch
-      # deprecated, always returns 0
       expose :upvotes,  :downvotes
       expose :author, :assignee, using: Entities::UserBasic
       expose :source_project_id, :target_project_id
-- 
cgit v1.2.1


From ddca57d3f25e52e5e4061dd3610f1269efe2400d Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 28 Dec 2015 11:33:40 +0100
Subject: Use String#delete for removing double quotes

---
 lib/gitlab/metrics/obfuscated_sql.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
index 7b15670aa6b..481aca56efb 100644
--- a/lib/gitlab/metrics/obfuscated_sql.rb
+++ b/lib/gitlab/metrics/obfuscated_sql.rb
@@ -37,7 +37,7 @@ module Gitlab
         # InfluxDB escapes double quotes upon output, so lets get rid of them
         # whenever we can.
         if Gitlab::Database.postgresql?
-          sql = sql.gsub('"', '')
+          sql = sql.delete('"')
         end
 
         sql
-- 
cgit v1.2.1


From 1be5668ae0e663015d384ea7d8b404f9eeb5b478 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 28 Dec 2015 13:14:48 +0100
Subject: Added host option for InfluxDB

---
 lib/gitlab/metrics.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index ce89be636d3..d6f60732455 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -52,11 +52,12 @@ module Gitlab
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
       @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
+        host = Settings.metrics['host']
         db   = Settings.metrics['database']
         user = Settings.metrics['username']
         pw   = Settings.metrics['password']
 
-        InfluxDB::Client.new(db, username: user, password: pw)
+        InfluxDB::Client.new(db, host: host, username: user, password: pw)
       end
     end
   end
-- 
cgit v1.2.1


From 4d925f2147884812e349031a19f0d7ced9d5fdaf Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 28 Dec 2015 18:00:32 +0100
Subject: Move InfluxDB settings to ApplicationSetting

---
 lib/gitlab/metrics.rb | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index d6f60732455..8039e8e9e9d 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -4,16 +4,29 @@ module Gitlab
     METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
     PATH_REGEX   = /^#{RAILS_ROOT}\/?/
 
+    # Returns the current settings, ensuring we _always_ have a default set of
+    # metrics settings (even during tests, when the migrations are lacking,
+    # etc). This ensures the application is able to boot up even when the
+    # migrations have not been executed.
+    def self.settings
+      ApplicationSetting.current || {
+        metrics_pool_size:             16,
+        metrics_timeout:               10,
+        metrics_enabled:               false,
+        metrics_method_call_threshold: 10
+      }
+    end
+
     def self.pool_size
-      Settings.metrics['pool_size'] || 16
+      settings[:metrics_pool_size]
     end
 
     def self.timeout
-      Settings.metrics['timeout'] || 10
+      settings[:metrics_timeout]
     end
 
     def self.enabled?
-      !!Settings.metrics['enabled']
+      settings[:metrics_enabled]
     end
 
     def self.mri?
@@ -21,7 +34,10 @@ module Gitlab
     end
 
     def self.method_call_threshold
-      Settings.metrics['method_call_threshold'] || 10
+      # This is memoized since this method is called for every instrumented
+      # method. Loading data from an external cache on every method call slows
+      # things down too much.
+      @method_call_threshold ||= settings[:metrics_method_call_threshold]
     end
 
     def self.pool
@@ -52,10 +68,10 @@ module Gitlab
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
       @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
-        host = Settings.metrics['host']
-        db   = Settings.metrics['database']
-        user = Settings.metrics['username']
-        pw   = Settings.metrics['password']
+        host = settings[:metrics_host]
+        db   = settings[:metrics_database]
+        user = settings[:metrics_username]
+        pw   = settings[:metrics_password]
 
         InfluxDB::Client.new(db, host: host, username: user, password: pw)
       end
-- 
cgit v1.2.1


From a3469d914aaf28a1184247cbe72e5197ce7ca006 Mon Sep 17 00:00:00 2001
From: Gabriel Mazetto <gabriel@gitlab.com>
Date: Mon, 28 Dec 2015 18:21:34 -0200
Subject: reCAPTCHA is configurable through Admin Settings, no reload needed.

---
 lib/gitlab/recaptcha.rb | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 lib/gitlab/recaptcha.rb

(limited to 'lib')

diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb
new file mode 100644
index 00000000000..70e7f25d518
--- /dev/null
+++ b/lib/gitlab/recaptcha.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Recaptcha
+    def self.load_configurations!
+      if current_application_settings.recaptcha_enabled
+        ::Recaptcha.configure do |config|
+          config.public_key  = current_application_settings.recaptcha_site_key
+          config.private_key = current_application_settings.recaptcha_private_key
+        end
+
+        true
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From ed214a11ca1566582a084b9dc4b9e79470c1cc18 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 28 Dec 2015 22:14:33 +0100
Subject: Handle missing settings table for metrics

This ensures we can still boot, even when the "application_settings"
table doesn't exist.
---
 lib/gitlab/metrics.rb | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 8039e8e9e9d..9470633b065 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -9,12 +9,16 @@ module Gitlab
     # etc). This ensures the application is able to boot up even when the
     # migrations have not been executed.
     def self.settings
-      ApplicationSetting.current || {
-        metrics_pool_size:             16,
-        metrics_timeout:               10,
-        metrics_enabled:               false,
-        metrics_method_call_threshold: 10
-      }
+      if ApplicationSetting.table_exists? and curr = ApplicationSetting.current
+        curr
+      else
+        {
+          metrics_pool_size:             16,
+          metrics_timeout:               10,
+          metrics_enabled:               false,
+          metrics_method_call_threshold: 10
+        }
+      end
     end
 
     def self.pool_size
-- 
cgit v1.2.1


From 03478e6d5b98a723fbb349dac2c8495f75909a08 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Tue, 29 Dec 2015 13:40:08 +0100
Subject: Strip newlines from obfuscated SQL

Newlines aren't really needed and they may mess with InfluxDB's line
protocol.
---
 lib/gitlab/metrics/obfuscated_sql.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
index 481aca56efb..2e932fb3049 100644
--- a/lib/gitlab/metrics/obfuscated_sql.rb
+++ b/lib/gitlab/metrics/obfuscated_sql.rb
@@ -40,7 +40,7 @@ module Gitlab
           sql = sql.delete('"')
         end
 
-        sql
+        sql.gsub("\n", ' ')
       end
     end
   end
-- 
cgit v1.2.1


From 620e7bb3d60c3685b494b26e256b793a47621da4 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Tue, 29 Dec 2015 13:40:42 +0100
Subject: Write to InfluxDB directly via UDP

This removes the need for Sidekiq and any overhead/problems introduced
by TCP. There are a few things to take into account:

1. When writing data to InfluxDB you may still get an error if the
   server becomes unavailable during the write. Because of this we're
   catching all exceptions and just ignore them (for now).
2. Writing via UDP apparently requires the timestamp to be in
   nanoseconds. Without this data either isn't written properly.
3. Due to the restrictions on UDP buffer sizes we're writing metrics one
   by one, instead of writing all of them at once.
---
 lib/gitlab/metrics.rb                    | 38 ++++++++++++++++++++++++++++++--
 lib/gitlab/metrics/metric.rb             |  2 +-
 lib/gitlab/metrics/obfuscated_sql.rb     |  2 +-
 lib/gitlab/metrics/sampler.rb            |  2 +-
 lib/gitlab/metrics/sidekiq_middleware.rb |  7 ------
 lib/gitlab/metrics/transaction.rb        |  2 +-
 6 files changed, 40 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 9470633b065..0869d007ca5 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -66,6 +66,39 @@ module Gitlab
       end
     end
 
+    def self.submit_metrics(metrics)
+      prepared = prepare_metrics(metrics)
+
+      pool.with do |connection|
+        prepared.each do |metric|
+          begin
+            connection.write_points([metric])
+          rescue StandardError
+          end
+        end
+      end
+    end
+
+    def self.prepare_metrics(metrics)
+      metrics.map do |hash|
+        new_hash = hash.symbolize_keys
+
+        new_hash[:tags].each do |key, value|
+          if value.blank?
+            new_hash[:tags].delete(key)
+          else
+            new_hash[:tags][key] = escape_value(value)
+          end
+        end
+
+        new_hash
+      end
+    end
+
+    def self.escape_value(value)
+      value.to_s.gsub('=', '\\=')
+    end
+
     @hostname = Socket.gethostname
 
     # When enabled this should be set before being used as the usual pattern
@@ -73,11 +106,12 @@ module Gitlab
     if enabled?
       @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
         host = settings[:metrics_host]
-        db   = settings[:metrics_database]
         user = settings[:metrics_username]
         pw   = settings[:metrics_password]
+        port = settings[:metrics_port]
 
-        InfluxDB::Client.new(db, host: host, username: user, password: pw)
+        InfluxDB::Client.
+          new(udp: { host: host, port: port }, username: user, password: pw)
       end
     end
   end
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index f592f4e571f..79241f56874 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -26,7 +26,7 @@ module Gitlab
             process_type:   Sidekiq.server? ? 'sidekiq' : 'rails'
           ),
           values:    @values,
-          timestamp: @created_at.to_i
+          timestamp: @created_at.to_i * 1_000_000_000
         }
       end
     end
diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
index 2e932fb3049..fe97d7a0534 100644
--- a/lib/gitlab/metrics/obfuscated_sql.rb
+++ b/lib/gitlab/metrics/obfuscated_sql.rb
@@ -40,7 +40,7 @@ module Gitlab
           sql = sql.delete('"')
         end
 
-        sql.gsub("\n", ' ')
+        sql.tr("\n", ' ')
       end
     end
   end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 828ee1f8c62..998578e1c0a 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -46,7 +46,7 @@ module Gitlab
       end
 
       def flush
-        MetricsWorker.perform_async(@metrics.map(&:to_hash))
+        Metrics.submit_metrics(@metrics.map(&:to_hash))
       end
 
       def sample_memory_usage
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index ec10707d1fb..ad441decfa2 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -5,13 +5,6 @@ module Gitlab
     # This middleware is intended to be used as a server-side middleware.
     class SidekiqMiddleware
       def call(worker, message, queue)
-        # We don't want to track the MetricsWorker itself as otherwise we'll end
-        # up in an infinite loop.
-        if worker.class == MetricsWorker
-          yield
-          return
-        end
-
         trans = Transaction.new
 
         begin
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 568f9d6ae0c..a61dbd989e7 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -59,7 +59,7 @@ module Gitlab
       end
 
       def submit
-        MetricsWorker.perform_async(@metrics.map(&:to_hash))
+        Metrics.submit_metrics(@metrics.map(&:to_hash))
       end
     end
   end
-- 
cgit v1.2.1


From 701e5de9108faa65a118e8822560d39fd8ea9996 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Tue, 29 Dec 2015 15:49:12 +0100
Subject: Use Gitlab::CurrentSettings for InfluxDB

This ensures we can still start up even when not connecting to a
database.
---
 lib/gitlab/metrics.rb | 36 +++++++++++-------------------------
 1 file changed, 11 insertions(+), 25 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 0869d007ca5..2d266ccfe9e 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,36 +1,21 @@
 module Gitlab
   module Metrics
+    extend Gitlab::CurrentSettings
+
     RAILS_ROOT   = Rails.root.to_s
     METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
     PATH_REGEX   = /^#{RAILS_ROOT}\/?/
 
-    # Returns the current settings, ensuring we _always_ have a default set of
-    # metrics settings (even during tests, when the migrations are lacking,
-    # etc). This ensures the application is able to boot up even when the
-    # migrations have not been executed.
-    def self.settings
-      if ApplicationSetting.table_exists? and curr = ApplicationSetting.current
-        curr
-      else
-        {
-          metrics_pool_size:             16,
-          metrics_timeout:               10,
-          metrics_enabled:               false,
-          metrics_method_call_threshold: 10
-        }
-      end
-    end
-
     def self.pool_size
-      settings[:metrics_pool_size]
+      current_application_settings[:metrics_pool_size] || 16
     end
 
     def self.timeout
-      settings[:metrics_timeout]
+      current_application_settings[:metrics_timeout] || 10
     end
 
     def self.enabled?
-      settings[:metrics_enabled]
+      current_application_settings[:metrics_enabled] || false
     end
 
     def self.mri?
@@ -41,7 +26,8 @@ module Gitlab
       # This is memoized since this method is called for every instrumented
       # method. Loading data from an external cache on every method call slows
       # things down too much.
-      @method_call_threshold ||= settings[:metrics_method_call_threshold]
+      @method_call_threshold ||=
+        (current_application_settings[:metrics_method_call_threshold] || 10)
     end
 
     def self.pool
@@ -105,10 +91,10 @@ module Gitlab
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
       @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
-        host = settings[:metrics_host]
-        user = settings[:metrics_username]
-        pw   = settings[:metrics_password]
-        port = settings[:metrics_port]
+        host = current_application_settings[:metrics_host]
+        user = current_application_settings[:metrics_username]
+        pw   = current_application_settings[:metrics_password]
+        port = current_application_settings[:metrics_port]
 
         InfluxDB::Client.
           new(udp: { host: host, port: port }, username: user, password: pw)
-- 
cgit v1.2.1


From ff98c631c1004247656677568989e5ed68fc88f3 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Wed, 30 Dec 2015 14:26:54 -0200
Subject: Make sure that is no pending migrations in Gitlab::CurrentSettings

---
 lib/gitlab/current_settings.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 46a4ef0e31f..7a86c09158e 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -38,7 +38,9 @@ module Gitlab
                  true
                end
 
-      use_db && ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings')
+      use_db && ActiveRecord::Base.connection.active? &&
+                !ActiveRecord::Migrator.needs_migration? &&
+                ActiveRecord::Base.connection.table_exists?('application_settings')
     end
   end
 end
-- 
cgit v1.2.1


From d9d2e8a3e8de62beff685457e4e305f93122b206 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Tue, 29 Dec 2015 21:58:16 -0500
Subject: Support a single directory traversal in RelativeLinkFilter

Prior, if we were viewing a blob at
`https://example.com/namespace/project/blob/master/doc/some-file.md` and
it contained a relative link such as `[README](../README.md)`, the
resulting link when viewing the blob would be:
`https://example.com/namespace/project/blob/README.md` which omits the
`master` ref, resulting in a 404.
---
 lib/banzai/filter/relative_link_filter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index 5a081125f21..66f166939e4 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -91,7 +91,7 @@ module Banzai
         parts = request_path.split('/')
         parts.pop if path_type(request_path) != 'tree'
 
-        while parts.length > 1 && path.start_with?('../')
+        while path.start_with?('../')
           parts.pop
           path.sub!('../', '')
         end
-- 
cgit v1.2.1


From c936e4e3c8282df17f2dc89b305233b7608003ca Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 30 Dec 2015 15:53:52 +0100
Subject: Removed various default metrics tags

While it's useful to keep track of the different versions (Ruby, GitLab,
etc) doing so for every point wastes disk space and possibly also RAM
(which InfluxDB is all to eager to gobble up). If we want to see the
performance differences between different GitLab versions simply looking
at the performance since the last release date should suffice.
---
 lib/gitlab/metrics/metric.rb | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index 79241f56874..753008df99a 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -19,11 +19,8 @@ module Gitlab
         {
           series: @series,
           tags:   @tags.merge(
-            hostname:       Metrics.hostname,
-            ruby_engine:    RUBY_ENGINE,
-            ruby_version:   RUBY_VERSION,
-            gitlab_version: Gitlab::VERSION,
-            process_type:   Sidekiq.server? ? 'sidekiq' : 'rails'
+            hostname:     Metrics.hostname,
+            process_type: Sidekiq.server? ? 'sidekiq' : 'rails'
           ),
           values:    @values,
           timestamp: @created_at.to_i * 1_000_000_000
-- 
cgit v1.2.1


From 3077cb52d904154b98ee3e9aced5b3aadae86941 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 30 Dec 2015 17:16:02 +0100
Subject: Use XPath for searching link nodes

This is a tad faster than letting Nokogiri figure out whether it should
evaluate the query as CSS or XPath and then actually evaluating it.
---
 lib/banzai/filter/reference_filter.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 8ca05ace88c..7198a8b03e2 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -124,7 +124,7 @@ module Banzai
       def replace_link_nodes_with_text(pattern)
         return doc if project.nil?
 
-        doc.search('a').each do |node|
+        doc.xpath('descendant-or-self::a').each do |node|
           klass = node.attr('class')
           next if klass && klass.include?('gfm')
 
@@ -162,7 +162,7 @@ module Banzai
       def replace_link_nodes_with_href(pattern)
         return doc if project.nil?
 
-        doc.search('a').each do |node|
+        doc.xpath('descendant-or-self::a').each do |node|
           klass = node.attr('class')
           next if klass && klass.include?('gfm')
 
-- 
cgit v1.2.1


From d3951dfaa13b9aaf11695ef10fa63456ac75cc48 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 30 Dec 2015 17:16:59 +0100
Subject: Don't use delegate to delegate trivial methods

Around 300 ms (in total) would be spent in these delegated methods due
to the extra stuff ActiveSupport adds to the compiled methods. Because
these delegations are so simple we can just manually define the methods,
saving around 275 milliseconds.
---
 lib/banzai/filter/abstract_reference_filter.rb | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 63ad8910c0f..230387c8383 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -47,7 +47,17 @@ module Banzai
         { object_sym => LazyReference.new(object_class, node.attr(data_reference)) }
       end
 
-      delegate :object_class, :object_sym, :references_in, to: :class
+      def object_class
+        self.class.object_class
+      end
+
+      def object_sym
+        self.class.object_sym
+      end
+
+      def references_in(*args, &block)
+        self.class.references_in(*args, &block)
+      end
 
       def find_object(project, id)
         # Implement in child class
-- 
cgit v1.2.1


From 054df415f94abe1e517a729e53cdb325d592d31b Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Wed, 30 Dec 2015 18:16:53 +0100
Subject: Optimize CSS expressions produced by Nokogiri

Nokogiri produces inefficient XPath expressions when given CSS
expressions such as "a.gfm". Luckily these expressions can be optimized
quite easily while still achieving the same results.

In the two cases where this optimization is applied the run time has
been reduced from around 170 ms to around 15 ms.
---
 lib/banzai/filter/redactor_filter.rb           |  2 +-
 lib/banzai/filter/reference_gatherer_filter.rb |  2 +-
 lib/banzai/querying.rb                         | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)
 create mode 100644 lib/banzai/querying.rb

(limited to 'lib')

diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index f01a32b5ae5..66f77902319 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -10,7 +10,7 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        doc.css('a.gfm').each do |node|
+        Querying.css(doc, 'a.gfm').each do |node|
           unless user_can_see_reference?(node)
             # The reference should be replaced by the original text,
             # which is not always the same as the rendered text.
diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb
index 12412ff7ea9..bef04112919 100644
--- a/lib/banzai/filter/reference_gatherer_filter.rb
+++ b/lib/banzai/filter/reference_gatherer_filter.rb
@@ -16,7 +16,7 @@ module Banzai
       end
 
       def call
-        doc.css('a.gfm').each do |node|
+        Querying.css(doc, 'a.gfm').each do |node|
           gather_references(node)
         end
 
diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb
new file mode 100644
index 00000000000..1e1b51e683e
--- /dev/null
+++ b/lib/banzai/querying.rb
@@ -0,0 +1,18 @@
+module Banzai
+  module Querying
+    # Searches a Nokogiri document using a CSS query, optionally optimizing it
+    # whenever possible.
+    #
+    # document - A document/element to search.
+    # query    - The CSS query to use.
+    #
+    # Returns a Nokogiri::XML::NodeSet.
+    def self.css(document, query)
+      # When using "a.foo" Nokogiri compiles this to "//a[...]" but
+      # "descendant::a[...]" is quite a bit faster and achieves the same result.
+      xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::')
+
+      document.xpath(xpath)
+    end
+  end
+end
-- 
cgit v1.2.1


From a6c60127e3e2966b2f29fa6e6e79503b130c2b02 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 31 Dec 2015 17:14:02 +0100
Subject: Removed tracking of raw SQL queries

This particular setup had 3 problems:

1. Storing SQL queries as tags is very inefficient as InfluxDB ends up
   indexing every query (and they can get pretty large). Storing these
   as values instead means we can't always display the SQL as easily.
2. We already instrument ActiveRecord query methods, thus we already
   have timing information about database queries.
3. SQL obfuscation is difficult to get right and I'd rather not expose
   sensitive data by accident.
---
 lib/gitlab/metrics/obfuscated_sql.rb            | 47 ------------------------
 lib/gitlab/metrics/subscribers/active_record.rb | 48 -------------------------
 2 files changed, 95 deletions(-)
 delete mode 100644 lib/gitlab/metrics/obfuscated_sql.rb
 delete mode 100644 lib/gitlab/metrics/subscribers/active_record.rb

(limited to 'lib')

diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb
deleted file mode 100644
index fe97d7a0534..00000000000
--- a/lib/gitlab/metrics/obfuscated_sql.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-module Gitlab
-  module Metrics
-    # Class for producing SQL queries with sensitive data stripped out.
-    class ObfuscatedSQL
-      REPLACEMENT = /
-        \d+(\.\d+)?      # integers, floats
-        | '.+?'          # single quoted strings
-        | \/.+?(?<!\\)\/ # regexps (including escaped slashes)
-      /x
-
-      MYSQL_REPLACEMENTS = /
-        ".+?" # double quoted strings
-      /x
-
-      # Regex to replace consecutive placeholders with a single one indicating
-      # the length. This can be useful when a "IN" statement uses thousands of
-      # IDs (storing this would just be a waste of space).
-      CONSECUTIVE = /(\?(\s*,\s*)?){2,}/
-
-      # sql - The raw SQL query as a String.
-      def initialize(sql)
-        @sql = sql
-      end
-
-      # Returns a new, obfuscated SQL query.
-      def to_s
-        regex = REPLACEMENT
-
-        if Gitlab::Database.mysql?
-          regex = Regexp.union(regex, MYSQL_REPLACEMENTS)
-        end
-
-        sql = @sql.gsub(regex, '?').gsub(CONSECUTIVE) do |match|
-          "#{match.count(',') + 1} values"
-        end
-
-        # InfluxDB escapes double quotes upon output, so lets get rid of them
-        # whenever we can.
-        if Gitlab::Database.postgresql?
-          sql = sql.delete('"')
-        end
-
-        sql.tr("\n", ' ')
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
deleted file mode 100644
index d947c128ce2..00000000000
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-module Gitlab
-  module Metrics
-    module Subscribers
-      # Class for tracking raw SQL queries.
-      #
-      # Queries are obfuscated before being logged to ensure no private data is
-      # exposed via InfluxDB/Grafana.
-      class ActiveRecord < ActiveSupport::Subscriber
-        attach_to :active_record
-
-        SERIES = 'sql_queries'
-
-        def sql(event)
-          return unless current_transaction
-
-          values = values_for(event)
-          tags   = tags_for(event)
-
-          current_transaction.add_metric(SERIES, values, tags)
-        end
-
-        private
-
-        def values_for(event)
-          { duration: event.duration }
-        end
-
-        def tags_for(event)
-          sql  = ObfuscatedSQL.new(event.payload[:sql]).to_s
-          tags = { sql: sql }
-
-          file, line = Metrics.last_relative_application_frame
-
-          if file and line
-            tags[:file] = file
-            tags[:line] = line
-          end
-
-          tags
-        end
-
-        def current_transaction
-          Transaction.current
-        end
-      end
-    end
-  end
-end
-- 
cgit v1.2.1


From 55ed6e1c96e4072af81ed51c4377f2c015f237d5 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 31 Dec 2015 17:47:07 +0100
Subject: Cache InfluxDB settings after the first use

This ensures we don't need to load anything from either PostgreSQL or
the Rails cache whenever creating new InfluxDB connections.
---
 lib/gitlab/metrics.rb | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 2d266ccfe9e..c5b98a0115e 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -6,16 +6,21 @@ module Gitlab
     METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
     PATH_REGEX   = /^#{RAILS_ROOT}\/?/
 
-    def self.pool_size
-      current_application_settings[:metrics_pool_size] || 16
-    end
-
-    def self.timeout
-      current_application_settings[:metrics_timeout] || 10
+    def self.settings
+      @settings ||= {
+        enabled:               current_application_settings[:metrics_enabled],
+        pool_size:             current_application_settings[:metrics_pool_size],
+        timeout:               current_application_settings[:metrics_timeout],
+        method_call_threshold: current_application_settings[:metrics_method_call_threshold],
+        host:                  current_application_settings[:metrics_host],
+        username:              current_application_settings[:metrics_username],
+        password:              current_application_settings[:metrics_password],
+        port:                  current_application_settings[:metrics_port]
+      }
     end
 
     def self.enabled?
-      current_application_settings[:metrics_enabled] || false
+      settings[:enabled] || false
     end
 
     def self.mri?
@@ -26,8 +31,7 @@ module Gitlab
       # This is memoized since this method is called for every instrumented
       # method. Loading data from an external cache on every method call slows
       # things down too much.
-      @method_call_threshold ||=
-        (current_application_settings[:metrics_method_call_threshold] || 10)
+      @method_call_threshold ||= settings[:method_call_threshold]
     end
 
     def self.pool
@@ -90,11 +94,11 @@ module Gitlab
     # When enabled this should be set before being used as the usual pattern
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
-      @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do
-        host = current_application_settings[:metrics_host]
-        user = current_application_settings[:metrics_username]
-        pw   = current_application_settings[:metrics_password]
-        port = current_application_settings[:metrics_port]
+      @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
+        host = settings[:host]
+        user = settings[:username]
+        pw   = settings[:password]
+        port = settings[:port]
 
         InfluxDB::Client.
           new(udp: { host: host, port: port }, username: user, password: pw)
-- 
cgit v1.2.1


From bd9f86bb8abb4759a0c72f94fb0492b1ff8619b5 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 31 Dec 2015 17:51:12 +0100
Subject: Use separate series for Rails/Sidekiq transactions

This removes the need for tagging all metrics with a "process_type" tag.
---
 lib/gitlab/metrics/metric.rb             | 3 +--
 lib/gitlab/metrics/rack_middleware.rb    | 4 +++-
 lib/gitlab/metrics/sidekiq_middleware.rb | 4 +++-
 lib/gitlab/metrics/transaction.rb        | 7 +++----
 4 files changed, 10 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index 753008df99a..8319e628a40 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -19,8 +19,7 @@ module Gitlab
         {
           series: @series,
           tags:   @tags.merge(
-            hostname:     Metrics.hostname,
-            process_type: Sidekiq.server? ? 'sidekiq' : 'rails'
+            hostname: Metrics.hostname
           ),
           values:    @values,
           timestamp: @created_at.to_i * 1_000_000_000
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 5c0587c4c51..bb9e4fcb918 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -4,6 +4,8 @@ module Gitlab
     class RackMiddleware
       CONTROLLER_KEY = 'action_controller.instance'
 
+      SERIES = 'rails_transactions'
+
       def initialize(app)
         @app = app
       end
@@ -30,7 +32,7 @@ module Gitlab
       end
 
       def transaction_from_env(env)
-        trans = Transaction.new
+        trans = Transaction.new(SERIES)
 
         trans.add_tag(:request_method, env['REQUEST_METHOD'])
         trans.add_tag(:request_uri, env['REQUEST_URI'])
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index ad441decfa2..6e804dd2562 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -4,8 +4,10 @@ module Gitlab
     #
     # This middleware is intended to be used as a server-side middleware.
     class SidekiqMiddleware
+      SERIES = 'sidekiq_transactions'
+
       def call(worker, message, queue)
-        trans = Transaction.new
+        trans = Transaction.new(SERIES)
 
         begin
           trans.run { yield }
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index a61dbd989e7..43a7dab5323 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,8 +4,6 @@ module Gitlab
     class Transaction
       THREAD_KEY = :_gitlab_metrics_transaction
 
-      SERIES = 'transactions'
-
       attr_reader :uuid, :tags
 
       def self.current
@@ -13,7 +11,8 @@ module Gitlab
       end
 
       # name - The name of this transaction as a String.
-      def initialize
+      def initialize(series)
+        @series  = series
         @metrics = []
         @uuid    = SecureRandom.uuid
 
@@ -55,7 +54,7 @@ module Gitlab
       end
 
       def track_self
-        add_metric(SERIES, { duration: duration }, @tags)
+        add_metric(@series, { duration: duration }, @tags)
       end
 
       def submit
-- 
cgit v1.2.1


From cafc784ee1d5d0a0279077272af8ee435bb110e4 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Thu, 31 Dec 2015 17:55:10 +0100
Subject: Removed tracking of hostnames for metrics

This isn't hugely useful and mostly wastes InfluxDB space. We can re-add
this whenever needed (but only once we really need it).
---
 lib/gitlab/metrics.rb        | 6 ------
 lib/gitlab/metrics/metric.rb | 6 ++----
 2 files changed, 2 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index c5b98a0115e..ee88ab34d6c 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -38,10 +38,6 @@ module Gitlab
       @pool
     end
 
-    def self.hostname
-      @hostname
-    end
-
     # Returns a relative path and line number based on the last application call
     # frame.
     def self.last_relative_application_frame
@@ -89,8 +85,6 @@ module Gitlab
       value.to_s.gsub('=', '\\=')
     end
 
-    @hostname = Socket.gethostname
-
     # When enabled this should be set before being used as the usual pattern
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index 8319e628a40..7ea9555cc8c 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -17,10 +17,8 @@ module Gitlab
       # Returns a Hash in a format that can be directly written to InfluxDB.
       def to_hash
         {
-          series: @series,
-          tags:   @tags.merge(
-            hostname: Metrics.hostname
-          ),
+          series:    @series,
+          tags:      @tags,
           values:    @values,
           timestamp: @created_at.to_i * 1_000_000_000
         }
-- 
cgit v1.2.1


From 1a1113f7c4ad6e5bceb898593cba6caecbdf0097 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Fri, 1 Jan 2016 22:11:34 -0500
Subject: Simplify `ContributionsCalendar#starting_year` and `#starting_month`

---
 lib/gitlab/contributions_calendar.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index 8a7f8dc5003..85583dce9ee 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -45,11 +45,11 @@ module Gitlab
     end
 
     def starting_year
-      (Time.now - 1.year).strftime("%Y")
+      1.year.ago.year
     end
 
     def starting_month
-      Date.today.strftime("%m").to_i
+      Date.today.month
     end
   end
 end
-- 
cgit v1.2.1


From 086cfc8685a6489ca032899307c77f828f515fbb Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Wed, 26 Aug 2015 23:36:17 -0700
Subject: Fix API project lookups when querying with a namespace with dots

Attempting to use the /projects/:id API by specifying :id in
"namespace/project" format would always result in a 404 if the namespace
contained a dot.

The reason? From http://guides.rubyonrails.org/routing.html#specifying-constraints:

"By default the :id parameter doesn't accept dots - this is because the dot is
used as a separator for formatted routes. If you need to use a dot within an
:id add a constraint which overrides this - for example id: /[^\/]+/ allows
anything except a slash."

Closes https://github.com/gitlabhq/gitlabhq/issues/9573
---
 lib/api/projects.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index a9e0960872a..0781236cf6d 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -3,7 +3,7 @@ module API
   class Projects < Grape::API
     before { authenticate! }
 
-    resource :projects do
+    resource :projects, requirements: { id: /[^\/]+/ } do
       helpers do
         def map_public_to_visibility_level(attrs)
           publik = attrs.delete(:public)
-- 
cgit v1.2.1


From 96075be6f4688a59335130dc796132ad4f232442 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 4 Jan 2016 11:37:46 +0100
Subject: Ability to increment custom transaction values

This will be used to store/increment the total query/view rendering
timings on a per transaction basis. This in turn can greatly reduce the
amount of metrics stored.
---
 lib/gitlab/metrics/transaction.rb | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 43a7dab5323..0aaebf262d4 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -19,7 +19,8 @@ module Gitlab
         @started_at  = nil
         @finished_at = nil
 
-        @tags = {}
+        @values = Hash.new(0)
+        @tags   = {}
       end
 
       def duration
@@ -44,6 +45,10 @@ module Gitlab
         @metrics << Metric.new(series, values, tags)
       end
 
+      def increment(name, value)
+        @values[name] += value
+      end
+
       def add_tag(key, value)
         @tags[key] = value
       end
@@ -54,7 +59,13 @@ module Gitlab
       end
 
       def track_self
-        add_metric(@series, { duration: duration }, @tags)
+        values = { duration: duration }
+
+        @values.each do |name, value|
+          values[name] = value
+        end
+
+        add_metric(@series, values, @tags)
       end
 
       def submit
-- 
cgit v1.2.1


From 66a997a91403eef62ffd9fb789e899619d021a26 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 4 Jan 2016 12:14:36 +0100
Subject: Track total query/view timings in transactions

---
 lib/gitlab/metrics/subscribers/action_view.rb   |  1 +
 lib/gitlab/metrics/subscribers/active_record.rb | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 lib/gitlab/metrics/subscribers/active_record.rb

(limited to 'lib')

diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 7e0dcf99d92..7c0105d543a 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -19,6 +19,7 @@ module Gitlab
           values = values_for(event)
           tags   = tags_for(event)
 
+          current_transaction.increment(:view_duration, event.duration)
           current_transaction.add_metric(SERIES, values, tags)
         end
 
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
new file mode 100644
index 00000000000..8008b3bc895
--- /dev/null
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Metrics
+    module Subscribers
+      # Class for tracking the total query duration of a transaction.
+      class ActiveRecord < ActiveSupport::Subscriber
+        attach_to :active_record
+
+        def sql(event)
+          return unless current_transaction
+
+          current_transaction.increment(:sql_duration, event.duration)
+        end
+
+        private
+
+        def current_transaction
+          Transaction.current
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 825b46f8a3eb620f99192217d414b72dffe597d7 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 4 Jan 2016 12:19:45 +0100
Subject: Track total method call times per transaction

This makes it easier to see where time is spent without having to
aggregate all the individual points in the method_calls series.
---
 lib/gitlab/metrics/instrumentation.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb
index 06fc2f25948..d9fce2e6758 100644
--- a/lib/gitlab/metrics/instrumentation.rb
+++ b/lib/gitlab/metrics/instrumentation.rb
@@ -123,6 +123,8 @@ module Gitlab
               duration = (Time.now - start) * 1000.0
 
               if duration >= Gitlab::Metrics.method_call_threshold
+                trans.increment(:method_duration, duration)
+
                 trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES,
                                  { duration: duration },
                                  method: #{label.inspect})
-- 
cgit v1.2.1


From 2ea464bb272fa52ff34a188a921f0bc90811ca45 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 4 Jan 2016 12:45:31 +0100
Subject: Use separate series for Rails/Sidekiq sample stats

This removes the need for any tags to differentiate between Sidekiq and
Rails statistics while still being able to separate the two.
---
 lib/gitlab/metrics/sampler.rb | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 998578e1c0a..1ea425bc904 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -50,12 +50,11 @@ module Gitlab
       end
 
       def sample_memory_usage
-        @metrics << Metric.new('memory_usage', value: System.memory_usage)
+        add_metric('memory_usage', value: System.memory_usage)
       end
 
       def sample_file_descriptors
-        @metrics << Metric.
-          new('file_descriptors', value: System.file_descriptor_count)
+        add_metric('file_descriptors', value: System.file_descriptor_count)
       end
 
       if Metrics.mri?
@@ -69,7 +68,7 @@ module Gitlab
           counts['Symbol'] = Symbol.all_symbols.length
 
           counts.each do |name, count|
-            @metrics << Metric.new('object_counts', { count: count }, type: name)
+            add_metric('object_counts', { count: count }, type: name)
           end
         end
       else
@@ -91,7 +90,17 @@ module Gitlab
 
         stats[:count] = stats[:minor_gc_count] + stats[:major_gc_count]
 
-        @metrics << Metric.new('gc_statistics', stats)
+        add_metric('gc_statistics', stats)
+      end
+
+      def add_metric(series, values, tags = {})
+        prefix = sidekiq? ? 'sidekiq_' : 'rails_'
+
+        @metrics << Metric.new("#{prefix}#{series}", values, tags)
+      end
+
+      def sidekiq?
+        Sidekiq.server?
       end
     end
   end
-- 
cgit v1.2.1


From 2ee8f555996baca6b470d223ffad65419b730398 Mon Sep 17 00:00:00 2001
From: Yorick Peterse <yorickpeterse@gmail.com>
Date: Mon, 4 Jan 2016 13:17:02 +0100
Subject: Automatically prefix transaction series names

This ensures Rails and Sidekiq transactions are split into the series
"rails_transactions" and "sidekiq_transactions" respectively.
---
 lib/gitlab/metrics/rack_middleware.rb    |  4 +---
 lib/gitlab/metrics/sidekiq_middleware.rb |  4 +---
 lib/gitlab/metrics/transaction.rb        | 15 +++++++++------
 3 files changed, 11 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index bb9e4fcb918..5c0587c4c51 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -4,8 +4,6 @@ module Gitlab
     class RackMiddleware
       CONTROLLER_KEY = 'action_controller.instance'
 
-      SERIES = 'rails_transactions'
-
       def initialize(app)
         @app = app
       end
@@ -32,7 +30,7 @@ module Gitlab
       end
 
       def transaction_from_env(env)
-        trans = Transaction.new(SERIES)
+        trans = Transaction.new
 
         trans.add_tag(:request_method, env['REQUEST_METHOD'])
         trans.add_tag(:request_uri, env['REQUEST_URI'])
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index 6e804dd2562..ad441decfa2 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -4,10 +4,8 @@ module Gitlab
     #
     # This middleware is intended to be used as a server-side middleware.
     class SidekiqMiddleware
-      SERIES = 'sidekiq_transactions'
-
       def call(worker, message, queue)
-        trans = Transaction.new(SERIES)
+        trans = Transaction.new
 
         begin
           trans.run { yield }
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 0aaebf262d4..68b86de0655 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -10,9 +10,7 @@ module Gitlab
         Thread.current[THREAD_KEY]
       end
 
-      # name - The name of this transaction as a String.
-      def initialize(series)
-        @series  = series
+      def initialize
         @metrics = []
         @uuid    = SecureRandom.uuid
 
@@ -40,9 +38,10 @@ module Gitlab
       end
 
       def add_metric(series, values, tags = {})
-        tags = tags.merge(transaction_id: @uuid)
+        tags   = tags.merge(transaction_id: @uuid)
+        prefix = sidekiq? ? 'sidekiq_' : 'rails_'
 
-        @metrics << Metric.new(series, values, tags)
+        @metrics << Metric.new("#{prefix}#{series}", values, tags)
       end
 
       def increment(name, value)
@@ -65,12 +64,16 @@ module Gitlab
           values[name] = value
         end
 
-        add_metric(@series, values, @tags)
+        add_metric('transactions', values, @tags)
       end
 
       def submit
         Metrics.submit_metrics(@metrics.map(&:to_hash))
       end
+
+      def sidekiq?
+        Sidekiq.server?
+      end
     end
   end
 end
-- 
cgit v1.2.1


From 4b027bc93a7875c3937f6b90ac1049b4a4d72da5 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Mon, 4 Jan 2016 14:29:06 +0100
Subject: Add DEBUG_BANZAI_CACHE env var to debug Banzai cache issue.

---
 lib/banzai/renderer.rb | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 115ae914524..910e1c6994e 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -1,7 +1,5 @@
 module Banzai
   module Renderer
-    CACHE_ENABLED = false
-
     # Convert a Markdown String into an HTML-safe String of HTML
     #
     # Note that while the returned HTML will have been sanitized of dangerous
@@ -20,13 +18,22 @@ module Banzai
       cache_key = context.delete(:cache_key)
       cache_key = full_cache_key(cache_key, context[:pipeline])
 
-      if cache_key && CACHE_ENABLED
-        Rails.cache.fetch(cache_key) do
-          cacheless_render(text, context)
+      cacheless = cacheless_render(text, context)
+
+      if cache_key && ENV["DEBUG_BANZAI_CACHE"]
+        cached = Rails.cache.fetch(cache_key) { cacheless }
+
+        if cached != cacheless
+          Rails.logger.warn "Banzai cache mismatch"
+          Rails.logger.warn "Text: #{text.inspect}"
+          Rails.logger.warn "Context: #{context.inspect}"
+          Rails.logger.warn "Cache key: #{cache_key.inspect}"
+          Rails.logger.warn "Cacheless: #{cacheless.inspect}"
+          Rails.logger.warn "With cache: #{cached.inspect}"
         end
-      else
-        cacheless_render(text, context)
       end
+
+      cacheless
     end
 
     def self.render_result(text, context = {})
-- 
cgit v1.2.1