Merge branch 'metrics-tuning' into 'master'

Tuning of metrics data to store

This removes data we don't really need, as well as making sure we don't overload any cache stores or databases.

See merge request !2265

8 files changed, 74 insertions, 136 deletions

diff --git a/spec/lib/gitlab/metrics/instrumentation_spec.rb b/spec/lib/gitlab/metrics/instrumentation_spec.rb
index a7eab9d11cc..2a37cd40dde 100644
--- a/spec/lib/gitlab/metrics/instrumentation_spec.rb
+++ b/spec/lib/gitlab/metrics/instrumentation_spec.rb
@@ -48,6 +48,9 @@ describe Gitlab::Metrics::Instrumentation do
         allow(described_class).to receive(:transaction).
           and_return(transaction)
 
+        expect(transaction).to receive(:increment).
+          with(:method_duration, a_kind_of(Numeric))
+
         expect(transaction).to receive(:add_metric).
           with(described_class::SERIES, an_instance_of(Hash),
                method: 'Dummy.foo')
@@ -102,6 +105,9 @@ describe Gitlab::Metrics::Instrumentation do
         allow(described_class).to receive(:transaction).
           and_return(transaction)
 
+        expect(transaction).to receive(:increment).
+          with(:method_duration, a_kind_of(Numeric))
+
         expect(transaction).to receive(:add_metric).
           with(described_class::SERIES, an_instance_of(Hash),
                method: 'Dummy#bar')
diff --git a/spec/lib/gitlab/metrics/metric_spec.rb b/spec/lib/gitlab/metrics/metric_spec.rb
index aa76315c79c..f718d536130 100644
--- a/spec/lib/gitlab/metrics/metric_spec.rb
+++ b/spec/lib/gitlab/metrics/metric_spec.rb
@@ -37,9 +37,6 @@ describe Gitlab::Metrics::Metric do
 
       it 'includes the tags' do
         expect(hash[:tags]).to be_an_instance_of(Hash)
-
-        expect(hash[:tags][:hostname]).to be_an_instance_of(String)
-        expect(hash[:tags][:process_type]).to be_an_instance_of(String)
       end
 
       it 'includes the values' do
diff --git a/spec/lib/gitlab/metrics/obfuscated_sql_spec.rb b/spec/lib/gitlab/metrics/obfuscated_sql_spec.rb
deleted file mode 100644
index 2b681c9fe34..00000000000
--- a/spec/lib/gitlab/metrics/obfuscated_sql_spec.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::Metrics::ObfuscatedSQL do
-  describe '#to_s' do
-    it 'replaces newlines with a space' do
-      sql = described_class.new("SELECT x\nFROM y")
-
-      expect(sql.to_s).to eq('SELECT x FROM y')
-    end
-
-    describe 'using single values' do
-      it 'replaces a single integer' do
-        sql = described_class.new('SELECT x FROM y WHERE a = 10')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE a = ?')
-      end
-
-      it 'replaces a single float' do
-        sql = described_class.new('SELECT x FROM y WHERE a = 10.5')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE a = ?')
-      end
-
-      it 'replaces a single quoted string' do
-        sql = described_class.new("SELECT x FROM y WHERE a = 'foo'")
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE a = ?')
-      end
-
-      if Gitlab::Database.mysql?
-        it 'replaces a double quoted string' do
-          sql = described_class.new('SELECT x FROM y WHERE a = "foo"')
-
-          expect(sql.to_s).to eq('SELECT x FROM y WHERE a = ?')
-        end
-      end
-
-      it 'replaces a single regular expression' do
-        sql = described_class.new('SELECT x FROM y WHERE a = /foo/')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE a = ?')
-      end
-
-      it 'replaces regular expressions using escaped slashes' do
-        sql = described_class.new('SELECT x FROM y WHERE a = /foo\/bar/')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE a = ?')
-      end
-    end
-
-    describe 'using consecutive values' do
-      it 'replaces multiple integers' do
-        sql = described_class.new('SELECT x FROM y WHERE z IN (10, 20, 30)')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE z IN (3 values)')
-      end
-
-      it 'replaces multiple floats' do
-        sql = described_class.new('SELECT x FROM y WHERE z IN (1.5, 2.5, 3.5)')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE z IN (3 values)')
-      end
-
-      it 'replaces multiple single quoted strings' do
-        sql = described_class.new("SELECT x FROM y WHERE z IN ('foo', 'bar')")
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE z IN (2 values)')
-      end
-
-      if Gitlab::Database.mysql?
-        it 'replaces multiple double quoted strings' do
-          sql = described_class.new('SELECT x FROM y WHERE z IN ("foo", "bar")')
-
-          expect(sql.to_s).to eq('SELECT x FROM y WHERE z IN (2 values)')
-        end
-      end
-
-      it 'replaces multiple regular expressions' do
-        sql = described_class.new('SELECT x FROM y WHERE z IN (/foo/, /bar/)')
-
-        expect(sql.to_s).to eq('SELECT x FROM y WHERE z IN (2 values)')
-      end
-    end
-
-    if Gitlab::Database.postgresql?
-      it 'replaces double quotes' do
-        sql = described_class.new('SELECT "x" FROM "y"')
-
-        expect(sql.to_s).to eq('SELECT x FROM y')
-      end
-    end
-  end
-end
diff --git a/spec/lib/gitlab/metrics/sampler_spec.rb b/spec/lib/gitlab/metrics/sampler_spec.rb
index 51a941c48cd..27211350fbe 100644
--- a/spec/lib/gitlab/metrics/sampler_spec.rb
+++ b/spec/lib/gitlab/metrics/sampler_spec.rb
@@ -51,8 +51,8 @@ describe Gitlab::Metrics::Sampler do
       expect(Gitlab::Metrics::System).to receive(:memory_usage).
         and_return(9000)
 
-      expect(Gitlab::Metrics::Metric).to receive(:new).
-        with('memory_usage', value: 9000).
+      expect(sampler).to receive(:add_metric).
+        with(/memory_usage/, value: 9000).
         and_call_original
 
       sampler.sample_memory_usage
@@ -64,8 +64,8 @@ describe Gitlab::Metrics::Sampler do
       expect(Gitlab::Metrics::System).to receive(:file_descriptor_count).
         and_return(4)
 
-      expect(Gitlab::Metrics::Metric).to receive(:new).
-        with('file_descriptors', value: 4).
+      expect(sampler).to receive(:add_metric).
+        with(/file_descriptors/, value: 4).
         and_call_original
 
       sampler.sample_file_descriptors
@@ -74,8 +74,8 @@ describe Gitlab::Metrics::Sampler do
 
   describe '#sample_objects' do
     it 'adds a metric containing the amount of allocated objects' do
-      expect(Gitlab::Metrics::Metric).to receive(:new).
-        with('object_counts', an_instance_of(Hash), an_instance_of(Hash)).
+      expect(sampler).to receive(:add_metric).
+        with(/object_counts/, an_instance_of(Hash), an_instance_of(Hash)).
         at_least(:once).
         and_call_original
 
@@ -87,11 +87,33 @@ describe Gitlab::Metrics::Sampler do
     it 'adds a metric containing garbage collection statistics' do
       expect(GC::Profiler).to receive(:total_time).and_return(0.24)
 
-      expect(Gitlab::Metrics::Metric).to receive(:new).
-        with('gc_statistics', an_instance_of(Hash)).
+      expect(sampler).to receive(:add_metric).
+        with(/gc_statistics/, an_instance_of(Hash)).
         and_call_original
 
       sampler.sample_gc
     end
   end
+
+  describe '#add_metric' do
+    it 'prefixes the series name for a Rails process' do
+      expect(sampler).to receive(:sidekiq?).and_return(false)
+
+      expect(Gitlab::Metrics::Metric).to receive(:new).
+        with('rails_cats', { value: 10 }, {}).
+        and_call_original
+
+      sampler.add_metric('cats', value: 10)
+    end
+
+    it 'prefixes the series name for a Sidekiq process' do
+      expect(sampler).to receive(:sidekiq?).and_return(true)
+
+      expect(Gitlab::Metrics::Metric).to receive(:new).
+        with('sidekiq_cats', { value: 10 }, {}).
+        and_call_original
+
+      sampler.add_metric('cats', value: 10)
+    end
+  end
 end
diff --git a/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb b/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb
index c6cd584663f..05e4fbbeb51 100644
--- a/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb
+++ b/spec/lib/gitlab/metrics/subscribers/action_view_spec.rb
@@ -28,6 +28,9 @@ describe Gitlab::Metrics::Subscribers::ActionView do
         line: 4
       }
 
+      expect(transaction).to receive(:increment).
+        with(:view_duration, 2.1)
+
       expect(transaction).to receive(:add_metric).
         with(described_class::SERIES, values, tags)
 
diff --git a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb
index 05b6cc14716..7bc070a4d09 100644
--- a/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb
+++ b/spec/lib/gitlab/metrics/subscribers/active_record_spec.rb
@@ -2,31 +2,34 @@ require 'spec_helper'
 
 describe Gitlab::Metrics::Subscribers::ActiveRecord do
   let(:transaction) { Gitlab::Metrics::Transaction.new }
-
-  let(:subscriber) { described_class.new }
+  let(:subscriber)  { described_class.new }
 
   let(:event) do
     double(:event, duration: 0.2,
                    payload:  { sql: 'SELECT * FROM users WHERE id = 10' })
   end
 
-  before do
-    allow(subscriber).to receive(:current_transaction).and_return(transaction)
+  describe '#sql' do
+    describe 'without a current transaction' do
+      it 'simply returns' do
+        expect_any_instance_of(Gitlab::Metrics::Transaction).
+          to_not receive(:increment)
 
-    allow(Gitlab::Metrics).to receive(:last_relative_application_frame).
-      and_return(['app/models/foo.rb', 4])
-  end
+        subscriber.sql(event)
+      end
+    end
 
-  describe '#sql' do
-    it 'tracks the execution of a SQL query' do
-      sql    = 'SELECT * FROM users WHERE id = ?'
-      values = { duration: 0.2 }
-      tags   = { sql: sql, file: 'app/models/foo.rb', line: 4 }
+    describe 'with a current transaction' do
+      it 'increments the :sql_duration value' do
+        expect(subscriber).to receive(:current_transaction).
+          at_least(:once).
+          and_return(transaction)
 
-      expect(transaction).to receive(:add_metric).
-        with(described_class::SERIES, values, tags)
+        expect(transaction).to receive(:increment).
+          with(:sql_duration, 0.2)
 
-      subscriber.sql(event)
+        subscriber.sql(event)
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/metrics/transaction_spec.rb b/spec/lib/gitlab/metrics/transaction_spec.rb
index 6862fc9e2d1..b9b94947afa 100644
--- a/spec/lib/gitlab/metrics/transaction_spec.rb
+++ b/spec/lib/gitlab/metrics/transaction_spec.rb
@@ -32,12 +32,24 @@ describe Gitlab::Metrics::Transaction do
   describe '#add_metric' do
     it 'adds a metric tagged with the transaction UUID' do
       expect(Gitlab::Metrics::Metric).to receive(:new).
-        with('foo', { number: 10 }, { transaction_id: transaction.uuid })
+        with('rails_foo', { number: 10 }, { transaction_id: transaction.uuid })
 
       transaction.add_metric('foo', number: 10)
     end
   end
 
+  describe '#increment' do
+    it 'increments a counter' do
+      transaction.increment(:time, 1)
+      transaction.increment(:time, 2)
+
+      expect(transaction).to receive(:add_metric).
+        with('transactions', { duration: 0.0, time: 3 }, {})
+
+      transaction.track_self
+    end
+  end
+
   describe '#add_tag' do
     it 'adds a tag' do
       transaction.add_tag(:foo, 'bar')
@@ -58,7 +70,7 @@ describe Gitlab::Metrics::Transaction do
   describe '#track_self' do
     it 'adds a metric for the transaction itself' do
       expect(transaction).to receive(:add_metric).
-        with(described_class::SERIES, { duration: transaction.duration }, {})
+        with('transactions', { duration: transaction.duration }, {})
 
       transaction.track_self
     end
diff --git a/spec/lib/gitlab/metrics_spec.rb b/spec/lib/gitlab/metrics_spec.rb
index 6c0682cac4d..c2782f95c8e 100644
--- a/spec/lib/gitlab/metrics_spec.rb
+++ b/spec/lib/gitlab/metrics_spec.rb
@@ -1,15 +1,9 @@
 require 'spec_helper'
 
 describe Gitlab::Metrics do
-  describe '.pool_size' do
-    it 'returns a Fixnum' do
-      expect(described_class.pool_size).to be_an_instance_of(Fixnum)
-    end
-  end
-
-  describe '.timeout' do
-    it 'returns a Fixnum' do
-      expect(described_class.timeout).to be_an_instance_of(Fixnum)
+  describe '.settings' do
+    it 'returns a Hash' do
+      expect(described_class.settings).to be_an_instance_of(Hash)
     end
   end
 
@@ -19,12 +13,6 @@ describe Gitlab::Metrics do
     end
   end
 
-  describe '.hostname' do
-    it 'returns a String containing the hostname' do
-      expect(described_class.hostname).to eq(Socket.gethostname)
-    end
-  end
-
   describe '.last_relative_application_frame' do
     it 'returns an Array containing a file path and line number' do
       file, line = described_class.last_relative_application_frame