Merge branch 'configure-randomize-metrics-sample-interval' into 'master'

See merge request !2406

2 files changed, 28 insertions, 5 deletions

diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index cdf7c168ff2..88a265c6af2 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -13,7 +13,8 @@ module Gitlab
         timeout:               current_application_settings[:metrics_timeout],
         method_call_threshold: current_application_settings[:metrics_method_call_threshold],
         host:                  current_application_settings[:metrics_host],
-        port:                  current_application_settings[:metrics_port]
+        port:                  current_application_settings[:metrics_port],
+        sample_interval:       current_application_settings[:metrics_sample_interval] || 15
       }
     end
 
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb
index 1ea425bc904..fc709222a9b 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/sampler.rb
@@ -7,9 +7,14 @@ module Gitlab
     # statistics, etc.
     class Sampler
       # interval - The sampling interval in seconds.
-      def initialize(interval = 15)
-        @interval = interval
-        @metrics  = []
+      def initialize(interval = Metrics.settings[:sample_interval])
+        interval_half = interval.to_f / 2
+
+        @interval       = interval
+        @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+        @last_step      = nil
+
+        @metrics = []
 
         @last_minor_gc = Delta.new(GC.stat[:minor_gc_count])
         @last_major_gc = Delta.new(GC.stat[:major_gc_count])
@@ -26,7 +31,7 @@ module Gitlab
           Thread.current.abort_on_exception = true
 
           loop do
-            sleep(@interval)
+            sleep(sleep_interval)
 
             sample
           end
@@ -102,6 +107,23 @@ module Gitlab
       def sidekiq?
         Sidekiq.server?
       end
+
+      # Returns the sleep interval with a random adjustment.
+      #
+      # The random adjustment is put in place to ensure we:
+      #
+      # 1. Don't generate samples at the exact same interval every time (thus
+      #    potentially missing anything that happens in between samples).
+      # 2. Don't sample data at the same interval two times in a row.
+      def sleep_interval
+        while step = @interval_steps.sample
+          if step != @last_step
+            @last_step = step
+
+            return @interval + @last_step
+          end
+        end
+      end
     end
   end
 end