From 552d38861ad77bf6a64b1e61a91e36fcd26d057c Mon Sep 17 00:00:00 2001 From: Peter Leitzen Date: Thu, 4 Apr 2019 15:38:37 +0000 Subject: Automatically set Prometheus step interval By computing the step interval passed to the query_range Prometheus API call we improve the performance on the Prometheus server and GitLab by reducing the amount of data points sent back and prevent Prometheus from sending errors when requesting longer intervals. --- ...-automatically-set-prometheus-step-interval.yml | 5 ++++ lib/gitlab/prometheus_client.rb | 32 ++++++++++++++++++---- spec/lib/gitlab/prometheus_client_spec.rb | 28 +++++++++++++++++++ spec/support/helpers/prometheus_helpers.rb | 10 +++++-- 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml diff --git a/changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml b/changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml new file mode 100644 index 00000000000..2c6edf45ae2 --- /dev/null +++ b/changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml @@ -0,0 +1,5 @@ +--- +title: Automatically set Prometheus step interval +merge_request: 26441 +author: +type: changed diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 45828c77a33..b4de7cd2bce 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -6,6 +6,14 @@ module Gitlab Error = Class.new(StandardError) QueryError = Class.new(Gitlab::PrometheusClient::Error) + # Target number of data points for `query_range`. + # Please don't exceed the limit of 11000 data points + # See https://github.com/prometheus/prometheus/blob/91306bdf24f5395e2601773316945a478b4b263d/web/api/v1/api.go#L347 + QUERY_RANGE_DATA_POINTS = 600 + + # Minimal value of the `step` parameter for `query_range` in seconds. + QUERY_RANGE_MIN_STEP = 60 + attr_reader :rest_client, :headers def initialize(rest_client) @@ -23,12 +31,18 @@ module Gitlab end def query_range(query, start: 8.hours.ago, stop: Time.now) + start = start.to_f + stop = stop.to_f + step = self.class.compute_step(start, stop) + get_result('matrix') do - json_api_get('query_range', - query: query, - start: start.to_f, - end: stop.to_f, - step: 1.minute.to_i) + json_api_get( + 'query_range', + query: query, + start: start, + end: stop, + step: step + ) end end @@ -40,6 +54,14 @@ module Gitlab json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f) end + def self.compute_step(start, stop) + diff = stop - start + + step = (diff / QUERY_RANGE_DATA_POINTS).ceil + + [QUERY_RANGE_MIN_STEP, step].max + end + private def json_api_get(type, args = {}) diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb index 4c3b8deefb9..2517ee71f24 100644 --- a/spec/lib/gitlab/prometheus_client_spec.rb +++ b/spec/lib/gitlab/prometheus_client_spec.rb @@ -230,4 +230,32 @@ describe Gitlab::PrometheusClient do let(:execute_query) { subject.query_range(prometheus_query) } end end + + describe '.compute_step' do + using RSpec::Parameterized::TableSyntax + + let(:now) { Time.now.utc } + + subject { described_class.compute_step(start, stop) } + + where(:time_interval_in_seconds, :step) do + 0 | 60 + 10.hours | 60 + 10.hours + 1 | 61 + # frontend options + 30.minutes | 60 + 3.hours | 60 + 8.hours | 60 + 1.day | 144 + 3.days | 432 + 1.week | 1008 + end + + with_them do + let(:start) { now - time_interval_in_seconds } + let(:stop) { now } + + it { is_expected.to eq(step) } + end + end end diff --git a/spec/support/helpers/prometheus_helpers.rb b/spec/support/helpers/prometheus_helpers.rb index ce1f9fce10d..08d1d7a6059 100644 --- a/spec/support/helpers/prometheus_helpers.rb +++ b/spec/support/helpers/prometheus_helpers.rb @@ -25,12 +25,16 @@ module PrometheusHelpers "https://prometheus.example.com/api/v1/query?#{query}" end - def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now.to_f) + def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now, step: nil) + start = start.to_f + stop = stop.to_f + step ||= Gitlab::PrometheusClient.compute_step(start, stop) + query = { query: prometheus_query, - start: start.to_f, + start: start, end: stop, - step: 1.minute.to_i + step: step }.to_query "https://prometheus.example.com/api/v1/query_range?#{query}" -- cgit v1.2.1