Merge branch '28717-additional-metrics-review-branch' into 'master'

Support additional prometheus metrics - review branch

Closes #28717

See merge request !11712

12 files changed, 319 insertions, 35 deletions

diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb
new file mode 100644
index 00000000000..cb95daf2260
--- /dev/null
+++ b/lib/gitlab/prometheus/additional_metrics_parser.rb
@@ -0,0 +1,34 @@
+module Gitlab
+  module Prometheus
+    module AdditionalMetricsParser
+      extend self
+
+      def load_groups_from_yaml
+        additional_metrics_raw.map(&method(:group_from_entry))
+      end
+
+      private
+
+      def validate!(obj)
+        raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid?
+      end
+
+      def group_from_entry(entry)
+        entry[:name] = entry.delete(:group)
+        entry[:metrics]&.map! do |entry|
+          Metric.new(entry).tap(&method(:validate!))
+        end
+
+        MetricGroup.new(entry).tap(&method(:validate!))
+      end
+
+      def additional_metrics_raw
+        load_yaml_file&.map(&:deep_symbolize_keys).freeze
+      end
+
+      def load_yaml_file
+        @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml'))
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb
new file mode 100644
index 00000000000..f54b2c6aaff
--- /dev/null
+++ b/lib/gitlab/prometheus/metric.rb
@@ -0,0 +1,16 @@
+module Gitlab
+  module Prometheus
+    class Metric
+      include ActiveModel::Model
+
+      attr_accessor :title, :required_metrics, :weight, :y_label, :queries
+
+      validates :title, :required_metrics, :weight, :y_label, :queries, presence: true
+
+      def initialize(params = {})
+        super(params)
+        @y_label ||= 'Values'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
new file mode 100644
index 00000000000..729fef34b35
--- /dev/null
+++ b/lib/gitlab/prometheus/metric_group.rb
@@ -0,0 +1,14 @@
+module Gitlab
+  module Prometheus
+    class MetricGroup
+      include ActiveModel::Model
+
+      attr_accessor :name, :priority, :metrics
+      validates :name, :priority, :metrics, presence: true
+
+      def self.all
+        AdditionalMetricsParser.load_groups_from_yaml
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb
new file mode 100644
index 00000000000..49cc0e16080
--- /dev/null
+++ b/lib/gitlab/prometheus/parsing_error.rb
@@ -0,0 +1,5 @@
+module Gitlab
+  module Prometheus
+    ParsingError = Class.new(StandardError)
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
new file mode 100644
index 00000000000..67c69d9ccf3
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class AdditionalMetricsDeploymentQuery < BaseQuery
+        include QueryAdditionalMetrics
+
+        def query(deployment_id)
+          Deployment.find_by(id: deployment_id).try do |deployment|
+            query_context = {
+              environment_slug: deployment.environment.slug,
+              environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"},
+              timeframe_start: (deployment.created_at - 30.minutes).to_f,
+              timeframe_end: (deployment.created_at + 30.minutes).to_f
+            }
+
+            query_metrics(query_context)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
new file mode 100644
index 00000000000..b5a679ddd79
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class AdditionalMetricsEnvironmentQuery < BaseQuery
+        include QueryAdditionalMetrics
+
+        def query(environment_id)
+          Environment.find_by(id: environment_id).try do |environment|
+            query_context = {
+              environment_slug: environment.slug,
+              environment_filter: %{container_name!="POD",environment="#{environment.slug}"},
+              timeframe_start: 8.hours.ago.to_f,
+              timeframe_end: Time.now.to_f
+            }
+
+            query_metrics(query_context)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb
index 2a2eb4ae57f..c60828165bd 100644
--- a/lib/gitlab/prometheus/queries/base_query.rb
+++ b/lib/gitlab/prometheus/queries/base_query.rb
@@ -3,7 +3,7 @@ module Gitlab
     module Queries
       class BaseQuery
         attr_accessor :client
-        delegate :query_range, :query, to: :client, prefix: true
+        delegate :query_range, :query, :label_values, :series, to: :client, prefix: true
 
         def raw_memory_usage_query(environment_slug)
           %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20}
diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb
index 2cc08731f8d..170f483540e 100644
--- a/lib/gitlab/prometheus/queries/deployment_query.rb
+++ b/lib/gitlab/prometheus/queries/deployment_query.rb
@@ -1,26 +1,31 @@
-module Gitlab::Prometheus::Queries
-  class DeploymentQuery < BaseQuery
-    def query(deployment_id)
-      deployment = Deployment.find_by(id: deployment_id)
-      environment_slug = deployment.environment.slug
+module Gitlab
+  module Prometheus
+    module Queries
+      class DeploymentQuery < BaseQuery
+        def query(deployment_id)
+          Deployment.find_by(id: deployment_id).try do |deployment|
+            environment_slug = deployment.environment.slug
 
-      memory_query = raw_memory_usage_query(environment_slug)
-      memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))}
-      cpu_query = raw_cpu_usage_query(environment_slug)
-      cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100}
+            memory_query = raw_memory_usage_query(environment_slug)
+            memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))}
+            cpu_query = raw_cpu_usage_query(environment_slug)
+            cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100}
 
-      timeframe_start = (deployment.created_at - 30.minutes).to_f
-      timeframe_end = (deployment.created_at + 30.minutes).to_f
+            timeframe_start = (deployment.created_at - 30.minutes).to_f
+            timeframe_end = (deployment.created_at + 30.minutes).to_f
 
-      {
-        memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end),
-        memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f),
-        memory_after: client_query(memory_avg_query, time: timeframe_end),
+            {
+              memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end),
+              memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f),
+              memory_after: client_query(memory_avg_query, time: timeframe_end),
 
-        cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end),
-        cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f),
-        cpu_after: client_query(cpu_avg_query, time: timeframe_end)
-      }
+              cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end),
+              cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f),
+              cpu_after: client_query(cpu_avg_query, time: timeframe_end)
+            }
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb
index 01d756d7284..66f29d95177 100644
--- a/lib/gitlab/prometheus/queries/environment_query.rb
+++ b/lib/gitlab/prometheus/queries/environment_query.rb
@@ -1,20 +1,25 @@
-module Gitlab::Prometheus::Queries
-  class EnvironmentQuery < BaseQuery
-    def query(environment_id)
-      environment = Environment.find_by(id: environment_id)
-      environment_slug = environment.slug
-      timeframe_start = 8.hours.ago.to_f
-      timeframe_end = Time.now.to_f
+module Gitlab
+  module Prometheus
+    module Queries
+      class EnvironmentQuery < BaseQuery
+        def query(environment_id)
+          Environment.find_by(id: environment_id).try do |environment|
+            environment_slug = environment.slug
+            timeframe_start = 8.hours.ago.to_f
+            timeframe_end = Time.now.to_f
 
-      memory_query = raw_memory_usage_query(environment_slug)
-      cpu_query = raw_cpu_usage_query(environment_slug)
+            memory_query = raw_memory_usage_query(environment_slug)
+            cpu_query = raw_cpu_usage_query(environment_slug)
 
-      {
-        memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end),
-        memory_current: client_query(memory_query, time: timeframe_end),
-        cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end),
-        cpu_current: client_query(cpu_query, time: timeframe_end)
-      }
+            {
+              memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end),
+              memory_current: client_query(memory_query, time: timeframe_end),
+              cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end),
+              cpu_current: client_query(cpu_query, time: timeframe_end)
+            }
+          end
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb
new file mode 100644
index 00000000000..d4894c87f8d
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb
@@ -0,0 +1,80 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      class MatchedMetricsQuery < BaseQuery
+        MAX_QUERY_ITEMS = 40.freeze
+
+        def query
+          groups_data.map do |group, data|
+            {
+              group: group.name,
+              priority: group.priority,
+              active_metrics: data[:active_metrics],
+              metrics_missing_requirements: data[:metrics_missing_requirements]
+            }
+          end
+        end
+
+        private
+
+        def groups_data
+          metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all)
+          lookup = active_series_lookup(metrics_groups)
+
+          groups = {}
+
+          metrics_groups.each do |group|
+            groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 }
+            active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) }
+
+            groups[group][:active_metrics] += active_metrics
+            groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics
+          end
+
+          groups
+        end
+
+        def active_series_lookup(metric_groups)
+          timeframe_start = 8.hours.ago
+          timeframe_end = Time.now
+
+          series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq
+
+          lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series|
+            client_series(*batched_series, start: timeframe_start, stop: timeframe_end)
+              .select(&method(:has_matching_label))
+              .map { |series_info| [series_info['__name__'], true] }
+          end
+          lookup.to_h
+        end
+
+        def has_matching_label(series_info)
+          series_info.key?('environment')
+        end
+
+        def available_metrics
+          @available_metrics ||= client_label_values || []
+        end
+
+        def filter_active_metrics(metric_group)
+          metric_group.metrics.select! do |metric|
+            metric.required_metrics.all?(&available_metrics.method(:include?))
+          end
+          metric_group
+        end
+
+        def groups_with_active_metrics(metric_groups)
+          metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? }
+        end
+
+        def metrics_with_required_series(metric_groups)
+          metric_groups.flat_map do |group|
+            group.metrics.select do |metric|
+              metric.required_metrics.all?(&available_metrics.method(:include?))
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
new file mode 100644
index 00000000000..e44be770544
--- /dev/null
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -0,0 +1,73 @@
+module Gitlab
+  module Prometheus
+    module Queries
+      module QueryAdditionalMetrics
+        def query_metrics(query_context)
+          query_processor = method(:process_query).curry[query_context]
+
+          groups = matched_metrics.map do |group|
+            metrics = group.metrics.map do |metric|
+              {
+                title: metric.title,
+                weight: metric.weight,
+                y_label: metric.y_label,
+                queries: metric.queries.map(&query_processor).select(&method(:query_with_result))
+              }
+            end
+
+            {
+              group: group.name,
+              priority: group.priority,
+              metrics: metrics.select(&method(:metric_with_any_queries))
+            }
+          end
+
+          groups.select(&method(:group_with_any_metrics))
+        end
+
+        private
+
+        def metric_with_any_queries(metric)
+          metric[:queries]&.count&.> 0
+        end
+
+        def group_with_any_metrics(group)
+          group[:metrics]&.count&.> 0
+        end
+
+        def query_with_result(query)
+          query[:result]&.any? do |item|
+            item&.[](:values)&.any? || item&.[](:value)&.any?
+          end
+        end
+
+        def process_query(context, query)
+          query_with_result = query.dup
+          result =
+            if query.key?(:query_range)
+              client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end])
+            else
+              client_query(query[:query] % context, time: context[:timeframe_end])
+            end
+          query_with_result[:result] = result&.map(&:deep_symbolize_keys)
+          query_with_result
+        end
+
+        def available_metrics
+          @available_metrics ||= client_label_values || []
+        end
+
+        def matched_metrics
+          result = Gitlab::Prometheus::MetricGroup.all.map do |group|
+            group.metrics.select! do |metric|
+              metric.required_metrics.all?(&available_metrics.method(:include?))
+            end
+            group
+          end
+
+          result.select { |group| group.metrics.any? }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 5b51a1779dd..aa94614bf18 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -29,6 +29,14 @@ module Gitlab
       end
     end
 
+    def label_values(name = '__name__')
+      json_api_get("label/#{name}/values")
+    end
+
+    def series(*matches, start: 8.hours.ago, stop: Time.now)
+      json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
+    end
+
     private
 
     def json_api_get(type, args = {})