Add latest changes from gitlab-org/gitlab@master

25 files changed, 851 insertions, 79 deletions

diff --git a/app/models/analytics/cycle_analytics/project_stage.rb b/app/models/analytics/cycle_analytics/project_stage.rb
index a312bd24e78..23f0db0829b 100644
--- a/app/models/analytics/cycle_analytics/project_stage.rb
+++ b/app/models/analytics/cycle_analytics/project_stage.rb
@@ -9,6 +9,7 @@ module Analytics
       belongs_to :project
 
       alias_attribute :parent, :project
+      alias_attribute :parent_id, :project_id
     end
   end
 end
diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb
index 0011ba10a9a..54e9a13d1ea 100644
--- a/app/models/concerns/analytics/cycle_analytics/stage.rb
+++ b/app/models/concerns/analytics/cycle_analytics/stage.rb
@@ -47,11 +47,17 @@ module Analytics
         !custom
       end
 
-      # The model that is going to be queried, Issue or MergeRequest
-      def subject_model
+      # The model class that is going to be queried, Issue or MergeRequest
+      def subject_class
         start_event.object_type
       end
 
+      def matches_with_stage_params?(stage_params)
+        default_stage? &&
+          start_event_identifier.to_s.eql?(stage_params[:start_event_identifier].to_s) &&
+          end_event_identifier.to_s.eql?(stage_params[:end_event_identifier].to_s)
+      end
+
       private
 
       def validate_stage_event_pairs
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 7706119681d..ca50820a879 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -591,3 +591,5 @@ class MergeRequestDiff < ApplicationRecord
     end
   end
 end
+
+MergeRequestDiff.prepend_if_ee('EE::MergeRequestDiff')
diff --git a/db/migrate/20190905074652_index_timestamp_columns_for_issue_metrics.rb b/db/migrate/20190905074652_index_timestamp_columns_for_issue_metrics.rb
new file mode 100644
index 00000000000..e468b2decd9
--- /dev/null
+++ b/db/migrate/20190905074652_index_timestamp_columns_for_issue_metrics.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class IndexTimestampColumnsForIssueMetrics < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(*index_arguments)
+  end
+
+  def down
+    remove_concurrent_index(*index_arguments)
+  end
+
+  private
+
+  def index_arguments
+    [
+      :issue_metrics,
+      [:issue_id, :first_mentioned_in_commit_at, :first_associated_with_milestone_at, :first_added_to_board_at],
+      {
+        name: 'index_issue_metrics_on_issue_id_and_timestamps'
+      }
+    ]
+  end
+end
diff --git a/db/migrate/20190919104119_index_timestamp_columns_for_merge_requests_creation_date.rb b/db/migrate/20190919104119_index_timestamp_columns_for_merge_requests_creation_date.rb
new file mode 100644
index 00000000000..5e0d80630cd
--- /dev/null
+++ b/db/migrate/20190919104119_index_timestamp_columns_for_merge_requests_creation_date.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class IndexTimestampColumnsForMergeRequestsCreationDate < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(*index_arguments)
+  end
+
+  def down
+    remove_concurrent_index(*index_arguments)
+  end
+
+  private
+
+  def index_arguments
+    [
+      :merge_requests,
+      [:target_project_id, :created_at],
+      {
+        name: 'index_merge_requests_target_project_id_created_at'
+      }
+    ]
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 352023ac40d..d1bd30a131e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1816,6 +1816,7 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do
     t.datetime "first_added_to_board_at"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["issue_id", "first_mentioned_in_commit_at", "first_associated_with_milestone_at", "first_added_to_board_at"], name: "index_issue_metrics_on_issue_id_and_timestamps"
     t.index ["issue_id"], name: "index_issue_metrics"
   end
 
@@ -2226,6 +2227,7 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do
     t.index ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_id_and_source_branch"
     t.index ["state", "merge_status"], name: "index_merge_requests_on_state_and_merge_status", where: "(((state)::text = 'opened'::text) AND ((merge_status)::text = 'can_be_merged'::text))"
     t.index ["target_branch"], name: "index_merge_requests_on_target_branch"
+    t.index ["target_project_id", "created_at"], name: "index_merge_requests_target_project_id_created_at"
     t.index ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid", unique: true
     t.index ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid_opened", where: "((state)::text = 'opened'::text)"
     t.index ["target_project_id", "merge_commit_sha", "id"], name: "index_merge_requests_on_tp_id_and_merge_commit_sha_and_id"
diff --git a/doc/user/project/merge_requests/browser_performance_testing.md b/doc/user/project/merge_requests/browser_performance_testing.md
index 08feb970334..3cfc30a68e9 100644
--- a/doc/user/project/merge_requests/browser_performance_testing.md
+++ b/doc/user/project/merge_requests/browser_performance_testing.md
@@ -44,7 +44,7 @@ For instance, consider the following workflow:
 First of all, you need to define a job in your `.gitlab-ci.yml` file that generates the
 [Performance report artifact](../../../ci/yaml/README.md#artifactsreportsperformance-premium).
 For more information on how the Performance job should look like, check the
-example on [Testing Browser Performance](../../../ci/examples/browser_performance.md).
+example on [Configuring Browser Performance Testing](#configuring-browser-performance-testing).
 
 GitLab then checks this report, compares key performance metrics for each page
 between the source and target branches, and shows the information right on the merge request.
@@ -60,11 +60,6 @@ report will be shown properly.
 
 ## Configuring Browser Performance Testing
 
-NOTE: **Note:**
-The job definition shown below is supported in GitLab 11.5 and later versions.
-It also requires GitLab Runner 11.5 or later. For earlier versions, use the
-[previous job definitions](#previous-job-definitions).
-
 This example shows how to run the [sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/)
 on your code by using GitLab CI/CD and [sitespeed.io](https://www.sitespeed.io)
 using Docker-in-Docker.
@@ -73,29 +68,35 @@ First, you need GitLab Runner with
 [docker-in-docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor).
 
 Once you set up the Runner, add a new job to `.gitlab-ci.yml` that generates the
-expected report:
+expected report.
+
+For GitLab 12.4 and later, to define the `performance` job, you must
+[include](../../../ci/yaml/README.md#includetemplate) the
+[`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml)
+that's provided as a part of your GitLab installation.
+For GitLab versions earlier than 12.4, you can copy and use the job as defined
+in that template.
+
+CAUTION: **Caution:**
+The job definition provided by the template does not support Kubernetes yet. For a complete example of a more complex setup
+that works in Kubernetes, see [here](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml).
+
+Add the following to your `.gitlab-ci.yml` file:
 
 ```yaml
+include:
+  template: Verify/Browser-Performance.gitlab-ci.yml
+
 performance:
-  stage: performance
-  image: docker:git
   variables:
     URL: https://example.com
-  services:
-    - docker:stable-dind
-  script:
-    - mkdir gitlab-exporter
-    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
-    - mkdir sitespeed-results
-    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL
-    - mv sitespeed-results/data/performance.json performance.json
-  artifacts:
-    paths:
-      - sitespeed-results/
-    reports:
-      performance: performance.json
 ```
 
+CAUTION: **Caution:**
+The job definition provided by the template is supported in GitLab 11.5 and later versions.
+It also requires GitLab Runner 11.5 or later. For earlier versions, use the
+[previous job definitions](#previous-job-definitions).
+
 The above example will create a `performance` job in your CI/CD pipeline and will run
 sitespeed.io against the webpage you defined in `URL` to gather key metrics.
 The [GitLab plugin for sitespeed.io](https://gitlab.com/gitlab-org/gl-performance)
@@ -106,6 +107,20 @@ take the latest Performance artifact available.
 The full HTML sitespeed.io report will also be saved as an artifact, and if you have
 [GitLab Pages](../pages/index.md) enabled, it can be viewed directly in your browser.
 
+It is also possible to customize options by setting the `SITESPEED_OPTIONS` variable.
+For example, this is how to override the number of runs sitespeed.io
+will make on the given URL:
+
+```yaml
+include:
+  template: Verify/Browser-Performance.gitlab-ci.yml
+
+performance:
+  variables:
+    URL: https://example.com
+    SITESPEED_OPTIONS: -n 5
+```
+
 For further customization options for sitespeed.io, including the ability to provide a
 list of URLs to test, please see the [Sitespeed.io Configuration](https://www.sitespeed.io/documentation/sitespeed.io/configuration/)
 documentation.
@@ -126,8 +141,9 @@ set this up:
    as an artifact is as simple as `echo $CI_ENVIRONMENT_URL > environment_url.txt`
    in your job's `script`.
 1. In the `performance` job, read the previous artifact into an environment
-   variable, like `$CI_ENVIRONMENT_URL`, and use it to parameterize the test
-   URLs.
+   variable, in this case `$URL` because this is what our sitespeed.io command
+   uses for the URL parameter. Because Review App URLs are dynamic, we define
+   the `URL` variable through `before_script` instead of `variables`.
 1. You can now run the sitespeed.io container against the desired hostname and
    paths.
 
@@ -138,6 +154,9 @@ stages:
   - deploy
   - performance
 
+include:
+  template: Verify/Browser-Performance.gitlab-ci.yml
+
 review:
   stage: deploy
   environment:
@@ -155,28 +174,12 @@ review:
     - master
 
 performance:
-  stage: performance
-  image: docker:git
-  services:
-    - docker:stable-dind
   dependencies:
     - review
-  script:
-    - export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
-    - mkdir gitlab-exporter
-    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
-    - mkdir sitespeed-results
-    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
-    - mv sitespeed-results/data/performance.json performance.json
-  artifacts:
-    paths:
-      - sitespeed-results/
-    reports:
-      performance: performance.json
+  before_script:
+    - export URL=$(cat environment_url.txt)
 ```
 
-A complete example can be found in our [Auto DevOps CI YML](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml).
-
 ### Previous job definitions
 
 CAUTION: **Caution:**
diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
new file mode 100644
index 00000000000..33cbe1a62ef
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class BaseQueryBuilder
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        delegate :subject_class, to: :stage
+
+        # rubocop: disable CodeReuse/ActiveRecord
+
+        def initialize(stage:, params: {})
+          @stage = stage
+          @params = params
+        end
+
+        def build
+          query = subject_class
+          query = filter_by_parent_model(query)
+          query = filter_by_time_range(query)
+          query = stage.start_event.apply_query_customization(query)
+          query = stage.end_event.apply_query_customization(query)
+          query.where(duration_condition)
+        end
+
+        private
+
+        attr_reader :stage, :params
+
+        def duration_condition
+          stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection)
+        end
+
+        def filter_by_parent_model(query)
+          if parent_class.eql?(Project)
+            if subject_class.eql?(Issue)
+              query.where(project_id: stage.parent_id)
+            elsif subject_class.eql?(MergeRequest)
+              query.where(target_project_id: stage.parent_id)
+            else
+              raise ArgumentError, "unknown subject_class: #{subject_class}"
+            end
+          else
+            raise ArgumentError, "unknown parent_class: #{parent_class}"
+          end
+        end
+
+        def filter_by_time_range(query)
+          from = params.fetch(:from, 30.days.ago)
+          to = params[:to]
+
+          query = query.where(subject_table[:created_at].gteq(from))
+          query = query.where(subject_table[:created_at].lteq(to)) if to
+          query
+        end
+
+        def subject_table
+          subject_class.arel_table
+        end
+
+        def parent_class
+          stage.parent.class
+        end
+
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
new file mode 100644
index 00000000000..0c0f737f2c9
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      # Arguments:
+      #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+      #   params:
+      #     current_user: an instance of User
+      #     from: DateTime
+      #     to: DateTime
+      class DataCollector
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(stage:, params: {})
+          @stage = stage
+          @params = params
+        end
+
+        def records_fetcher
+          strong_memoize(:records_fetcher) do
+            RecordsFetcher.new(stage: stage, query: query, params: params)
+          end
+        end
+
+        def median
+          strong_memoize(:median) do
+            Median.new(stage: stage, query: query)
+          end
+        end
+
+        private
+
+        attr_reader :stage, :params
+
+        def query
+          BaseQueryBuilder.new(stage: stage, params: params).build
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
index 711645800fb..8e70236ce75 100644
--- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb
+++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb
@@ -92,8 +92,8 @@ module Gitlab
             name: 'production',
             custom: false,
             relative_position: 7,
-            start_event_identifier: :merge_request_merged,
-            end_event_identifier: :merge_request_first_deployed_to_production
+            start_event_identifier: :issue_created,
+            end_event_identifier: :production_stage_end
           }
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
new file mode 100644
index 00000000000..41883a80338
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class Median
+        include StageQueryHelpers
+
+        def initialize(stage:, query:)
+          @stage = stage
+          @query = query
+        end
+
+        def seconds
+          @query = @query.select(median_duration_in_seconds.as('median'))
+          result = execute_query(@query).first || {}
+
+          result['median'] ? result['median'].to_i : nil
+        end
+
+        private
+
+        attr_reader :stage
+
+        def percentile_cont
+          percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+          Arel::Nodes::NamedFunction.new(
+            'percentile_cont(0.5) WITHIN GROUP',
+            [percentile_cont_ordering]
+          )
+        end
+
+        def median_duration_in_seconds
+          Arel::Nodes::Extract.new(percentile_cont, :epoch)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
new file mode 100644
index 00000000000..90d03142b2a
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      class RecordsFetcher
+        include Gitlab::Utils::StrongMemoize
+        include StageQueryHelpers
+        include Gitlab::CycleAnalytics::MetricsTables
+
+        MAX_RECORDS = 20
+
+        MAPPINGS = {
+          Issue => {
+            finder_class: IssuesFinder,
+            serializer_class: AnalyticsIssueSerializer,
+            includes_for_query: { project: [:namespace], author: [] },
+            columns_for_select: %I[title iid id created_at author_id project_id]
+          },
+          MergeRequest => {
+            finder_class: MergeRequestsFinder,
+            serializer_class: AnalyticsMergeRequestSerializer,
+            includes_for_query: { target_project: [:namespace], author: [] },
+            columns_for_select: %I[title iid id created_at author_id state target_project_id]
+          }
+        }.freeze
+
+        delegate :subject_class, to: :stage
+
+        def initialize(stage:, query:, params: {})
+          @stage = stage
+          @query = query
+          @params = params
+        end
+
+        def serialized_records
+          strong_memoize(:serialized_records) do
+            # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records
+            if default_test_stage? || default_staging_stage?
+              AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] })
+            else
+              records.map do |record|
+                project = record.project
+                attributes = record.attributes.merge({
+                  project_path: project.path,
+                  namespace_path: project.namespace.path,
+                  author: record.author
+                })
+                serializer.represent(attributes)
+              end
+            end
+          end
+        end
+
+        private
+
+        attr_reader :stage, :query, :params
+
+        def finder_query
+          MAPPINGS
+            .fetch(subject_class)
+            .fetch(:finder_class)
+            .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class))
+            .execute
+        end
+
+        def columns
+          MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name|
+            subject_class.arel_table[column_name]
+          end
+        end
+
+        # EE will override this to include Group rules
+        def finder_params
+          {
+            Project => { project_id: stage.parent_id }
+          }
+        end
+
+        def default_test_stage?
+          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage)
+        end
+
+        def default_staging_stage?
+          stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage)
+        end
+
+        def serializer
+          MAPPINGS.fetch(subject_class).fetch(:serializer_class).new
+        end
+
+        # Loading Ci::Build records instead of MergeRequest records
+        # rubocop: disable CodeReuse/ActiveRecord
+        def ci_build_records
+          ci_build_join = mr_metrics_table
+            .join(build_table)
+            .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
+            .join_sources
+
+          q = ordered_and_limited_query
+            .joins(ci_build_join)
+            .select(build_table[:id], round_duration_to_seconds.as('total_time'))
+
+          results = execute_query(q).to_a
+
+          Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] }))
+        end
+
+        def ordered_and_limited_query
+          query
+            .reorder(stage.end_event.timestamp_projection.desc)
+            .limit(MAX_RECORDS)
+        end
+
+        def records
+          results = finder_query
+            .merge(ordered_and_limited_query)
+            .select(*columns, round_duration_to_seconds.as('total_time'))
+
+          # using preloader instead of includes to avoid AR generating a large column list
+          ActiveRecord::Associations::Preloader.new.preload(
+            results,
+            MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
+          )
+
+          results
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
new file mode 100644
index 00000000000..34c726b2254
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageQueryHelpers
+        def execute_query(query)
+          ActiveRecord::Base.connection.execute(query.to_sql)
+        end
+
+        def zero_interval
+          Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")])
+        end
+
+        def round_duration_to_seconds
+          Arel::Nodes::Extract.new(duration, :epoch)
+        end
+
+        def duration
+          Arel::Nodes::Subtraction.new(
+            stage.end_event.timestamp_projection,
+            stage.start_event.timestamp_projection
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
new file mode 100644
index 00000000000..eced181e966
--- /dev/null
+++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
@@ -0,0 +1,29 @@
+# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html
+
+stages:
+  - build
+  - test
+  - deploy
+  - performance
+
+performance:
+  stage: performance
+  image: docker:git
+  variables:
+    URL: https://example.com
+    SITESPEED_VERSION: 6.3.1
+    SITESPEED_OPTIONS: ''
+  services:
+    - docker:stable-dind
+  script:
+    - mkdir gitlab-exporter
+    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
+    - mkdir sitespeed-results
+    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
+    - mv sitespeed-results/data/performance.json performance.json
+  artifacts:
+    paths:
+      - performance.json
+      - sitespeed-results/
+    reports:
+      performance: performance.json
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 8ec7a725871..c10d94d96ab 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -6030,6 +6030,15 @@ msgstr ""
 msgid "Environments|An error occurred while fetching the environments."
 msgstr ""
 
+msgid "Environments|An error occurred while fetching the logs"
+msgstr ""
+
+msgid "Environments|An error occurred while fetching the logs - Error: %{message}"
+msgstr ""
+
+msgid "Environments|An error occurred while fetching the logs for this environment or pod. Please try again"
+msgstr ""
+
 msgid "Environments|An error occurred while making the request."
 msgstr ""
 
@@ -6075,9 +6084,6 @@ msgstr ""
 msgid "Environments|No deployments yet"
 msgstr ""
 
-msgid "Environments|No pod name has been specified"
-msgstr ""
-
 msgid "Environments|Note that this action will stop the environment, but it will %{emphasisStart}not%{emphasisEnd} have an effect on any existing deployment due to no “stop environment action” being defined in the %{ciConfigLinkStart}.gitlab-ci.yml%{ciConfigLinkEnd} file."
 msgstr ""
 
@@ -14986,9 +14992,6 @@ msgstr ""
 msgid "Something went wrong on our end."
 msgstr ""
 
-msgid "Something went wrong on our end. %{message}"
-msgstr ""
-
 msgid "Something went wrong on our end. Please try again!"
 msgstr ""
 
diff --git a/spec/factories/analytics/cycle_analytics/project_stages.rb b/spec/factories/analytics/cycle_analytics/project_stages.rb
new file mode 100644
index 00000000000..6f8c140ed8a
--- /dev/null
+++ b/spec/factories/analytics/cycle_analytics/project_stages.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :cycle_analytics_project_stage, class: Analytics::CycleAnalytics::ProjectStage do
+    project
+    sequence(:name) { |n| "Stage ##{n}" }
+    hidden { false }
+    issue_stage
+
+    trait :issue_stage do
+      start_event_identifier { Gitlab::Analytics::CycleAnalytics::StageEvents::IssueCreated.identifier }
+      end_event_identifier { Gitlab::Analytics::CycleAnalytics::StageEvents::IssueStageEnd.identifier }
+    end
+  end
+end
diff --git a/spec/frontend/fixtures/static/environments_logs.html b/spec/frontend/fixtures/static/environments_logs.html
index 6179d3dbc23..ccf9c364154 100644
--- a/spec/frontend/fixtures/static/environments_logs.html
+++ b/spec/frontend/fixtures/static/environments_logs.html
@@ -1,29 +1,102 @@
-<div class="js-kubernetes-logs" data-logs-path="/root/kubernetes-app/environments/1/logs">
-  <div class="build-page">
+<div
+  class="js-kubernetes-logs"
+  data-current-environment-name="production"
+  data-environments-path="/root/my-project/environments.json"
+  data-logs-page="/root/my-project/environments/1/logs"
+  data-logs-path="/root/my-project/environments/1/logs.json"
+>
+  <div class="build-page-pod-logs">
     <div class="build-trace-container prepend-top-default">
-      <div class="top-bar js-top-bar">
-        <div class="truncated-info hidden-xs pull-left"></div>
-        <div class="dropdown prepend-left-10 js-pod-dropdown">
-          <button aria-expanded="false" class="dropdown-menu-toggle" data-toggle="dropdown" type="button">
-            <i class="fa fa-chevron-down"></i>
-          </button>
-          <div class="dropdown-menu dropdown-menu-selectable dropdown-menu-drop-up"></div>
+      <div class="top-bar js-top-bar d-flex">
+        <div class="row">
+          <div class="form-group col-6" role="group">
+            <label class="d-block col-form-label-sm col-form-label">
+              Environment
+            </label>
+            <div class="dropdown js-environment-dropdown d-flex">
+              <button
+                aria-expanded="false"
+                class="dropdown-menu-toggle d-flex align-content-center align-self-center"
+                data-toggle="dropdown"
+                type="button"
+              >
+                <i aria-hidden="true" data-hidden="true" class="fa fa-chevron-down"></i>
+                <div class="dropdown-toggle-text">
+                  &nbsp;
+                </div>
+              </button>
+              <div class="dropdown-menu dropdown-menu-selectable dropdown-menu-drop-up"></div>
+            </div>
+          </div>
+          <div class="form-group col-6" role="group">
+            <label class="d-block col-form-label-sm col-form-label">
+              Pod logs from
+            </label>
+            <div class="dropdown js-pod-dropdown d-flex">
+              <button
+                aria-expanded="false"
+                class="dropdown-menu-toggle d-flex align-content-center align-self-center"
+                data-toggle="dropdown"
+                type="button"
+              >
+                <i aria-hidden="true" data-hidden="true" class="fa fa-chevron-down"></i>
+                <div class="dropdown-toggle-text">
+                  &nbsp;
+                </div>
+              </button>
+              <div class="dropdown-menu dropdown-menu-selectable dropdown-menu-drop-up"></div>
+            </div>
+          </div>
         </div>
-        <div class="controllers pull-right">
-          <div class="has-tooltip controllers-buttons" data-container="body" data-placement="top" title="Scroll to top">
-            <button class="js-scroll-up btn-scroll btn-transparent btn-blank" disabled type="button"></button>
+        <div class="controllers align-self-end">
+          <div
+            class="has-tooltip controllers-buttons"
+            data-container="body"
+            data-placement="top"
+            title="Scroll to top"
+          >
+            <button
+              class="js-scroll-up btn-scroll btn-transparent btn-blank"
+              disabled
+              type="button"
+            ></button>
           </div>
-          <div class="has-tooltip controllers-buttons" data-container="body" data-placement="top" title="Scroll to bottom">
-            <button class="js-scroll-down btn-scroll btn-transparent btn-blank" disabled type="button"></button>
+          <div
+            class="has-tooltip controllers-buttons"
+            data-container="body"
+            data-placement="top"
+            title="Scroll to bottom"
+          >
+            <button
+              class="js-scroll-down btn-scroll btn-transparent btn-blank"
+              disabled
+              type="button"
+            ></button>
           </div>
-          <div class="refresh-control pull-right">
-            <div class="has-tooltip controllers-buttons" data-container="body" data-placement="top" title="Refresh">
-              <button class="js-refresh-log btn-default btn-refresh" disabled type="button"></button>
+          <div class="refresh-control">
+            <div
+              class="has-tooltip controllers-buttons"
+              data-container="body"
+              data-placement="top"
+              title="Refresh"
+            >
+              <button
+                class="js-refresh-log btn btn-default btn-refresh h-32-px"
+                disabled
+                type="button"
+              ></button>
             </div>
           </div>
         </div>
       </div>
-      <pre class="build-trace" id="build-trace"><code class="bash js-build-output"><div class="build-loader-animation js-build-refresh"></div></code></pre>
+      <pre class="build-trace" id="build-trace">
+        <code class="bash js-build-output"></code>
+        <div class="build-loader-animation js-build-refresh">
+          <div class="dot"></div>
+          <div class="dot"></div>
+          <div class="dot"></div>
+        </div>
+      </pre>
     </div>
   </div>
 </div>
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb
new file mode 100644
index 00000000000..0fc9d3c1e9e
--- /dev/null
+++ b/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder do
+  let_it_be(:project) { create(:project, :empty_repo) }
+  let_it_be(:mr1) { create(:merge_request, target_project: project, source_project: project, allow_broken: true, created_at: 3.months.ago) }
+  let_it_be(:mr2) { create(:merge_request, target_project: project, source_project: project, allow_broken: true, created_at: 1.month.ago) }
+  let(:params) { {} }
+  let(:records) do
+    stage = build(:cycle_analytics_project_stage, {
+      start_event_identifier: :merge_request_created,
+      end_event_identifier: :merge_request_merged,
+      project: project
+    })
+    described_class.new(stage: stage, params: params).build.to_a
+  end
+
+  before do
+    mr1.metrics.update!(merged_at: 1.month.ago)
+    mr2.metrics.update!(merged_at: Time.now)
+  end
+
+  around do |example|
+    Timecop.freeze { example.run }
+  end
+
+  describe 'date range parameters' do
+    context 'when filters by only the `from` parameter' do
+      before do
+        params[:from] = 4.months.ago
+      end
+
+      it { expect(records.size).to eq(2) }
+    end
+
+    context 'when filters by both `from` and `to` parameters' do
+      before do
+        params.merge!(from: 4.months.ago, to: 2.months.ago)
+      end
+
+      it { expect(records.size).to eq(1) }
+    end
+
+    context 'invalid date range is provided' do
+      before do
+        params.merge!(from: 1.month.ago, to: 10.months.ago)
+      end
+
+      it { expect(records.size).to eq(0) }
+    end
+  end
+
+  it 'scopes query within the target project' do
+    other_mr = create(:merge_request, source_project: create(:project), allow_broken: true, created_at: 2.months.ago)
+    other_mr.metrics.update!(merged_at: 1.month.ago)
+
+    params[:from] = 1.year.ago
+
+    expect(records.size).to eq(2)
+  end
+end
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb
new file mode 100644
index 00000000000..334cab0b799
--- /dev/null
+++ b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::RecordsFetcher do
+  around do |example|
+    Timecop.freeze { example.run }
+  end
+
+  let_it_be(:project) { create(:project, :empty_repo) }
+  let_it_be(:user) { create(:user) }
+
+  subject do
+    Gitlab::Analytics::CycleAnalytics::DataCollector.new(
+      stage: stage,
+      params: {
+        from: 1.year.ago,
+        current_user: user
+      }
+    ).records_fetcher.serialized_records
+  end
+
+  describe '#serialized_records' do
+    shared_context 'when records are loaded by maintainer' do
+      before do
+        project.add_user(user, Gitlab::Access::MAINTAINER)
+      end
+
+      it 'returns all records' do
+        expect(subject.size).to eq(2)
+      end
+    end
+
+    describe 'for issue based stage' do
+      let_it_be(:issue1) { create(:issue, project: project) }
+      let_it_be(:issue2) { create(:issue, project: project, confidential: true) }
+      let(:stage) do
+        build(:cycle_analytics_project_stage, {
+          start_event_identifier: :plan_stage_start,
+          end_event_identifier: :issue_first_mentioned_in_commit,
+          project: project
+        })
+      end
+
+      before do
+        issue1.metrics.update(first_added_to_board_at: 3.days.ago, first_mentioned_in_commit_at: 2.days.ago)
+        issue2.metrics.update(first_added_to_board_at: 3.days.ago, first_mentioned_in_commit_at: 2.days.ago)
+      end
+
+      context 'when records are loaded by guest' do
+        before do
+          project.add_user(user, Gitlab::Access::GUEST)
+        end
+
+        it 'filters out confidential issues' do
+          expect(subject.size).to eq(1)
+          expect(subject.first[:iid].to_s).to eq(issue1.iid.to_s)
+        end
+      end
+
+      include_context 'when records are loaded by maintainer'
+    end
+
+    describe 'for merge request based stage' do
+      let(:mr1) { create(:merge_request, created_at: 5.days.ago, source_project: project, allow_broken: true) }
+      let(:mr2) { create(:merge_request, created_at: 4.days.ago, source_project: project, allow_broken: true) }
+      let(:stage) do
+        build(:cycle_analytics_project_stage, {
+          start_event_identifier: :merge_request_created,
+          end_event_identifier: :merge_request_merged,
+          project: project
+        })
+      end
+
+      before do
+        mr1.metrics.update(merged_at: 3.days.ago)
+        mr2.metrics.update(merged_at: 3.days.ago)
+      end
+
+      include_context 'when records are loaded by maintainer'
+    end
+
+    describe 'special case' do
+      let(:mr1) { create(:merge_request, source_project: project, allow_broken: true, created_at: 20.days.ago) }
+      let(:mr2) { create(:merge_request, source_project: project, allow_broken: true, created_at: 19.days.ago) }
+      let(:ci_build1) { create(:ci_build) }
+      let(:ci_build2) { create(:ci_build) }
+      let(:default_stages) { Gitlab::Analytics::CycleAnalytics::DefaultStages }
+      let(:stage) { build(:cycle_analytics_project_stage, default_stages.params_for_test_stage.merge(project: project)) }
+
+      before do
+        mr1.metrics.update!({
+          merged_at: 5.days.ago,
+          first_deployed_to_production_at: 1.day.ago,
+          latest_build_started_at: 5.days.ago,
+          latest_build_finished_at: 1.day.ago,
+          pipeline: ci_build1.pipeline
+        })
+        mr2.metrics.update!({
+          merged_at: 10.days.ago,
+          first_deployed_to_production_at: 5.days.ago,
+          latest_build_started_at: 9.days.ago,
+          latest_build_finished_at: 7.days.ago,
+          pipeline: ci_build2.pipeline
+        })
+      end
+
+      context 'returns build records' do
+        shared_examples 'orders build records by `latest_build_finished_at`' do
+          it 'orders by `latest_build_finished_at`' do
+            build_ids = subject.map { |item| item[:id] }
+
+            expect(build_ids).to eq([ci_build1.id, ci_build2.id])
+          end
+        end
+
+        context 'when requesting records for default test stage' do
+          include_examples 'orders build records by `latest_build_finished_at`'
+        end
+
+        context 'when requesting records for default staging stage' do
+          before do
+            stage.assign_attributes(default_stages.params_for_staging_stage)
+          end
+
+          include_examples 'orders build records by `latest_build_finished_at`'
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb
index 6959778661b..aa12bc21d22 100644
--- a/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb
@@ -26,6 +26,13 @@ describe Gitlab::CycleAnalytics::CodeStage do
 
   it_behaves_like 'base stage'
 
+  context 'when using the new query backend' do
+    include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do
+      let(:expected_record_count) { 2 }
+      let(:expected_ordered_attribute_values) { [mr_2.title, mr_1.title] }
+    end
+  end
+
   describe '#project_median' do
     around do |example|
       Timecop.freeze { example.run }
diff --git a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb
index b99debf8cac..497db88d850 100644
--- a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb
@@ -21,6 +21,13 @@ describe Gitlab::CycleAnalytics::IssueStage do
 
   it_behaves_like 'base stage'
 
+  context 'when using the new query backend' do
+    include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do
+      let(:expected_record_count) { 3 }
+      let(:expected_ordered_attribute_values) { [issue_3.title, issue_2.title, issue_1.title] }
+    end
+  end
+
   describe '#median' do
     around do |example|
       Timecop.freeze { example.run }
diff --git a/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb
index 60d180015c4..01a46f5ba65 100644
--- a/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb
@@ -21,6 +21,13 @@ describe Gitlab::CycleAnalytics::PlanStage do
 
   it_behaves_like 'base stage'
 
+  context 'when using the new query backend' do
+    include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do
+      let(:expected_record_count) { 2 }
+      let(:expected_ordered_attribute_values) { [issue_1.title, issue_2.title] }
+    end
+  end
+
   describe '#project_median' do
     around do |example|
       Timecop.freeze { example.run }
diff --git a/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb
index b3a6c2c4239..c5b17aafdd2 100644
--- a/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb
@@ -52,3 +52,21 @@ shared_examples 'calculate #median with date range' do
     it { expect(stage.project_median).to eq(nil) }
   end
 end
+
+shared_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do
+  let(:stage_params) { Gitlab::Analytics::CycleAnalytics::DefaultStages.send("params_for_#{stage_name}_stage").merge(project: project) }
+  let(:stage) { Analytics::CycleAnalytics::ProjectStage.new(stage_params) }
+  let(:data_collector) { Gitlab::Analytics::CycleAnalytics::DataCollector.new(stage: stage, params: { from: stage_options[:from], current_user: project.creator }) }
+  let(:attribute_to_verify) { :title }
+
+  context 'provides the same results as the old implementation' do
+    it 'for the median' do
+      expect(data_collector.median.seconds).to eq(ISSUES_MEDIAN)
+    end
+
+    it 'for the list of event records' do
+      records = data_collector.records_fetcher.serialized_records
+      expect(records.map { |event| event[attribute_to_verify] }).to eq(expected_ordered_attribute_values)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb
index bbb53cefae7..e347f36dfce 100644
--- a/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb
+++ b/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb
@@ -12,17 +12,20 @@ describe Gitlab::CycleAnalytics::TestStage do
   it_behaves_like 'base stage'
 
   describe '#median' do
+    let(:mr_1) { create(:merge_request, :closed, source_project: project, created_at: 60.minutes.ago) }
+    let(:mr_2) { create(:merge_request, :closed, source_project: project, created_at: 40.minutes.ago, source_branch: 'A') }
+    let(:mr_3) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'B') }
+    let(:mr_4) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'C') }
+    let(:mr_5) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'D') }
+    let(:ci_build1) { create(:ci_build, project: project) }
+    let(:ci_build2) { create(:ci_build, project: project) }
+
     before do
       issue_1 = create(:issue, project: project, created_at: 90.minutes.ago)
       issue_2 = create(:issue, project: project, created_at: 60.minutes.ago)
       issue_3 = create(:issue, project: project, created_at: 60.minutes.ago)
-      mr_1 = create(:merge_request, :closed, source_project: project, created_at: 60.minutes.ago)
-      mr_2 = create(:merge_request, :closed, source_project: project, created_at: 40.minutes.ago, source_branch: 'A')
-      mr_3 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'B')
-      mr_4 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'C')
-      mr_5 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'D')
-      mr_1.metrics.update!(latest_build_started_at: 32.minutes.ago, latest_build_finished_at: 2.minutes.ago)
-      mr_2.metrics.update!(latest_build_started_at: 62.minutes.ago, latest_build_finished_at: 32.minutes.ago)
+      mr_1.metrics.update!(latest_build_started_at: 32.minutes.ago, latest_build_finished_at: 2.minutes.ago, pipeline_id: ci_build1.commit_id)
+      mr_2.metrics.update!(latest_build_started_at: 62.minutes.ago, latest_build_finished_at: 32.minutes.ago, pipeline_id: ci_build2.commit_id)
       mr_3.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil)
       mr_4.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil)
       mr_5.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil)
@@ -43,5 +46,13 @@ describe Gitlab::CycleAnalytics::TestStage do
     end
 
     include_examples 'calculate #median with date range'
+
+    context 'when using the new query backend' do
+      include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do
+        let(:expected_record_count) { 2 }
+        let(:attribute_to_verify) { :id }
+        let(:expected_ordered_attribute_values) { [mr_1.metrics.pipeline.builds.first.id, mr_2.metrics.pipeline.builds.first.id] }
+      end
+    end
   end
 end
diff --git a/spec/support/shared_examples/cycle_analytics_stage_examples.rb b/spec/support/shared_examples/cycle_analytics_stage_shared_examples.rb
index dc2ea229171..afa035d039a 100644
--- a/spec/support/shared_examples/cycle_analytics_stage_examples.rb
+++ b/spec/support/shared_examples/cycle_analytics_stage_shared_examples.rb
@@ -55,11 +55,11 @@ shared_examples_for 'cycle analytics stage' do
     end
   end
 
-  describe '#subject_model' do
+  describe '#subject_class' do
     it 'infers the model from the start event' do
       stage = described_class.new(valid_params)
 
-      expect(stage.subject_model).to eq(MergeRequest)
+      expect(stage.subject_class).to eq(MergeRequest)
     end
   end
 
@@ -78,4 +78,30 @@ shared_examples_for 'cycle analytics stage' do
       expect(stage.end_event).to be_a_kind_of(Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestMerged)
     end
   end
+
+  describe '#matches_with_stage_params?' do
+    let(:params) { Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage }
+
+    it 'matches with default stage params' do
+      stage = described_class.new(params)
+
+      expect(stage).to be_default_stage
+      expect(stage).to be_matches_with_stage_params(params)
+    end
+
+    it "mismatches when the stage is custom" do
+      stage = described_class.new(params.merge(custom: true))
+
+      expect(stage).not_to be_default_stage
+      expect(stage).not_to be_matches_with_stage_params(params)
+    end
+  end
+
+  describe '#parent_id' do
+    it "delegates to 'parent_name'_id attribute" do
+      stage = described_class.new(parent: parent)
+
+      expect(stage.parent_id).to eq(parent.id)
+    end
+  end
 end