Add latest changes from gitlab-org/gitlab@13-0-stable-ee

15 files changed, 357 insertions, 94 deletions

diff --git a/scripts/build_assets_image b/scripts/build_assets_image
index 6354c2df798..9eb1ccd5515 100755
--- a/scripts/build_assets_image
+++ b/scripts/build_assets_image
@@ -1,5 +1,3 @@
-#!/bin/bash
-
 # Exit early if we don't want to build the image
 if [[ "${BUILD_ASSETS_IMAGE}" != "true" ]]
 then
@@ -19,15 +17,19 @@ ASSETS_IMAGE_PATH=${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME}
 mkdir -p assets_container.build/public
 cp -r public/assets assets_container.build/public/
 cp Dockerfile.assets assets_container.build/
-docker build -t ${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG} -f assets_container.build/Dockerfile.assets assets_container.build/
-docker tag ${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG} ${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
-docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
-docker push ${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG}
-docker push ${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
+
+COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG}
+COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
+COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
+
+DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION"
 
 # Also tag the image with GitLab version, if running on a tag pipeline, so
 # other projects can simply use that instead of computing the slug.
 if [ -n "$CI_COMMIT_TAG" ]; then
-    docker tag ${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG} ${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
-    docker push ${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
+  DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION"
 fi
+
+echo "building assets image for destinations: $DESTINATIONS"
+
+/kaniko/executor --context=assets_container.build --dockerfile=assets_container.build/Dockerfile.assets $DESTINATIONS
diff --git a/scripts/clean-old-cached-assets b/scripts/clean-old-cached-assets
index 9a373439e5e..20889b7ffe6 100755
--- a/scripts/clean-old-cached-assets
+++ b/scripts/clean-old-cached-assets
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Clean up cached files that are older than 4 days
 find tmp/cache/assets/sprockets/ -type f -mtime +4 -execdir rm -- "{}" \;
diff --git a/scripts/create_postgres_user.sh b/scripts/create_postgres_user.sh
index 8a049bcd7fb..4c9db68c9d3 100644
--- a/scripts/create_postgres_user.sh
+++ b/scripts/create_postgres_user.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 psql -h postgres -U postgres postgres <<EOF
 CREATE USER gitlab;
diff --git a/scripts/frontend/webpack_dev_server.js b/scripts/frontend/webpack_dev_server.js
new file mode 100755
index 00000000000..8026a8d47e2
--- /dev/null
+++ b/scripts/frontend/webpack_dev_server.js
@@ -0,0 +1,68 @@
+const nodemon = require('nodemon');
+
+const DEV_SERVER_HOST = process.env.DEV_SERVER_HOST || 'localhost';
+const DEV_SERVER_PORT = process.env.DEV_SERVER_PORT || '3808';
+const STATIC_MODE = process.env.DEV_SERVER_STATIC && process.env.DEV_SERVER_STATIC != 'false';
+const DLL_MODE = process.env.WEBPACK_VENDOR_DLL && process.env.WEBPACK_VENDOR_DLL != 'false';
+
+const baseConfig = {
+  ignoreRoot: ['.git', 'node_modules/*/'],
+  noUpdateNotifier: true,
+  signal: 'SIGTERM',
+  delay: 1000,
+};
+
+// run webpack in compile-once mode and watch for changes
+if (STATIC_MODE) {
+  nodemon({
+    exec: `rm -rf public/assets/webpack ; yarn run webpack && exec ruby -run -e httpd public/ -p ${DEV_SERVER_PORT}`,
+    watch: [
+      'config/webpack.config.js',
+      'app/assets/javascripts',
+      'ee/app/assets/javascripts',
+      // ensure we refresh when running yarn install
+      'node_modules/.yarn-integrity',
+    ],
+    ext: 'js,json,vue',
+    ...baseConfig,
+  });
+}
+
+// run webpack through webpack-dev-server, optionally compiling a DLL to reduce memory
+else {
+  let watch = ['config/webpack.config.js'];
+
+  // if utilizing the vendor DLL, we need to restart the process when dependency changes occur
+  if (DLL_MODE) {
+    watch.push(
+      'config/webpack.vendor.config.js',
+      // ensure we refresh when running yarn install
+      'node_modules/.yarn-integrity',
+      'package.json',
+      'yarn.lock',
+    );
+  }
+  nodemon({
+    exec: 'webpack-dev-server --config config/webpack.config.js',
+    watch,
+    ...baseConfig,
+  });
+}
+
+// print useful messages for nodemon events
+nodemon
+  .on('start', function() {
+    console.log(`Starting webpack webserver on http://${DEV_SERVER_HOST}:${DEV_SERVER_PORT}`);
+    if (STATIC_MODE) {
+      console.log('You are starting webpack in compile-once mode');
+      console.log('The JavaScript assets are recompiled only if they change');
+      console.log('If you change them often, you might want to unset DEV_SERVER_STATIC');
+    }
+  })
+  .on('quit', function() {
+    console.log('Shutting down webpack process');
+    process.exit();
+  })
+  .on('restart', function(files) {
+    console.log('Restarting webpack process due to: ', files);
+  });
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index 8db47afdd4d..e6f2c9885d9 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -20,7 +20,7 @@ module GitalyTest
       'HOME' => File.expand_path('tmp/tests'),
       'GEM_PATH' => Gem.path.join(':'),
       'BUNDLE_APP_CONFIG' => File.join(File.dirname(gemfile), '.bundle/config'),
-      'BUNDLE_FLAGS' => "--jobs=4 --retry=3",
+      'BUNDLE_FLAGS' => "--jobs=4 --retry=3 --quiet",
       'BUNDLE_INSTALL_FLAGS' => nil,
       'BUNDLE_GEMFILE' => gemfile,
       'RUBYOPT' => nil,
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index e80d752f09f..d2cc16f4f8b 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -6,12 +6,17 @@ export BUNDLE_INSTALL_FLAGS="--without=production --jobs=$(nproc) --path=vendor
 
 if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
   bundle --version
-  bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check
+  run_timed_command "bundle install --clean ${BUNDLE_INSTALL_FLAGS}"
+  run_timed_command "bundle check"
+  # When we test multiple versions of PG in the same pipeline, we have a single `setup-test-env`
+  # job but the `pg` gem needs to be rebuilt since it includes extensions (https://guides.rubygems.org/gems-with-extensions).
+  # Uncomment the following line if multiple versions of PG are tested in the same pipeline.
+  # run_timed_command "bundle pristine pg"
 fi
 
 # Only install knapsack after bundle install! Otherwise oddly some native
 # gems could not be found under some circumstance. No idea why, hours wasted.
-retry gem install knapsack --no-document
+run_timed_command "gem install knapsack --no-document"
 
 cp config/gitlab.yml.example config/gitlab.yml
 sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml
@@ -33,6 +38,9 @@ if [ -f config/database_geo.yml ]; then
   sed -i 's/username: git/username: postgres/g' config/database_geo.yml
 fi
 
+cp config/cable.yml.example config/cable.yml
+sed -i 's|url:.*$|url: redis://redis:6379|g' config/cable.yml
+
 cp config/resque.yml.example config/resque.yml
 sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml
 
diff --git a/scripts/prepare_postgres_fdw.sh b/scripts/prepare_postgres_fdw.sh
index 69442d2881b..246f3acc569 100755
--- a/scripts/prepare_postgres_fdw.sh
+++ b/scripts/prepare_postgres_fdw.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 psql -h postgres -U postgres gitlabhq_geo_test <<EOF
 CREATE EXTENSION postgres_fdw;
diff --git a/scripts/regenerate-schema b/scripts/regenerate-schema
new file mode 100755
index 00000000000..cedd612f766
--- /dev/null
+++ b/scripts/regenerate-schema
@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'open3'
+require 'uri'
+
+class SchemaRegenerator
+  ##
+  # Filename of the schema
+  #
+  # This file is being regenerated by this script.
+  FILENAME = 'db/structure.sql'
+
+  ##
+  # Directories where migrations are stored
+  #
+  # The methods +hide_migrations+ and +unhide_migrations+ will rename
+  # these to disable/enable migrations.
+  MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
+
+  def execute
+    Dir.chdir(File.expand_path('..', __dir__)) do
+      checkout_ref
+      checkout_clean_schema
+      hide_migrations
+      reset_db
+      unhide_migrations
+      migrate
+    ensure
+      unhide_migrations
+    end
+  end
+
+  private
+
+  ##
+  # Git checkout +CI_COMMIT_SHA+.
+  #
+  # When running from CI, checkout the clean commit,
+  # not the merged result.
+  def checkout_ref
+    return unless ci?
+
+    run %Q[git checkout #{source_ref}]
+    run %q[git clean -f -- db]
+  end
+
+  ##
+  # Checkout the clean schema from the target branch
+  def checkout_clean_schema
+    remote_checkout_clean_schema || local_checkout_clean_schema
+  end
+
+  ##
+  # Get clean schema from remote servers
+  #
+  # This script might run in CI, using a shallow clone, so to checkout
+  # the file, fetch the target branch from the server.
+  def remote_checkout_clean_schema
+    return false unless project_url
+    return false unless target_project_url
+
+    run %Q[git remote add target_project #{target_project_url}.git]
+    run %Q[git fetch target_project #{target_branch}:#{target_branch}]
+
+    local_checkout_clean_schema
+  end
+
+  ##
+  # Git checkout the schema from target branch.
+  #
+  # Ask git to checkout the schema from the target branch and reset
+  # the file to unstage the changes.
+  def local_checkout_clean_schema
+    run %Q[git checkout #{merge_base} -- #{FILENAME}]
+    run %Q[git reset -- #{FILENAME}]
+  end
+
+  ##
+  # Move migrations to where Rails will not find them.
+  #
+  # To reset the database to clean schema defined in +FILENAME+, move
+  # the migrations to a path where Rails will not find them, otherwise
+  # +db:reset+ would abort. Later when the migrations should be
+  # applied, use +unhide_migrations+ to bring them back.
+  def hide_migrations
+    MIGRATION_DIRS.each do |dir|
+      File.rename(dir, "#{dir}__")
+    end
+  end
+
+  ##
+  # Undo the effect of +hide_migrations+.
+  #
+  # Place back the migrations which might be moved by
+  # +hide_migrations+.
+  def unhide_migrations
+    error = nil
+
+    MIGRATION_DIRS.each do |dir|
+      File.rename("#{dir}__", dir)
+    rescue Errno::ENOENT
+      nil
+    rescue StandardError => e
+      # Save error for later, but continue with other dirs first
+      error = e
+    end
+
+    raise error if error
+  end
+
+  ##
+  # Run rake task to reset the database.
+  def reset_db
+    run %q[bin/rails db:reset RAILS_ENV=test]
+  end
+
+  ##
+  # Run rake task to run migrations.
+  def migrate
+    run %q[bin/rails db:migrate RAILS_ENV=test]
+  end
+
+  ##
+  # Run the given +cmd+.
+  #
+  # The command is colored green, and the output of the command is
+  # colored gray.
+  # When the command failed an exception is raised.
+  def run(cmd)
+    puts "\e[32m$ #{cmd}\e[37m"
+    stdout_str, stderr_str, status = Open3.capture3(cmd)
+    puts "#{stdout_str}#{stderr_str}\e[0m"
+    raise("Command failed: #{stderr_str}") unless status.success?
+
+    stdout_str
+  end
+
+  ##
+  # Return the base commit between source and target branch.
+  def merge_base
+    @merge_base ||= run("git merge-base #{target_branch} #{source_ref}").chomp
+  end
+
+  ##
+  # Return the name of the target branch
+  #
+  # Get source ref from CI environment variable, or read the +TARGET+
+  # environment+ variable, or default to +HEAD+.
+  def target_branch
+    ENV['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'] || ENV['TARGET'] || 'master'
+  end
+
+  ##
+  # Return the source ref
+  #
+  # Get source ref from CI environment variable, or default to +HEAD+.
+  def source_ref
+    ENV['CI_COMMIT_SHA'] || 'HEAD'
+  end
+
+  ##
+  # Return the source project URL from CI environment variable.
+  def project_url
+    ENV['CI_PROJECT_URL']
+  end
+
+  ##
+  # Return the target project URL from CI environment variable.
+  def target_project_url
+    ENV['CI_MERGE_REQUEST_PROJECT_URL']
+  end
+
+  ##
+  # Return whether the script is running from CI
+  def ci?
+    ENV['CI']
+  end
+end
+
+SchemaRegenerator.new.execute
diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb
index e3ed7143ea2..a9659071a2f 100755
--- a/scripts/review_apps/automated_cleanup.rb
+++ b/scripts/review_apps/automated_cleanup.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'gitlab'
-require_relative File.expand_path('../../lib/quality/helm_client.rb', __dir__)
 require_relative File.expand_path('../../lib/quality/helm3_client.rb', __dir__)
 require_relative File.expand_path('../../lib/quality/kubernetes_client.rb', __dir__)
 
@@ -9,7 +8,6 @@ class AutomatedCleanup
   attr_reader :project_path, :gitlab_token
 
   DEPLOYMENTS_PER_PAGE = 100
-  HELM_RELEASES_BATCH_SIZE = 5
   IGNORED_HELM_ERRORS = [
     'transport is closing',
     'error upgrading connection',
@@ -45,18 +43,8 @@ class AutomatedCleanup
     self.class.ee? ? 'review-apps-ee' : 'review-apps-ce'
   end
 
-  def helm3?
-    !ENV['HELM_3'].nil?
-  end
-
-  def helm_client_class
-    helm3? ? Quality::Helm3Client : Quality::HelmClient
-  end
-
   def helm
-    @helm ||= helm_client_class.new(
-      tiller_namespace: review_apps_namespace,
-      namespace: review_apps_namespace)
+    @helm ||= Quality::Helm3Client.new(namespace: review_apps_namespace)
   end
 
   def kubernetes
@@ -88,7 +76,7 @@ class AutomatedCleanup
       if deployed_at < delete_threshold
         deleted_environment = delete_environment(environment, deployment)
         if deleted_environment
-          release = helm_client_class::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
+          release = Quality::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
           releases_to_delete << release
         end
       else
@@ -117,7 +105,7 @@ class AutomatedCleanup
       # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases
       next unless release.name.start_with?('review-')
 
-      if release.status.casecmp('failed') == 0 || release.last_update < threshold
+      if release.status == 'failed' || release.last_update < threshold
         releases_to_delete << release
       else
         print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving')
@@ -154,7 +142,6 @@ class AutomatedCleanup
 
   def helm_releases
     args = ['--all', '--date']
-    args << "--max #{HELM_RELEASES_BATCH_SIZE}" unless helm3?
 
     helm.releases(args: args)
   end
@@ -170,7 +157,7 @@ class AutomatedCleanup
     helm.delete(release_name: releases_names)
     kubernetes.cleanup(release_name: releases_names, wait: false)
 
-  rescue helm_client_class::CommandFailedError => ex
+  rescue Quality::Helm3Client::CommandFailedError => ex
     raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS)
 
     puts "Ignoring the following Helm error:\n#{ex}\n"
diff --git a/scripts/review_apps/gcp_cleanup.sh b/scripts/review_apps/gcp_cleanup.sh
index ea6b60ed5ff..f289a50f629 100755
--- a/scripts/review_apps/gcp_cleanup.sh
+++ b/scripts/review_apps/gcp_cleanup.sh
@@ -1,7 +1,14 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 source scripts/utils.sh
 
+function setup_gcp_dependencies() {
+  apk add jq
+
+  gcloud auth activate-service-account --key-file="${REVIEW_APPS_GCP_CREDENTIALS}"
+  gcloud config set project "${REVIEW_APPS_GCP_PROJECT}"
+}
+
 # These scripts require the following environment variables:
 # - REVIEW_APPS_GCP_REGION - e.g `us-central1`
 # - KUBE_NAMESPACE - e.g `review-apps-ee`
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 915b4f5050b..097fe9c8cca 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -95,37 +95,6 @@ function delete_failed_release() {
   fi
 }
 
-function helm2_deploy_exists() {
-  local namespace="${1}"
-  local release="${2}"
-  local deploy_exists
-
-  echoinfo "Checking if Helm 2 ${release} exists in the ${namespace} namespace..." true
-
-  kubectl get cm -l OWNER=TILLER -n ${namespace} | grep ${release} 2>&1
-  deploy_exists=$?
-
-  echoinfo "Helm 2 release for ${release} is ${deploy_exists}"
-  return $deploy_exists
-}
-
-function delete_helm2_release() {
-  local namespace="${KUBE_NAMESPACE}"
-  local release="${CI_ENVIRONMENT_SLUG}"
-
-  if [ -z "${release}" ]; then
-    echoerr "No release given, aborting the delete!"
-    return
-  fi
-
-  if ! helm2_deploy_exists "${namespace}" "${release}"; then
-    echoinfo "No Review App with ${release} is currently deployed by Helm 2."
-  else
-    echoinfo "Cleaning up ${release} installed by Helm 2"
-    kubectl_cleanup_release "${namespace}" "${release}"
-  fi
-}
-
 function get_pod() {
   local namespace="${KUBE_NAMESPACE}"
   local release="${CI_ENVIRONMENT_SLUG}"
@@ -267,7 +236,6 @@ function base_config_changed() {
 function deploy() {
   local namespace="${KUBE_NAMESPACE}"
   local release="${CI_ENVIRONMENT_SLUG}"
-  local edition="${GITLAB_EDITION-ce}"
   local base_config_file_ref="master"
   if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi
   local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml"
@@ -275,13 +243,13 @@ function deploy() {
   echoinfo "Deploying ${release}..." true
 
   IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
-  gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${edition}"
-  gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${edition}"
-  gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-webservice-${edition}"
-  gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${edition}"
+  gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-ee"
+  gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-ee"
+  gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-webservice-ee"
+  gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-ee"
   gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
   gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
-  gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${edition}"
+  gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-ee"
 
   create_application_secret
 
@@ -290,7 +258,7 @@ HELM_CMD=$(cat << EOF
     --namespace="${namespace}" \
     --install \
     --wait \
-    --timeout 900s \
+    --timeout 15m \
     --set ci.branch="${CI_COMMIT_REF_NAME}" \
     --set ci.commit.sha="${CI_COMMIT_SHORT_SHA}" \
     --set ci.job.url="${CI_JOB_URL}" \
@@ -304,8 +272,10 @@ HELM_CMD=$(cat << EOF
     --set gitlab.gitaly.image.tag="v${GITALY_VERSION}" \
     --set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \
     --set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \
+    --set gitlab.sidekiq.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
     --set gitlab.sidekiq.image.repository="${gitlab_sidekiq_image_repository}" \
     --set gitlab.sidekiq.image.tag="${CI_COMMIT_REF_SLUG}" \
+    --set gitlab.unicorn.annotations.commit="${CI_COMMIT_SHORT_SHA}" \
     --set gitlab.unicorn.image.repository="${gitlab_unicorn_image_repository}" \
     --set gitlab.unicorn.image.tag="${CI_COMMIT_REF_SLUG}" \
     --set gitlab.unicorn.workhorse.image="${gitlab_workhorse_image_repository}" \
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 70ddb61e588..0c9d3505ff3 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 function retrieve_tests_metadata() {
   mkdir -p knapsack/ rspec_flaky/ rspec_profiling/
@@ -15,9 +15,13 @@ function retrieve_tests_metadata() {
 function update_tests_metadata() {
   echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
 
-  scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*_pg9_*.json
+  scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json
   if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
-    scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+    else
+      echo "Not uplaoding report to S3 as the pipeline is not a scheduled one."
+    fi
   fi
 
   rm -f knapsack/rspec*.json
@@ -28,12 +32,20 @@ function update_tests_metadata() {
   scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
 
   if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then
-    scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+      scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
+    else
+      echo "Not uploading report to S3 as the pipeline is not a scheduled one."
+    fi
   fi
 
   rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
 
-  scripts/insert-rspec-profiling-data
+  if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
+    scripts/insert-rspec-profiling-data
+  else
+    echo "Not inserting profiling data as the pipeline is not a scheduled one."
+  fi
 }
 
 function rspec_simple_job() {
@@ -41,16 +53,14 @@ function rspec_simple_job() {
 
   export NO_KNAPSACK="1"
 
-  scripts/gitaly-test-spawn
-
   bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
 }
 
 function rspec_paralellized_job() {
-  read -ra job_name <<< "$CI_JOB_NAME"
+  read -ra job_name <<< "${CI_JOB_NAME}"
   local test_tool="${job_name[0]}"
   local test_level="${job_name[1]}"
-  local database="${job_name[2]}"
+  local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg11 1/24' would become 'rspec_unit_pg11_1_24'
   local rspec_opts="${1}"
   local spec_folder_prefix=""
 
@@ -59,7 +69,13 @@ function rspec_paralellized_job() {
   fi
 
   export KNAPSACK_LOG_LEVEL="debug"
-  export KNAPSACK_REPORT_PATH="knapsack/${test_tool}_${test_level}_${database}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+  export KNAPSACK_REPORT_PATH="knapsack/${report_name}_report.json"
+
+  # There's a bug where artifacts are sometimes not downloaded. Since specs can run without the Knapsack report, we can
+  # handle the missing artifact gracefully here. See https://gitlab.com/gitlab-org/gitlab/-/issues/212349.
+  if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
+    echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
+  fi
 
   cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
 
@@ -74,8 +90,8 @@ function rspec_paralellized_job() {
     export KNAPSACK_GENERATE_REPORT="true"
     export FLAKY_RSPEC_GENERATE_REPORT="true"
     export SUITE_FLAKY_RSPEC_REPORT_PATH="${FLAKY_RSPEC_SUITE_REPORT_PATH}"
-    export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${test_tool}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
-    export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${test_tool}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json"
+    export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${report_name}_report.json"
+    export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${report_name}_report.json"
 
     if [[ ! -f $FLAKY_RSPEC_REPORT_PATH ]]; then
       echo "{}" > "${FLAKY_RSPEC_REPORT_PATH}"
@@ -86,11 +102,9 @@ function rspec_paralellized_job() {
     fi
   fi
 
-  scripts/gitaly-test-spawn
-
   mkdir -p tmp/memory_test
 
-  export MEMORY_TEST_PATH="tmp/memory_test/${test_tool}_${test_level}_${database}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
+  export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv"
 
   knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
 
diff --git a/scripts/schema_changed.sh b/scripts/schema_changed.sh
index e8c120e92e1..427e0128df7 100755
--- a/scripts/schema_changed.sh
+++ b/scripts/schema_changed.sh
@@ -2,13 +2,13 @@
 
 schema_changed() {
   if [ ! -z "$(git diff --name-only -- db/structure.sql)" ]; then
-    printf "db/structure.sql after rake db:migrate:reset is different from one in the repository"
+    printf "Schema changes are not cleanly committed to db/structure.sql\n"
     printf "The diff is as follows:\n"
     diff=$(git diff -p --binary -- db/structure.sql)
     printf "%s" "$diff"
     exit 1
   else
-    printf "db/structure.sql after rake db:migrate:reset matches one in the repository"
+    printf "Schema changes are correctly applied to db/structure.sql\n"
   fi
 }
 
diff --git a/scripts/security-harness b/scripts/security-harness
index c101cd03454..b9492e16066 100755
--- a/scripts/security-harness
+++ b/scripts/security-harness
@@ -19,7 +19,7 @@ end
 
 HOOK_PATH = File.expand_path("../.git/hooks/pre-push", __dir__)
 HOOK_DATA = <<~HOOK
-  #!/bin/bash
+  #!/usr/bin/env bash
 
   set -e
 
diff --git a/scripts/utils.sh b/scripts/utils.sh
index d1f98bb3f62..897f8d5a8b8 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -18,11 +18,8 @@ function setup_db_user_only() {
 }
 
 function setup_db() {
-    setup_db_user_only
-
-    bundle exec rake db:drop db:create db:structure:load db:migrate
-
-    bundle exec rake gitlab:db:setup_ee
+    run_timed_command "setup_db_user_only"
+    run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate gitlab:db:setup_ee"
 }
 
 function install_api_client_dependencies_with_apk() {
@@ -38,6 +35,24 @@ function install_gitlab_gem() {
   gem install gitlab --no-document --version 4.13.0
 }
 
+function run_timed_command() {
+  local cmd="${1}"
+  local start=$(date +%s)
+  echosuccess "\$ ${cmd}"
+  eval "${cmd}"
+  local ret=$?
+  local end=$(date +%s)
+  local runtime=$((end-start))
+
+  if [[ $ret -eq 0 ]]; then
+    echosuccess "==> '${cmd}' succeeded in ${runtime} seconds."
+    return 0
+  else
+    echoerr "==> '${cmd}' failed (${ret}) in ${runtime} seconds."
+    return $ret
+  fi
+}
+
 function echoerr() {
   local header="${2}"
 
@@ -58,6 +73,16 @@ function echoinfo() {
   fi
 }
 
+function echosuccess() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n\033[0;32m** %s **\n\033[0m" "${1}" >&2;
+  else
+    printf "\033[0;32m%s\n\033[0m" "${1}" >&2;
+  fi
+}
+
 function get_job_id() {
   local job_name="${1}"
   local query_string="${2:+&${2}}"