Merge branch 'master' into 'id-test-codeowners'id-test-codeowners

# Conflicts:
#   .gitlab/CODEOWNERS

8 files changed, 231 insertions, 72 deletions

diff --git a/scripts/create_mysql_user.sh b/scripts/create_mysql_user.sh
deleted file mode 100644
index 35f68c581f3..00000000000
--- a/scripts/create_mysql_user.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-mysql --user=root --host=mysql <<EOF
-CREATE USER IF NOT EXISTS 'gitlab'@'%';
-GRANT ALL PRIVILEGES ON gitlabhq_test.* TO 'gitlab'@'%';
-FLUSH PRIVILEGES;
-EOF
diff --git a/scripts/insert-rspec-profiling-data b/scripts/insert-rspec-profiling-data
index b34379764e0..88c9d8c12b1 100755
--- a/scripts/insert-rspec-profiling-data
+++ b/scripts/insert-rspec-profiling-data
@@ -14,10 +14,6 @@ module RspecProfiling
         Result.establish_connection(results_url)
       end
 
-      def prepared?
-        connection.data_source_exists?(table)
-      end
-
       def results_url
         ENV['RSPEC_PROFILING_POSTGRES_URL']
       end
diff --git a/scripts/lint-changelog-yaml b/scripts/lint-changelog-yaml
deleted file mode 100755
index 06d502c4676..00000000000
--- a/scripts/lint-changelog-yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'yaml'
-
-invalid_changelogs = Dir['changelogs/**/*'].reject do |changelog|
-  next true if changelog =~ /((README|archive)\.md|unreleased(-ee)?)$/
-  next false unless changelog.end_with?('.yml')
-
-  begin
-    YAML.load_file(changelog)
-  rescue => exception
-    puts exception
-  end
-end
-
-if invalid_changelogs.any?
-  puts
-  puts "Invalid changelogs found!\n"
-  puts invalid_changelogs.sort
-  exit 1
-else
-  puts "All changelogs are valid YAML.\n"
-  exit 0
-end
diff --git a/scripts/lint-rugged b/scripts/lint-rugged
index 9466c62a415..d862571c1c5 100755
--- a/scripts/lint-rugged
+++ b/scripts/lint-rugged
@@ -1,6 +1,9 @@
 #!/usr/bin/env ruby
 
 ALLOWED = [
+  # https://gitlab.com/gitlab-org/gitaly/issues/760
+  'lib/elasticsearch/git/repository.rb',
+
   # Needed to handle repositories that are not in any storage
   'lib/gitlab/bare_repository_import/repository.rb',
 
@@ -10,7 +13,14 @@ ALLOWED = [
   # Reverted Rugged calls due to Gitaly atop NFS performance
   # See https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code.
   'lib/gitlab/git/rugged_impl/',
-  'lib/gitlab/gitaly_client/storage_settings.rb'
+  'lib/gitlab/gitaly_client/storage_settings.rb',
+
+  # Needed for logging
+  'config/initializers/peek.rb',
+  'config/initializers/lograge.rb',
+  'lib/gitlab/grape_logging/loggers/perf_logger.rb',
+  'lib/gitlab/rugged_instrumentation.rb',
+  'lib/peek/views/rugged.rb'
 ].freeze
 
 rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines
diff --git a/scripts/merge-html-reports b/scripts/merge-html-reports
new file mode 100755
index 00000000000..7d1e15186c8
--- /dev/null
+++ b/scripts/merge-html-reports
@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+
+main_report_file = ARGV.shift
+unless main_report_file
+  puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+  exit 1
+end
+
+base_artifact_url = ARGV.shift
+unless base_artifact_url
+  puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+  exit 1
+end
+
+# Create the base report with empty body tag
+new_report = Nokogiri::HTML.parse(File.read(ARGV[0]))
+new_report.at_css('body').remove
+empty_body = Nokogiri::XML::Node.new('body', new_report)
+new_report.at_css('head').add_next_sibling(empty_body)
+
+ARGV.each do |report_file|
+  report = Nokogiri::HTML.parse(File.read(report_file))
+
+  report.css('a').each do |link|
+    link_suffix = link['href'].slice(19..-1)
+    link['href'] = base_artifact_url + link_suffix
+  end
+
+  header = report.css('div #rspec-header')
+  tests = report.css('dt[id^="example_group_"]')
+
+  tests.each do |test|
+    title = test.parent
+    group = title.parent
+    script = title.css('script')
+
+    if script.inner_html.include? 'makeYellow'
+      test.remove_class('passed')
+      test.add_class('not_implemented')
+
+      group.remove_class('passed')
+      group.add_class('not_implemented')
+      header.add_class('not_implemented')
+
+      script.remove
+      test.next_sibling.remove
+      test.next_sibling.remove
+
+    elsif script.inner_html.include? 'makeRed'
+      test.remove_class('passed')
+      test.add_class('failed')
+
+      group.remove_class('passed')
+      group.add_class('failed')
+      header.add_class('failed')
+
+      script.remove
+      test.next_sibling.remove
+      test.next_sibling.remove
+    end
+  end
+
+  duration = report.at_css('p#duration')
+  totals = report.at_css('p#totals')
+
+  duration_script = report.css('div.results script')[-2]
+  totals_script = report.css('div.results script')[-1]
+
+  duration_text = duration_script.text.slice(49..-3)
+  totals_text = totals_script.text.slice(47..-3)
+
+  duration.inner_html = duration_text
+  totals.inner_html = totals_text
+
+  duration_script.remove
+  totals_script.remove
+
+  # Add the new result after the last one to keep the test order
+  new_report.css('body')[-1].add_next_sibling(report.at_css('body'))
+end
+
+File.write(main_report_file, new_report)
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index bc47884ee45..4935c1342a3 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -52,7 +52,7 @@ function delete() {
 function get_pod() {
   local app_name="${1}"
   local status="${2-Running}"
-  get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name"
+  get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
   echoinfo "Waiting till '${app_name}' pod is ready" true
   echoinfo "Running '${get_pod_cmd}'"
 
@@ -69,7 +69,6 @@ function get_pod() {
       break
     fi
 
-    printf "."
     let "elapsed_seconds+=interval"
     sleep ${interval}
   done
@@ -190,18 +189,12 @@ function deploy() {
   gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
   gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
 
-  # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
-  if [ "$CI_ENVIRONMENT_SLUG" != "production" ] && previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
-    echo "Deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
-    delete
-  fi
-
   create_application_secret
 
 HELM_CMD=$(cat << EOF
   helm upgrade --install \
-    --wait \
-    --timeout 600 \
+    --atomic \
+    --timeout 900 \
     --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
     --set global.appConfig.enableUsagePing=false \
     --set global.imagePullPolicy=Always \
@@ -347,32 +340,6 @@ function display_deployment_debug() {
   fi
 }
 
-function wait_for_review_app_to_be_accessible() {
-  echoinfo "Waiting for the Review App at ${CI_ENVIRONMENT_URL} to be accessible..." true
-
-  local interval=5
-  local elapsed_seconds=0
-  local max_seconds=$((2 * 60))
-  while true; do
-    local review_app_http_code
-    review_app_http_code=$(curl --silent --output /dev/null --max-time 5 --write-out "%{http_code}" "${CI_ENVIRONMENT_URL}/users/sign_in")
-    if [[ "${review_app_http_code}" -eq "200" ]] || [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
-      break
-    fi
-
-    printf "."
-    let "elapsed_seconds+=interval"
-    sleep ${interval}
-  done
-
-  if [[ "${review_app_http_code}" -eq "200" ]]; then
-    echoinfo "The Review App at ${CI_ENVIRONMENT_URL} is ready after ${elapsed_seconds} seconds!"
-  else
-    echoerr "The Review App at ${CI_ENVIRONMENT_URL} isn't ready after ${max_seconds} seconds of polling..."
-    exit 1
-  fi
-}
-
 function add_license() {
   if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
 
diff --git a/scripts/static-analysis b/scripts/static-analysis
index 642c50ec0a8..6fd64fbf9da 100755
--- a/scripts/static-analysis
+++ b/scripts/static-analysis
@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 # We don't have auto-loading here
+require_relative '../lib/gitlab'
 require_relative '../lib/gitlab/popen'
 require_relative '../lib/gitlab/popen/runner'
 
@@ -36,6 +37,10 @@ tasks = [
   %w[scripts/lint-rugged]
 ]
 
+if Gitlab.ee?
+  tasks.unshift(%w[ruby -rbundler/setup scripts/ee_specific_check/ee_specific_check.rb])
+end
+
 static_analysis = Gitlab::Popen::Runner.new
 
 static_analysis.run(tasks) do |cmd, &run|
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 4a6567b8a62..c1bdef5b847 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -29,6 +29,134 @@ function setup_db() {
     if [ "$GITLAB_DATABASE" = "mysql" ]; then
         bundle exec rake add_limits_mysql
     fi
+
+    bundle exec rake gitlab:db:setup_ee
+}
+
+function install_api_client_dependencies_with_apk() {
+  apk add --update openssl curl jq
+}
+
+function install_api_client_dependencies_with_apt() {
+  apt update && apt install jq -y
+}
+
+function install_gitlab_gem() {
+  gem install gitlab --no-document
+}
+
+function echoerr() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2;
+  else
+    printf "\033[0;31m%s\n\033[0m" "${1}" >&2;
+  fi
+}
+
+function echoinfo() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2;
+  else
+    printf "\033[0;33m%s\n\033[0m" "${1}" >&2;
+  fi
+}
+
+function get_job_id() {
+  local job_name="${1}"
+  local query_string="${2:+&${2}}"
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  local max_page=3
+  local page=1
+
+  while true; do
+    local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
+    echoinfo "GET ${url}"
+
+    local job_id
+    job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last")
+    [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break
+
+    let "page++"
+  done
+
+  if [[ "${job_id}" == "" ]]; then
+    echoerr "The '${job_name}' job ID couldn't be retrieved!"
+  else
+    echoinfo "The '${job_name}' job ID is ${job_id}"
+    echo "${job_id}"
+  fi
+}
+
+function play_job() {
+  local job_name="${1}"
+  local job_id
+  job_id=$(get_job_id "${job_name}" "scope=manual");
+  if [ -z "${job_id}" ]; then return; fi
+
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play"
+  echoinfo "POST ${url}"
+
+  local job_url
+  job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url")
+  echoinfo "Manual job '${job_name}' started at: ${job_url}"
+}
+
+function wait_for_job_to_be_done() {
+  local job_name="${1}"
+  local query_string="${2}"
+  local job_id
+  job_id=$(get_job_id "${job_name}" "${query_string}")
+  if [ -z "${job_id}" ]; then return; fi
+
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  echoinfo "Waiting for the '${job_name}' job to finish..."
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}"
+  echoinfo "GET ${url}"
+
+  # In case the job hasn't finished yet. Keep trying until the job times out.
+  local interval=30
+  local elapsed_seconds=0
+  while true; do
+    local job_status
+    job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g)
+    [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break
+
+    printf "."
+    let "elapsed_seconds+=interval"
+    sleep ${interval}
+  done
+
+  local elapsed_minutes=$((elapsed_seconds / 60))
+  echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes."
+
+  if [[ "${job_status}" == "failed" ]]; then
+    echoerr "The '${job_name}' failed."
+  elif [[ "${job_status}" == "manual" ]]; then
+    echoinfo "The '${job_name}' is manual."
+  else
+    echoinfo "The '${job_name}' passed."
+  fi
 }
 
 function install_api_client_dependencies_with_apk() {