Add latest changes from gitlab-org/gitlab@13-3-stable-ee

8 files changed, 198 insertions, 175 deletions

diff --git a/scripts/prepare_postgres_fdw.sh b/scripts/prepare_postgres_fdw.sh
deleted file mode 100755
index 246f3acc569..00000000000
--- a/scripts/prepare_postgres_fdw.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-psql -h postgres -U postgres gitlabhq_geo_test <<EOF
-CREATE EXTENSION postgres_fdw;
-CREATE SERVER gitlab_secondary FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'localhost', dbname 'gitlabhq_test');
-CREATE USER MAPPING FOR current_user SERVER gitlab_secondary OPTIONS (user 'postgres', password '');
-CREATE SCHEMA gitlab_secondary;
-IMPORT FOREIGN SCHEMA public FROM SERVER gitlab_secondary INTO gitlab_secondary;
-GRANT USAGE ON FOREIGN SERVER gitlab_secondary TO current_user;
-EOF
-
-# Ensure the FDW setting is enabled
-sed -i '/fdw:/d' config/database_geo.yml
-sed -i '/gitlabhq_geo_test/a\
-\ \ fdw: true' config/database_geo.yml
diff --git a/scripts/regenerate-schema b/scripts/regenerate-schema
index cedd612f766..e2d46d14218 100755
--- a/scripts/regenerate-schema
+++ b/scripts/regenerate-schema
@@ -3,6 +3,7 @@
 # frozen_string_literal: true
 
 require 'open3'
+require 'fileutils'
 require 'uri'
 
 class SchemaRegenerator
@@ -19,11 +20,19 @@ class SchemaRegenerator
   # these to disable/enable migrations.
   MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
 
+  ##
+  # Directory where we store schema versions
+  #
+  # The remove_schema_migration_files removes files added in this
+  # directory when it runs.
+  SCHEMA_MIGRATIONS_DIR = 'db/schema_migrations/'
+
   def execute
     Dir.chdir(File.expand_path('..', __dir__)) do
       checkout_ref
       checkout_clean_schema
       hide_migrations
+      remove_schema_migration_files
       reset_db
       unhide_migrations
       migrate
@@ -111,6 +120,35 @@ class SchemaRegenerator
   end
 
   ##
+  # Remove files added to db/schema_migrations
+  #
+  # In order to properly reset the database and re-run migrations
+  # the schema migrations for new migrations must be removed.
+  def remove_schema_migration_files
+    (untracked_schema_migrations + commited_schema_migrations).each do |schema_migration|
+      FileUtils.rm(schema_migration)
+    end
+  end
+
+  ##
+  # List of untracked schema migrations
+  #
+  # Get a list of schema migrations that are not tracked so we can remove them
+  def untracked_schema_migrations
+    git_command = "git ls-files --others --exclude-standard -- #{SCHEMA_MIGRATIONS_DIR}"
+    run(git_command).chomp.split("\n")
+  end
+
+  ##
+  # List of untracked schema migrations
+  #
+  # Get a list of schema migrations that have been committed since the last
+  def commited_schema_migrations
+    git_command = "git diff --name-only --diff-filter=A #{merge_base} -- #{SCHEMA_MIGRATIONS_DIR}"
+    run(git_command).chomp.split("\n")
+  end
+
+  ##
   # Run rake task to reset the database.
   def reset_db
     run %q[bin/rails db:reset RAILS_ENV=test]
diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh
index 74291f6aef4..59189c94cde 100755
--- a/scripts/review_apps/review-apps.sh
+++ b/scripts/review_apps/review-apps.sh
@@ -147,9 +147,9 @@ function disable_sign_ups() {
   run_task "${ruby_cmd}"
 
   # Disable sign-ups
-  curl  --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false"
+  retry 'curl --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false"'
 
-  local signup_enabled=$(curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled")
+  local signup_enabled=$(retry 'curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled"')
   if [[ "${signup_enabled}" == "false" ]]; then
     echoinfo "Sign-ups have been disabled successfully."
   else
diff --git a/scripts/schema_changed.sh b/scripts/schema_changed.sh
index 427e0128df7..f564f717e95 100755
--- a/scripts/schema_changed.sh
+++ b/scripts/schema_changed.sh
@@ -10,6 +10,16 @@ schema_changed() {
   else
     printf "Schema changes are correctly applied to db/structure.sql\n"
   fi
+
+  if [ ! -z "$(git add -A -n db/schema_migrations)" ]; then
+    printf "Schema version files have not been committed to the repository:\n"
+    printf "The following files should be committed:\n"
+    diff=$(git add -A -n db/schema_migrations)
+    printf "%s" "$diff"
+    exit 2
+  else
+    printf "Schema changes are correctly applied to db/structure.sql and db/schema_migrations/\n"
+  fi
 }
 
 schema_changed
diff --git a/scripts/slack b/scripts/slack
new file mode 100755
index 00000000000..60bc70a8542
--- /dev/null
+++ b/scripts/slack
@@ -0,0 +1,17 @@
+#!/bin/bash
+# This is copied from:
+# https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/bin/slack
+#
+# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set).
+# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it).
+
+CHANNEL=$1
+MSG=$2
+ICON_EMOJI=$3
+
+if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then
+    echo "Missing argument(s) - Use: $0 channel message icon_emoji"
+    echo "and set CI_SLACK_WEBHOOK_URL environment variable."
+else
+    curl -X POST --data-urlencode 'payload={"channel": "#'"$CHANNEL"'", "username": "GitLab QA Bot", "text": "'"$MSG"'", "icon_emoji": "'":$ICON_EMOJI:"'"}' "$CI_SLACK_WEBHOOK_URL"
+fi
diff --git a/scripts/sync-stable-branch.sh b/scripts/sync-stable-branch.sh
index 59ab52844fb..62538002278 100644
--- a/scripts/sync-stable-branch.sh
+++ b/scripts/sync-stable-branch.sh
@@ -51,7 +51,8 @@ then
   echo 'Commit not available, triggering a merge train'
 fi
 
-curl -X POST \
+curl -f \
+  -X POST \
   -F token="$MERGE_TRAIN_TRIGGER_TOKEN" \
   -F ref=master \
   -F "variables[MERGE_FOSS]=1" \
diff --git a/scripts/trigger-build b/scripts/trigger-build
index 9f0df21e7f1..7fc550d86ee 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@@ -29,10 +29,17 @@ module Trigger
   end
 
   class Base
-    def invoke!(post_comment: false, downstream_job_name: nil)
+    # Can be overridden
+    def self.access_token
+      ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
+    end
+
+    def initialize
       # gitlab-bot's token "GitLab multi-project pipeline polling"
-      Gitlab.private_token = access_token
+      Gitlab.private_token = self.class.access_token
+    end
 
+    def invoke!(post_comment: false, downstream_job_name: nil)
       pipeline_variables = variables
 
       puts "Triggering downstream pipeline on #{downstream_project_path}"
@@ -74,14 +81,9 @@ module Trigger
       raise NotImplementedError
     end
 
-    # Must be overridden
+    # Can be overridden
     def trigger_token
-      raise NotImplementedError
-    end
-
-    # Must be overridden
-    def access_token
-      raise NotImplementedError
+      ENV['CI_JOB_TOKEN']
     end
 
     # Can be overridden
@@ -133,14 +135,6 @@ module Trigger
       ENV['OMNIBUS_BRANCH'] || 'master'
     end
 
-    def trigger_token
-      ENV['CI_JOB_TOKEN']
-    end
-
-    def access_token
-      ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-    end
-
     def extra_variables
       # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
       # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
@@ -176,10 +170,6 @@ module Trigger
       ENV['BUILD_TRIGGER_TOKEN']
     end
 
-    def access_token
-      ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-    end
-
     def extra_variables
       edition = Trigger.ee? ? 'EE' : 'CE'
 
@@ -205,6 +195,112 @@ module Trigger
     end
   end
 
+  class Docs < Base
+    def self.access_token
+      ENV['DOCS_API_TOKEN']
+    end
+
+    SUCCESS_MESSAGE = <<~MSG
+    => You should now be able to preview your changes under the following URL:
+
+    %<app_url>s
+
+    => For more information, see the documentation
+    => https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live
+
+    => If something doesn't work, drop a line in the #docs chat channel.
+    MSG
+
+    # Create a remote branch in gitlab-docs and immediately cancel the pipeline
+    # to avoid race conditions, since a triggered pipeline will also run right
+    # after the branch creation. This only happens the very first time a branch
+    # is created and will be skipped in subsequent runs. Read more in
+    # https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
+    #
+    def deploy!
+      create_remote_branch!
+      cancel_latest_pipeline!
+      invoke!.wait!
+      display_success_message
+    end
+
+    #
+    # Remove a remote branch in gitlab-docs.
+    #
+    def cleanup!
+      Gitlab.delete_branch(downstream_project_path, ref)
+      puts "=> Remote branch '#{downstream_project_path}' deleted"
+    end
+
+    private
+
+    def downstream_project_path
+      ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs'
+    end
+
+    def ref
+      if ENV['CI_MERGE_REQUEST_IID'].nil?
+        "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}"
+      else
+        "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}"
+      end
+    end
+
+    def extra_variables
+      {
+        "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME']
+      }
+    end
+
+    def slug
+      case ENV['CI_PROJECT_PATH']
+      when 'gitlab-org/gitlab-foss'
+        'ce'
+      when 'gitlab-org/gitlab'
+        'ee'
+      when 'gitlab-org/gitlab-runner'
+        'runner'
+      when 'gitlab-org/omnibus-gitlab'
+        'omnibus'
+      when 'gitlab-org/charts/gitlab'
+        'charts'
+      end
+    end
+
+    def app_url
+      "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}"
+    end
+
+    def create_remote_branch!
+      Gitlab.create_branch(downstream_project_path, ref, 'master')
+      puts "=> Remote branch '#{ref}' created"
+    rescue Gitlab::Error::BadRequest
+      puts "=> Remote branch '#{ref}' already exists!"
+    end
+
+    def cancel_latest_pipeline!
+      pipelines = nil
+
+      # Wait until the pipeline is started
+      loop do
+        sleep 1
+        puts "=> Waiting for pipeline to start..."
+        pipelines = Gitlab.pipelines(downstream_project_path, { ref: ref })
+        break if pipelines.any?
+      end
+
+      # Get the first pipeline ID which should be the only one for the branch
+      pipeline_id = pipelines.first.id
+
+      # Cancel the pipeline
+      Gitlab.cancel_pipeline(downstream_project_path, pipeline_id)
+    end
+
+    def display_success_message
+      format(SUCCESS_MESSAGE, app_url: app_url)
+    end
+  end
+
   class CommitComment
     def self.post!(downstream_pipeline)
       Gitlab.create_commit_comment(
@@ -282,6 +378,18 @@ when 'omnibus'
   Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
 when 'cng'
   Trigger::CNG.new.invoke!.wait!
+when 'docs'
+  docs_trigger = Trigger::Docs.new
+
+  case ARGV[1]
+  when 'deploy'
+    docs_trigger.deploy!
+  when 'cleanup'
+    docs_trigger.cleanup!
+  else
+    puts 'usage: trigger-build docs <deploy|cleanup>'
+    exit 1
+  end
 else
   puts "Please provide a valid option:
   omnibus - Triggers a pipeline that builds the omnibus-gitlab package
diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs
deleted file mode 100755
index 2957dde6fc0..00000000000
--- a/scripts/trigger-build-docs
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'gitlab'
-
-#
-# Configure credentials to be used with gitlab gem
-#
-Gitlab.configure do |config|
-  config.endpoint      = 'https://gitlab.com/api/v4'
-  config.private_token = ENV["DOCS_API_TOKEN"] # GitLab Docs bot access token with Developer access to gitlab-docs
-end
-
-#
-# The remote docs project
-#
-GITLAB_DOCS_REPO = 'gitlab-org/gitlab-docs'.freeze
-
-#
-# This is the branch that will be created in the gitlab-docs project.
-# Name it after the product we're previewing and the ID of the MR that
-# kicked the review app.
-#
-def docs_branch
-  # Check if CI_MERGE_REQUEST_IID is present. This requires pipelines
-  # for merge requests to be enabled.
-  if ENV["CI_MERGE_REQUEST_IID"].nil?
-    "docs-preview-#{slug}-#{ENV["CI_COMMIT_REF_SLUG"]}"
-  else
-    "docs-preview-#{slug}-#{ENV["CI_MERGE_REQUEST_IID"]}"
-  end
-end
-
-#
-# Create a remote branch in gitlab-docs and immediately cancel the pipeline
-# to avoid race conditions, since a triggered pipeline will also run right
-# after the branch creation. This only happens the very first time a branch
-# is created and will be skipped in subsequent runs. Read more in
-# https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
-#
-def create_remote_branch
-  Gitlab.create_branch(GITLAB_DOCS_REPO, docs_branch, 'master')
-  puts "=> Remote branch '#{docs_branch}' created"
-
-  pipelines = nil
-
-  # Wait until the pipeline is started
-  loop do
-    sleep 1
-    puts "=> Waiting for pipeline to start..."
-    pipelines = Gitlab.pipelines(GITLAB_DOCS_REPO, { ref: docs_branch })
-    break if pipelines.any?
-  end
-
-  # Get the first pipeline ID which should be the only one for the branch
-  pipeline_id = pipelines.first.id
-
-  # Cancel the pipeline
-  Gitlab.cancel_pipeline(GITLAB_DOCS_REPO, pipeline_id)
-rescue Gitlab::Error::BadRequest
-  puts "=> Remote branch '#{docs_branch}' already exists"
-end
-
-#
-# Remove a remote branch in gitlab-docs
-#
-def remove_remote_branch
-  Gitlab.delete_branch(GITLAB_DOCS_REPO, docs_branch)
-  puts "=> Remote branch '#{docs_branch}' deleted"
-end
-
-#
-# Define suffix in review app URL based on project
-#
-def slug
-  case ENV["CI_PROJECT_PATH"]
-  when 'gitlab-org/gitlab-foss'
-    'ce'
-  when 'gitlab-org/gitlab'
-    'ee'
-  when 'gitlab-org/gitlab-runner'
-    'runner'
-  when 'gitlab-org/omnibus-gitlab'
-    'omnibus'
-  when 'gitlab-org/charts/gitlab'
-    'charts'
-  end
-end
-
-#
-# Overriding vars in https://gitlab.com/gitlab-org/gitlab-docs/blob/master/.gitlab-ci.yml
-#
-def param_name
-  "BRANCH_#{slug.upcase}"
-end
-
-#
-# Trigger a pipeline in gitlab-docs
-#
-def trigger_pipeline
-  # The review app URL
-  app_url = "http://#{docs_branch}.#{ENV["DOCS_REVIEW_APPS_DOMAIN"]}/#{slug}"
-
-  # Create the cross project pipeline using CI_JOB_TOKEN
-  pipeline = Gitlab.run_trigger(GITLAB_DOCS_REPO, ENV["CI_JOB_TOKEN"], docs_branch, { param_name => ENV["CI_COMMIT_REF_NAME"] })
-
-  puts "=> Follow the status of the triggered pipeline:"
-  puts ""
-  puts pipeline.web_url
-  puts ""
-  puts "=> In a few minutes, you will be able to preview your changes under the following URL:"
-  puts ""
-  puts app_url
-  puts ""
-  puts "=> For more information, see the documentation"
-  puts "=> https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live"
-  puts ""
-  puts "=> If something doesn't work, drop a line in the #docs chat channel."
-  puts ""
-end
-
-#
-# When the first argument is deploy then create the branch and trigger pipeline
-# When it is 'stop', it deleted the remote branch. That way, we ensure there
-# are no stale remote branches and the Review server doesn't fill.
-#
-case ARGV[0]
-when 'deploy'
-  create_remote_branch
-  trigger_pipeline
-when 'cleanup'
-  remove_remote_branch
-else
-  puts "Please provide a valid option:
-  deploy  - Creates the remote branch and triggers a pipeline
-  cleanup - Deletes the remote branch and stops the Review App"
-end