Merge branch 'test-pg' into 'master'

Test both PostgreSQL and MySQL for the win

Closes #2582

See merge request !10156

1 files changed, 150 insertions, 64 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f188ee29223..12385b293c0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1"
+image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
 
 cache:
   key: "ruby-233"
@@ -10,8 +10,6 @@ variables:
   RAILS_ENV: "test"
   NODE_ENV: "test"
   SIMPLECOV: "true"
-  SETUP_DB: "true"
-  USE_BUNDLE_INSTALL: "true"
   GIT_DEPTH: "20"
   PHANTOMJS_VERSION: "2.1.1"
   GET_SOURCES_ATTEMPTS: "3"
@@ -19,12 +17,9 @@ variables:
   KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
 
 before_script:
-  - source ./scripts/prepare_build.sh
-  - cp config/gitlab.yml.example config/gitlab.yml
   - bundle --version
-  - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
-  - retry gem install knapsack fog-aws mime-types
-  - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
+  - . scripts/utils.sh
+  - ./scripts/prepare_build.sh
 
 stages:
 - prepare
@@ -52,20 +47,34 @@ stages:
     paths:
       - knapsack/
 
-.use-db: &use-db
+.use-pg: &use-pg
+  services:
+    - postgres:latest
+    - redis:alpine
+
+.use-mysql: &use-mysql
   services:
     - mysql:latest
     - redis:alpine
 
+.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
+  only:
+    - /\-(?i)mysql$/
+    - master@gitlab-org/gitlab-ce
+    - master@gitlab/gitlabhq
+    - tags@gitlab-org/gitlab-ce
+    - tags@gitlab/gitlabhq
+    - //@gitlab-org/gitlab-ee
+    - //@gitlab/gitlab-ee
+
 .rspec-knapsack: &rspec-knapsack
   stage: test
   <<: *dedicated-runner
-  <<: *use-db
   script:
     - JOB_NAME=( $CI_JOB_NAME )
-    - export CI_NODE_INDEX=${JOB_NAME[1]}
-    - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export CI_NODE_INDEX=${JOB_NAME[-2]}
+    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
     - export CACHE_CLASSES=true
     - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
@@ -78,15 +87,23 @@ stages:
       - knapsack/
       - tmp/capybara/
 
+.rspec-knapsack-pg: &rspec-knapsack-pg
+  <<: *rspec-knapsack
+  <<: *use-pg
+
+.rspec-knapsack-mysql: &rspec-knapsack-mysql
+  <<: *rspec-knapsack
+  <<: *use-mysql
+  <<: *only-master-and-ee-or-mysql
+
 .spinach-knapsack: &spinach-knapsack
   stage: test
   <<: *dedicated-runner
-  <<: *use-db
   script:
     - JOB_NAME=( $CI_JOB_NAME )
-    - export CI_NODE_INDEX=${JOB_NAME[1]}
-    - export CI_NODE_TOTAL=${JOB_NAME[2]}
-    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export CI_NODE_INDEX=${JOB_NAME[-2]}
+    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
     - export CACHE_CLASSES=true
     - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
@@ -99,6 +116,15 @@ stages:
       - knapsack/
       - tmp/capybara/
 
+.spinach-knapsack-pg: &spinach-knapsack-pg
+  <<: *spinach-knapsack
+  <<: *use-pg
+
+.spinach-knapsack-mysql: &spinach-knapsack-mysql
+  <<: *spinach-knapsack
+  <<: *use-mysql
+  <<: *only-master-and-ee-or-mysql
+
 # Prepare and merge knapsack tests
 knapsack:
   <<: *knapsack-state
@@ -116,8 +142,8 @@ update-knapsack:
   <<: *dedicated-runner
   stage: post-test
   script:
-    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json
-    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json
+    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
+    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
     - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
   only:
@@ -127,7 +153,7 @@ update-knapsack:
     - master@gitlab/gitlab-ee
 
 setup-test-env:
-  <<: *use-db
+  <<: *use-pg
   <<: *dedicated-runner
   stage: prepare
   script:
@@ -142,37 +168,69 @@ setup-test-env:
       - public/assets
       - tmp/tests
 
-rspec 0 20: *rspec-knapsack
-rspec 1 20: *rspec-knapsack
-rspec 2 20: *rspec-knapsack
-rspec 3 20: *rspec-knapsack
-rspec 4 20: *rspec-knapsack
-rspec 5 20: *rspec-knapsack
-rspec 6 20: *rspec-knapsack
-rspec 7 20: *rspec-knapsack
-rspec 8 20: *rspec-knapsack
-rspec 9 20: *rspec-knapsack
-rspec 10 20: *rspec-knapsack
-rspec 11 20: *rspec-knapsack
-rspec 12 20: *rspec-knapsack
-rspec 13 20: *rspec-knapsack
-rspec 14 20: *rspec-knapsack
-rspec 15 20: *rspec-knapsack
-rspec 16 20: *rspec-knapsack
-rspec 17 20: *rspec-knapsack
-rspec 18 20: *rspec-knapsack
-rspec 19 20: *rspec-knapsack
-
-spinach 0 10: *spinach-knapsack
-spinach 1 10: *spinach-knapsack
-spinach 2 10: *spinach-knapsack
-spinach 3 10: *spinach-knapsack
-spinach 4 10: *spinach-knapsack
-spinach 5 10: *spinach-knapsack
-spinach 6 10: *spinach-knapsack
-spinach 7 10: *spinach-knapsack
-spinach 8 10: *spinach-knapsack
-spinach 9 10: *spinach-knapsack
+rspec pg 0 20: *rspec-knapsack-pg
+rspec pg 1 20: *rspec-knapsack-pg
+rspec pg 2 20: *rspec-knapsack-pg
+rspec pg 3 20: *rspec-knapsack-pg
+rspec pg 4 20: *rspec-knapsack-pg
+rspec pg 5 20: *rspec-knapsack-pg
+rspec pg 6 20: *rspec-knapsack-pg
+rspec pg 7 20: *rspec-knapsack-pg
+rspec pg 8 20: *rspec-knapsack-pg
+rspec pg 9 20: *rspec-knapsack-pg
+rspec pg 10 20: *rspec-knapsack-pg
+rspec pg 11 20: *rspec-knapsack-pg
+rspec pg 12 20: *rspec-knapsack-pg
+rspec pg 13 20: *rspec-knapsack-pg
+rspec pg 14 20: *rspec-knapsack-pg
+rspec pg 15 20: *rspec-knapsack-pg
+rspec pg 16 20: *rspec-knapsack-pg
+rspec pg 17 20: *rspec-knapsack-pg
+rspec pg 18 20: *rspec-knapsack-pg
+rspec pg 19 20: *rspec-knapsack-pg
+
+rspec mysql 0 20: *rspec-knapsack-mysql
+rspec mysql 1 20: *rspec-knapsack-mysql
+rspec mysql 2 20: *rspec-knapsack-mysql
+rspec mysql 3 20: *rspec-knapsack-mysql
+rspec mysql 4 20: *rspec-knapsack-mysql
+rspec mysql 5 20: *rspec-knapsack-mysql
+rspec mysql 6 20: *rspec-knapsack-mysql
+rspec mysql 7 20: *rspec-knapsack-mysql
+rspec mysql 8 20: *rspec-knapsack-mysql
+rspec mysql 9 20: *rspec-knapsack-mysql
+rspec mysql 10 20: *rspec-knapsack-mysql
+rspec mysql 11 20: *rspec-knapsack-mysql
+rspec mysql 12 20: *rspec-knapsack-mysql
+rspec mysql 13 20: *rspec-knapsack-mysql
+rspec mysql 14 20: *rspec-knapsack-mysql
+rspec mysql 15 20: *rspec-knapsack-mysql
+rspec mysql 16 20: *rspec-knapsack-mysql
+rspec mysql 17 20: *rspec-knapsack-mysql
+rspec mysql 18 20: *rspec-knapsack-mysql
+rspec mysql 19 20: *rspec-knapsack-mysql
+
+spinach pg 0 10: *spinach-knapsack-pg
+spinach pg 1 10: *spinach-knapsack-pg
+spinach pg 2 10: *spinach-knapsack-pg
+spinach pg 3 10: *spinach-knapsack-pg
+spinach pg 4 10: *spinach-knapsack-pg
+spinach pg 5 10: *spinach-knapsack-pg
+spinach pg 6 10: *spinach-knapsack-pg
+spinach pg 7 10: *spinach-knapsack-pg
+spinach pg 8 10: *spinach-knapsack-pg
+spinach pg 9 10: *spinach-knapsack-pg
+
+spinach mysql 0 10: *spinach-knapsack-mysql
+spinach mysql 1 10: *spinach-knapsack-mysql
+spinach mysql 2 10: *spinach-knapsack-mysql
+spinach mysql 3 10: *spinach-knapsack-mysql
+spinach mysql 4 10: *spinach-knapsack-mysql
+spinach mysql 5 10: *spinach-knapsack-mysql
+spinach mysql 6 10: *spinach-knapsack-mysql
+spinach mysql 7 10: *spinach-knapsack-mysql
+spinach mysql 8 10: *spinach-knapsack-mysql
+spinach mysql 9 10: *spinach-knapsack-mysql
 
 # Other generic tests
 .ruby-static-analysis: &ruby-static-analysis
@@ -228,24 +286,37 @@ rake ee_compat_check:
     paths:
       - ee_compat_check/patches/*.patch
 
-rake db:migrate:reset:
+.db-migrate-reset: &db-migrate-reset
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   script:
     - bundle exec rake db:migrate:reset
 
-rake db:rollback:
+rake pg db:migrate:reset:
+  <<: *db-migrate-reset
+  <<: *use-pg
+
+rake mysql db:migrate:reset:
+  <<: *db-migrate-reset
+  <<: *use-mysql
+
+.db-rollback: &db-rollback
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   script:
     - bundle exec rake db:rollback STEP=120
     - bundle exec rake db:migrate
 
-rake db:seed_fu:
+rake pg db:rollback:
+  <<: *db-rollback
+  <<: *use-pg
+
+rake mysql db:rollback:
+  <<: *db-rollback
+  <<: *use-mysql
+
+.db-seed_fu: &db-seed_fu
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   variables:
     SIZE: "1"
@@ -261,6 +332,14 @@ rake db:seed_fu:
     paths:
       - log/development.log
 
+rake pg db:seed_fu:
+  <<: *db-seed_fu
+  <<: *use-pg
+
+rake mysql db:seed_fu:
+  <<: *db-seed_fu
+  <<: *use-mysql
+
 rake gitlab:assets:compile:
   stage: test
   <<: *dedicated-runner
@@ -285,7 +364,7 @@ rake karma:
     paths:
       - vendor/ruby
   stage: test
-  <<: *use-db
+  <<: *use-pg
   <<: *dedicated-runner
   variables:
     BABEL_ENV: "coverage"
@@ -342,9 +421,8 @@ bundler:audit:
   script:
     - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
 
-migration paths:
+.migration-paths: &migration-paths
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   variables:
     SETUP_DB: "false"
@@ -356,13 +434,21 @@ migration paths:
   script:
     - git fetch origin v8.14.10
     - git checkout -f FETCH_HEAD
-    - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3
+    - bundle install $BUNDLE_INSTALL_FLAGS
     - bundle exec rake db:drop db:create db:schema:load db:seed_fu
     - git checkout $CI_COMMIT_SHA
-    - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3
-    - source scripts/prepare_build.sh
+    - bundle install $BUNDLE_INSTALL_FLAGS
+    - . scripts/prepare_build.sh
     - bundle exec rake db:migrate
 
+migration pg paths:
+  <<: *migration-paths
+  <<: *use-pg
+
+migration mysql paths:
+  <<: *migration-paths
+  <<: *use-mysql
+
 coverage:
   stage: post-test
   services: []
@@ -409,7 +495,7 @@ trigger_docs:
   before_script:
     - apk update && apk add curl
   variables:
-    GIT_STRATEGY: none
+    GIT_STRATEGY: "none"
   cache: {}
   artifacts: {}
   script: