diff options
Diffstat (limited to '.gitlab-ci.yml')
| -rw-r--r-- | .gitlab-ci.yml | 447 |
1 files changed, 255 insertions, 192 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 476307e7076..45f1638f871 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,31 +1,30 @@ -image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1" +image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6" cache: - key: "ruby-233" + key: "ruby-233-with-yarn" paths: - vendor/ruby + - .yarn-cache/ variables: MYSQL_ALLOW_EMPTY_PASSWORD: "1" RAILS_ENV: "test" + NODE_ENV: "test" SIMPLECOV: "true" - SETUP_DB: "true" - USE_BUNDLE_INSTALL: "true" GIT_DEPTH: "20" + GIT_SUBMODULE_STRATEGY: "none" PHANTOMJS_VERSION: "2.1.1" GET_SOURCES_ATTEMPTS: "3" KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json before_script: - - source ./scripts/prepare_build.sh - - cp config/gitlab.yml.example config/gitlab.yml - bundle --version - - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS' - - retry gem install knapsack fog-aws mime-types - - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql' + - source scripts/utils.sh + - source scripts/prepare_build.sh stages: +- build - prepare - test - post-test @@ -51,21 +50,43 @@ stages: paths: - knapsack/ -.use-db: &use-db +.use-pg: &use-pg + services: + - postgres:latest + - redis:alpine + +.use-mysql: &use-mysql services: - mysql:latest - redis:alpine +.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql + only: + - /mysql/ + - master@gitlab-org/gitlab-ce + - master@gitlab/gitlabhq + - tags@gitlab-org/gitlab-ce + - tags@gitlab/gitlabhq + - //@gitlab-org/gitlab-ee + - //@gitlab/gitlab-ee + +# Skip all jobs except the ones that begin with 'docs/'. +# Used for commits including ONLY documentation changes. +# https://docs.gitlab.com/ce/development/writing_documentation.html#testing +.except-docs: &except-docs + except: + - /(^docs[\/-].*|.*-docs$)/ + .rspec-knapsack: &rspec-knapsack stage: test <<: *dedicated-runner - <<: *use-db script: - JOB_NAME=( $CI_JOB_NAME ) - - export CI_NODE_INDEX=${JOB_NAME[1]} - - export CI_NODE_TOTAL=${JOB_NAME[2]} + - export CI_NODE_INDEX=${JOB_NAME[-2]} + - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true + - export CACHE_CLASSES=true - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - knapsack rspec "--color --format documentation" artifacts: @@ -76,16 +97,27 @@ stages: - knapsack/ - tmp/capybara/ +.rspec-knapsack-pg: &rspec-knapsack-pg + <<: *rspec-knapsack + <<: *use-pg + <<: *except-docs + +.rspec-knapsack-mysql: &rspec-knapsack-mysql + <<: *rspec-knapsack + <<: *use-mysql + <<: *only-master-and-ee-or-mysql + <<: *except-docs + .spinach-knapsack: &spinach-knapsack stage: test <<: *dedicated-runner - <<: *use-db script: - JOB_NAME=( $CI_JOB_NAME ) - - export CI_NODE_INDEX=${JOB_NAME[1]} - - export CI_NODE_TOTAL=${JOB_NAME[2]} + - export CI_NODE_INDEX=${JOB_NAME[-2]} + - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true + - export CACHE_CLASSES=true - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)' artifacts: @@ -96,10 +128,42 @@ stages: - knapsack/ - tmp/capybara/ +.spinach-knapsack-pg: &spinach-knapsack-pg + <<: *spinach-knapsack + <<: *use-pg + <<: *except-docs + +.spinach-knapsack-mysql: &spinach-knapsack-mysql + <<: *spinach-knapsack + <<: *use-mysql + <<: *only-master-and-ee-or-mysql + <<: *except-docs + +.only-canonical-masters: &only-canonical-masters + only: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee + +# Trigger a package build on omnibus-gitlab repository + +build-package: + before_script: [] + services: [] + variables: + SETUP_DB: "false" + USE_BUNDLE_INSTALL: "false" + stage: build + when: manual + script: + - scripts/trigger-build + # Prepare and merge knapsack tests knapsack: <<: *knapsack-state <<: *dedicated-runner + <<: *except-docs stage: prepare script: - mkdir -p knapsack/${CI_PROJECT_NAME}/ @@ -111,27 +175,22 @@ knapsack: update-knapsack: <<: *knapsack-state <<: *dedicated-runner + <<: *only-canonical-masters stage: post-test script: - - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json - - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json + - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json + - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee setup-test-env: - <<: *use-db + <<: *use-pg <<: *dedicated-runner + <<: *except-docs stage: prepare script: - node --version - - yarn --version - - yarn install --pure-lockfile - - yarn check # ensure that yarn.lock matches package.json + - yarn install --pure-lockfile --cache-folder .yarn-cache - bundle exec rake gitlab:assets:compile - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' artifacts: @@ -141,68 +200,121 @@ setup-test-env: - public/assets - tmp/tests -rspec 0 20: *rspec-knapsack -rspec 1 20: *rspec-knapsack -rspec 2 20: *rspec-knapsack -rspec 3 20: *rspec-knapsack -rspec 4 20: *rspec-knapsack -rspec 5 20: *rspec-knapsack -rspec 6 20: *rspec-knapsack -rspec 7 20: *rspec-knapsack -rspec 8 20: *rspec-knapsack -rspec 9 20: *rspec-knapsack -rspec 10 20: *rspec-knapsack -rspec 11 20: *rspec-knapsack -rspec 12 20: *rspec-knapsack -rspec 13 20: *rspec-knapsack -rspec 14 20: *rspec-knapsack -rspec 15 20: *rspec-knapsack -rspec 16 20: *rspec-knapsack -rspec 17 20: *rspec-knapsack -rspec 18 20: *rspec-knapsack -rspec 19 20: *rspec-knapsack - -spinach 0 10: *spinach-knapsack -spinach 1 10: *spinach-knapsack -spinach 2 10: *spinach-knapsack -spinach 3 10: *spinach-knapsack -spinach 4 10: *spinach-knapsack -spinach 5 10: *spinach-knapsack -spinach 6 10: *spinach-knapsack -spinach 7 10: *spinach-knapsack -spinach 8 10: *spinach-knapsack -spinach 9 10: *spinach-knapsack - -# Other generic tests +rspec-pg 0 20: *rspec-knapsack-pg +rspec-pg 1 20: *rspec-knapsack-pg +rspec-pg 2 20: *rspec-knapsack-pg +rspec-pg 3 20: *rspec-knapsack-pg +rspec-pg 4 20: *rspec-knapsack-pg +rspec-pg 5 20: *rspec-knapsack-pg +rspec-pg 6 20: *rspec-knapsack-pg +rspec-pg 7 20: *rspec-knapsack-pg +rspec-pg 8 20: *rspec-knapsack-pg +rspec-pg 9 20: *rspec-knapsack-pg +rspec-pg 10 20: *rspec-knapsack-pg +rspec-pg 11 20: *rspec-knapsack-pg +rspec-pg 12 20: *rspec-knapsack-pg +rspec-pg 13 20: *rspec-knapsack-pg +rspec-pg 14 20: *rspec-knapsack-pg +rspec-pg 15 20: *rspec-knapsack-pg +rspec-pg 16 20: *rspec-knapsack-pg +rspec-pg 17 20: *rspec-knapsack-pg +rspec-pg 18 20: *rspec-knapsack-pg +rspec-pg 19 20: *rspec-knapsack-pg + +rspec-mysql 0 20: *rspec-knapsack-mysql +rspec-mysql 1 20: *rspec-knapsack-mysql +rspec-mysql 2 20: *rspec-knapsack-mysql +rspec-mysql 3 20: *rspec-knapsack-mysql +rspec-mysql 4 20: *rspec-knapsack-mysql +rspec-mysql 5 20: *rspec-knapsack-mysql +rspec-mysql 6 20: *rspec-knapsack-mysql +rspec-mysql 7 20: *rspec-knapsack-mysql +rspec-mysql 8 20: *rspec-knapsack-mysql +rspec-mysql 9 20: *rspec-knapsack-mysql +rspec-mysql 10 20: *rspec-knapsack-mysql +rspec-mysql 11 20: *rspec-knapsack-mysql +rspec-mysql 12 20: *rspec-knapsack-mysql +rspec-mysql 13 20: *rspec-knapsack-mysql +rspec-mysql 14 20: *rspec-knapsack-mysql +rspec-mysql 15 20: *rspec-knapsack-mysql +rspec-mysql 16 20: *rspec-knapsack-mysql +rspec-mysql 17 20: *rspec-knapsack-mysql +rspec-mysql 18 20: *rspec-knapsack-mysql +rspec-mysql 19 20: *rspec-knapsack-mysql + +spinach-pg 0 10: *spinach-knapsack-pg +spinach-pg 1 10: *spinach-knapsack-pg +spinach-pg 2 10: *spinach-knapsack-pg +spinach-pg 3 10: *spinach-knapsack-pg +spinach-pg 4 10: *spinach-knapsack-pg +spinach-pg 5 10: *spinach-knapsack-pg +spinach-pg 6 10: *spinach-knapsack-pg +spinach-pg 7 10: *spinach-knapsack-pg +spinach-pg 8 10: *spinach-knapsack-pg +spinach-pg 9 10: *spinach-knapsack-pg + +spinach-mysql 0 10: *spinach-knapsack-mysql +spinach-mysql 1 10: *spinach-knapsack-mysql +spinach-mysql 2 10: *spinach-knapsack-mysql +spinach-mysql 3 10: *spinach-knapsack-mysql +spinach-mysql 4 10: *spinach-knapsack-mysql +spinach-mysql 5 10: *spinach-knapsack-mysql +spinach-mysql 6 10: *spinach-knapsack-mysql +spinach-mysql 7 10: *spinach-knapsack-mysql +spinach-mysql 8 10: *spinach-knapsack-mysql +spinach-mysql 9 10: *spinach-knapsack-mysql + +# Static analysis jobs .ruby-static-analysis: &ruby-static-analysis variables: SIMPLECOV: "false" SETUP_DB: "false" - USE_BUNDLE_INSTALL: "true" -.exec: &exec +.rake-exec: &rake-exec <<: *ruby-static-analysis <<: *dedicated-runner + <<: *except-docs stage: test script: - - bundle exec $CI_JOB_NAME + - bundle exec rake $CI_JOB_NAME -rubocop: +static-analysis: <<: *ruby-static-analysis <<: *dedicated-runner + <<: *except-docs stage: test script: - - bundle exec "rubocop --require rubocop-rspec" - -rake haml_lint: *exec -rake scss_lint: *exec -rake config_lint: *exec -rake brakeman: *exec -rake flay: *exec -license_finder: *exec -rake downtime_check: *exec -rake ee_compat_check: - <<: *exec + - scripts/static-analysis + +# Documentation checks: +# - Check validity of relative links +# - Make sure cURL examples in API docs use the full switches +docs lint: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine" + stage: test + <<: *dedicated-runner + cache: {} + dependencies: [] + before_script: [] + script: + - scripts/lint-doc.sh + - mv doc/ /nanoc/content/ + - cd /nanoc + # Build HTML from Markdown + - bundle exec nanoc + # Check the internal links + - bundle exec nanoc check internal_links + +downtime_check: + <<: *rake-exec + except: + - master + - tags + - /^[\d-]+-stable(-ee)?$/ + - /(^docs[\/-].*|.*-docs$)/ + +ee_compat_check: + <<: *rake-exec only: - branches@gitlab-org/gitlab-ce except: @@ -221,25 +333,66 @@ rake ee_compat_check: paths: - ee_compat_check/patches/*.patch -rake db:migrate:reset: +# DB migration, rollback, and seed jobs +.db-migrate-reset: &db-migrate-reset stage: test - <<: *use-db <<: *dedicated-runner + <<: *except-docs script: - bundle exec rake db:migrate:reset -rake db:rollback: +db:migrate:reset-pg: + <<: *db-migrate-reset + <<: *use-pg + +db:migrate:reset-mysql: + <<: *db-migrate-reset + <<: *use-mysql + +.migration-paths: &migration-paths stage: test - <<: *use-db <<: *dedicated-runner + variables: + SETUP_DB: "false" + <<: *only-canonical-masters + script: + - git fetch origin v8.14.10 + - git checkout -f FETCH_HEAD + - bundle install $BUNDLE_INSTALL_FLAGS + - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - git checkout $CI_COMMIT_SHA + - bundle install $BUNDLE_INSTALL_FLAGS + - . scripts/prepare_build.sh + - bundle exec rake db:migrate + +migration:path-pg: + <<: *migration-paths + <<: *use-pg + +migration:path-mysql: + <<: *migration-paths + <<: *use-mysql + +.db-rollback: &db-rollback + stage: test + <<: *dedicated-runner + <<: *except-docs script: - bundle exec rake db:rollback STEP=120 - bundle exec rake db:migrate -rake db:seed_fu: +db:rollback-pg: + <<: *db-rollback + <<: *use-pg + +db:rollback-mysql: + <<: *db-rollback + <<: *use-mysql + +.db-seed_fu: &db-seed_fu stage: test - <<: *use-db <<: *dedicated-runner + <<: *except-docs variables: SIZE: "1" SETUP_DB: "false" @@ -254,9 +407,19 @@ rake db:seed_fu: paths: - log/development.log -rake gitlab:assets:compile: +db:seed_fu-pg: + <<: *db-seed_fu + <<: *use-pg + +db:seed_fu-mysql: + <<: *db-seed_fu + <<: *use-mysql + +# Frontend-related jobs +gitlab:assets:compile: stage: test <<: *dedicated-runner + <<: *except-docs dependencies: [] variables: NODE_ENV: "production" @@ -266,21 +429,19 @@ rake gitlab:assets:compile: SKIP_STORAGE_VALIDATION: "true" WEBPACK_REPORT: "true" script: - - bundle exec rake yarn:install gitlab:assets:compile + - yarn install --pure-lockfile --production --cache-folder .yarn-cache + - bundle exec rake gitlab:assets:compile artifacts: name: webpack-report expire_in: 31d paths: - webpack-report/ -rake karma: - cache: - paths: - - vendor/ruby - - node_modules +karma: stage: test - <<: *use-db + <<: *use-pg <<: *dedicated-runner + <<: *except-docs variables: BABEL_ENV: "coverage" script: @@ -292,81 +453,11 @@ rake karma: paths: - coverage-javascript/ -docs:check:apilint: - image: "phusion/baseimage" - stage: test - <<: *dedicated-runner - variables: - GIT_DEPTH: "3" - cache: {} - dependencies: [] - before_script: [] - script: - - scripts/lint-doc.sh - -docs:check:links: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine" - stage: test - <<: *dedicated-runner - variables: - GIT_DEPTH: "3" - cache: {} - dependencies: [] - before_script: [] - script: - - mv doc/ /nanoc/content/ - - cd /nanoc - # Build HTML from Markdown - - bundle exec nanoc - # Check the internal links - - bundle exec nanoc check internal_links - -bundler:check: - stage: test - <<: *dedicated-runner - <<: *ruby-static-analysis - script: - - bundle check - -bundler:audit: - stage: test - <<: *ruby-static-analysis - <<: *dedicated-runner - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee - script: - - "bundle exec bundle-audit check --update --ignore CVE-2016-4658" - -migration paths: - stage: test - <<: *use-db - <<: *dedicated-runner - variables: - SETUP_DB: "false" - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee - script: - - git fetch origin v8.5.9 - - git checkout -f FETCH_HEAD - - cp config/resque.yml.example config/resque.yml - - sed -i 's/localhost/redis/g' config/resque.yml - - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3 - - bundle exec rake db:drop db:create db:schema:load db:seed_fu - - git checkout $CI_COMMIT_SHA - - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3 - - source scripts/prepare_build.sh - - bundle exec rake db:migrate - coverage: stage: post-test services: [] <<: *dedicated-runner + <<: *except-docs variables: SETUP_DB: "false" USE_BUNDLE_INSTALL: "true" @@ -380,21 +471,9 @@ coverage: - coverage/index.html - coverage/assets/ -lint:javascript: - <<: *dedicated-runner - cache: - paths: - - node_modules/ - stage: test - before_script: [] - script: - - yarn run eslint - lint:javascript:report: <<: *dedicated-runner - cache: - paths: - - node_modules/ + <<: *except-docs stage: post-test before_script: [] script: @@ -415,7 +494,7 @@ trigger_docs: before_script: - apk update && apk add curl variables: - GIT_STRATEGY: none + GIT_STRATEGY: "none" cache: {} artifacts: {} script: @@ -425,30 +504,14 @@ trigger_docs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee -# Notify slack in the end -notify:slack: - stage: post-test - <<: *dedicated-runner - variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "false" - script: - - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>" - when: on_failure - only: - - master@gitlab-org/gitlab-ce - - tags@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - tags@gitlab-org/gitlab-ee - pages: before_script: [] stage: pages <<: *dedicated-runner dependencies: - coverage - - rake karma - - rake gitlab:assets:compile + - karma + - gitlab:assets:compile - lint:javascript:report script: - mv public/ .public/ |