diff options
| -rw-r--r-- | .gitlab-ci.yml | 26 | ||||
| -rw-r--r-- | .gitlab/ci/cng.gitlab-ci.yml | 7 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 47 | ||||
| -rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 167 | ||||
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 102 | ||||
| -rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 17 | ||||
| -rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 21 | ||||
| -rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 6 | ||||
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 288 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 33 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 129 | ||||
| -rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 45 | ||||
| -rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 30 | ||||
| -rw-r--r-- | .gitlab/ci/yaml.gitlab-ci.yml | 7 |
14 files changed, 474 insertions, 451 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f926cbc2939..4c7a8c05b37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,15 @@ image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33" +stages: + - build + - prepare + - quick-test + - test + - review + - qa + - post-test + - pages + variables: RAILS_ENV: "test" NODE_ENV: "test" @@ -11,25 +21,9 @@ variables: FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json BUILD_ASSETS_IMAGE: "false" -before_script: - - date - - source scripts/utils.sh - - source scripts/prepare_build.sh - - date - after_script: - date -stages: - - build - - prepare - - merge - - test - - review - - qa - - post-test - - pages - include: - local: .gitlab/ci/global.gitlab-ci.yml - local: .gitlab/ci/cng.gitlab-ci.yml diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index d624e8d09f6..a43d3694103 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -1,16 +1,15 @@ cloud-native-image: image: ruby:2.6-alpine - before_script: [] dependencies: [] stage: post-test allow_failure: true variables: GIT_DEPTH: "1" - cache: {} when: manual script: - install_gitlab_gem - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng only: - - tags@gitlab-org/gitlab-ce - - tags@gitlab-org/gitlab-ee + refs: + - tags@gitlab-org/gitlab-ce + - tags@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index d724ab07663..1fad731543c 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -1,24 +1,34 @@ -.review-docs: &review-docs - extends: .single-script-job-dedicated-runner +.review-docs: + extends: + - .default-tags + - .default-retry + image: ruby:2.6-alpine + stage: review + dependencies: [] variables: - SCRIPT_NAME: trigger-build-docs + GIT_STRATEGY: none environment: name: review-docs/$CI_COMMIT_REF_SLUG # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX on_stop: review-docs-cleanup + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs + - chmod 755 trigger-build-docs # Trigger a manual docs build in gitlab-docs only on non docs-only branches. # Useful to preview the docs changes live. review-docs-deploy-manual: extends: - .review-docs - - .no-docs-and-no-qa - stage: review + - .except-docs-qa script: - gem install gitlab --no-document - - ./$SCRIPT_NAME deploy + - ./trigger-build-docs deploy when: manual only: - branches@gitlab-org/gitlab-ce @@ -27,39 +37,40 @@ review-docs-deploy-manual: # Always trigger a docs build in gitlab-docs only on docs-only branches. # Useful to preview the docs changes live. review-docs-deploy: - <<: *review-docs - stage: review + extends: + - .review-docs + - .except-qa script: - gem install gitlab --no-document - - ./$SCRIPT_NAME deploy + - ./trigger-build-docs deploy only: - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ce - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-ee - except: - - /(^qa[\/-].*|.*-qa$)/ # Cleanup remote environment of gitlab-docs review-docs-cleanup: - <<: *review-docs - stage: review + extends: + - .review-docs + - .except-qa environment: name: review-docs/$CI_COMMIT_REF_SLUG action: stop script: - gem install gitlab --no-document - - ./$SCRIPT_NAME cleanup + - ./trigger-build-docs cleanup when: manual only: - branches@gitlab-org/gitlab-ce - branches@gitlab-org/gitlab-ee docs lint: - extends: .dedicated-runner + extends: + - .default-tags + - .default-retry + - .except-qa image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint" stage: test - cache: {} dependencies: [] - before_script: [] script: - scripts/lint-doc.sh - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX @@ -72,5 +83,3 @@ docs lint: - bundle exec nanoc check internal_links # Check the internal anchor links - bundle exec nanoc check internal_anchors - except: - - /(^qa[\/-].*|.*-qa$)/ diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 5c3278fcf53..df38cb4ff8e 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,23 +1,19 @@ -.assets-compile-cache: &assets-compile-cache +.assets-compile-cache: cache: - key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" paths: - vendor/ruby/ - .yarn-cache/ - tmp/cache/assets/sprockets -.use-pg: &use-pg - services: - - name: postgres:9.6.14 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - .gitlab:assets:compile-metadata: - <<: *assets-compile-cache - extends: .dedicated-no-docs-pull-cache-job + extends: + - .default-tags + - .default-retry + - .assets-compile-cache + - .default-before_script + - .except-docs image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 - dependencies: - - setup-test-env + dependencies: ["setup-test-env"] services: - docker:19.03.0-dind variables: @@ -30,6 +26,14 @@ NODE_OPTIONS: --max_old_space_size=3584 DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 + cache: + key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" + artifacts: + name: webpack-report + expire_in: 31d + paths: + - webpack-report/ + - public/assets/ script: - node --version - retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline @@ -42,43 +46,41 @@ - install_api_client_dependencies_with_apt - play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - artifacts: - name: webpack-report - expire_in: 31d - paths: - - webpack-report/ - - public/assets/ only: - /.+/@gitlab-org/gitlab-ce - /.+/@gitlab-org/gitlab-ee - /.+/@gitlab/gitlabhq - /.+/@gitlab/gitlab-ee tags: - - docker - gitlab-org + - docker gitlab:assets:compile: extends: .gitlab:assets:compile-metadata + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee cache: policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee gitlab:assets:compile pull-cache: extends: .gitlab:assets:compile-metadata - cache: - policy: pull except: refs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - /(^docs[\/-].+|.+-docs$)/ + cache: + policy: pull .compile-assets-metadata: - extends: .dedicated-runner - <<: *use-pg - <<: *assets-compile-cache + extends: + - .default-tags + - .default-retry + - .assets-compile-cache + - .default-before_script + - .use-pg stage: prepare script: - node --version @@ -89,6 +91,8 @@ gitlab:assets:compile pull-cache: variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 + cache: + key: "assets-compile:test:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" artifacts: expire_in: 7d paths: @@ -96,30 +100,34 @@ gitlab:assets:compile pull-cache: - public/assets compile-assets: - extends: .compile-assets-metadata + extends: + - .compile-assets-metadata + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee cache: policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee compile-assets pull-cache: extends: .compile-assets-metadata - cache: - policy: pull except: refs: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - /(^docs[\/-].+|.+-docs$)/ + cache: + policy: pull karma: - extends: .dedicated-no-docs-pull-cache-job - <<: *use-pg - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs + dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 @@ -142,12 +150,14 @@ karma: junit: junit_karma.xml jest: - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - <<: *use-pg - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"] script: - scripts/gitaly-test-spawn - date @@ -170,36 +180,41 @@ jest: - tmp/jest/jest/ policy: pull-push -qa:internal: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] - script: +.qa: + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + before_script: - cd qa/ - bundle install + +qa:internal: + extends: .qa + script: - bundle exec rspec - dependencies: - - setup-test-env qa:selectors: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] + extends: .qa script: - - cd qa/ - - bundle install - bundle exec bin/qa Test::Sanity::Selectors - dependencies: - - setup-test-env -.qa-frontend-node: &qa-frontend-node - extends: .dedicated-no-docs-no-db-pull-cache-job - stage: test +.qa-frontend-node: + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + dependencies: [] cache: key: "$CI_JOB_NAME" paths: - .yarn-cache/ policy: pull-push - dependencies: [] - before_script: [] script: - date - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline @@ -207,23 +222,28 @@ qa:selectors: - yarn run webpack-prod qa-frontend-node:8: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:carbon qa-frontend-node:10: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:dubnium qa-frontend-node:latest: - <<: *qa-frontend-node + extends: .qa-frontend-node image: node:latest allow_failure: true lint:javascript:report: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + variables: + SETUP_DB: "false" stage: post-test dependencies: [] - before_script: [] script: - date - yarn run eslint-report || true # ignore exit code @@ -234,12 +254,15 @@ lint:javascript:report: - eslint-report.html jsdoc: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + variables: + SETUP_DB: "false" stage: post-test - dependencies: - - compile-assets - - compile-assets pull-cache - before_script: [] + dependencies: ["compile-assets", "compile-assets pull-cache"] script: - date - yarn run jsdoc || true # ignore exit code diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 56fe9739d5f..04135447ca4 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -1,83 +1,56 @@ -.dedicated-runner: +.default-tags: + tags: + - gitlab-org + +.default-retry: retry: max: 2 # This is confusing but this means "3 runs at max". when: - unknown_failure - api_failure - runner_system_failure - tags: - - gitlab-org - -.default-cache: &default-cache - key: "debian-stretch-ruby-2.6.3-node-12.x" - paths: - - vendor/ruby - - .yarn-cache/ - - vendor/gitaly-ruby -.dedicated-runner-default-cache: - extends: .dedicated-runner - cache: - <<: *default-cache +.default-before_script: + before_script: + - date + - source scripts/utils.sh + - source scripts/prepare_build.sh + - date # Jobs that only need to pull cache -.dedicated-pull-cache-job: - extends: .dedicated-runner +.default-cache: cache: - <<: *default-cache + key: "debian-stretch-ruby-2.6.3-node-12.x" + paths: + - vendor/ruby + - .yarn-cache/ + - vendor/gitaly-ruby policy: pull - stage: test -.no-docs: +.except-docs: except: refs: - /(^docs[\/-].+|.+-docs$)/ -.no-docs-and-no-qa: +.except-qa: except: refs: - - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ -.dedicated-no-docs-pull-cache-job: - extends: - - .dedicated-pull-cache-job - - .no-docs - -.dedicated-no-docs-and-no-qa-pull-cache-job: - extends: - - .dedicated-pull-cache-job - - .no-docs-and-no-qa - -# Jobs that do not need a DB -.dedicated-no-docs-no-db-pull-cache-job: - extends: .dedicated-no-docs-pull-cache-job - variables: - SETUP_DB: "false" - -# Jobs that need a dedicated runner, with no cache -.dedicated-no-docs: - extends: - - .dedicated-runner - - .no-docs +.except-docs-qa: + except: + refs: + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ -.single-script-job-dedicated-runner: - extends: .dedicated-runner - image: ruby:2.6-alpine - stage: test - cache: {} - dependencies: [] - variables: - GIT_STRATEGY: none - before_script: - # We don't clone the repo by using GIT_STRATEGY: none and only download the - # single script we need here so it's much faster than cloning. - - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}" - - apk add --update openssl - - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME - - chmod 755 $(basename $SCRIPT_NAME) +.except-docs-qa-geo: + except: + refs: + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ + - /(^geo[\/-].*|.*-geo$)/ -.review-only: &review-only +.review-only: only: refs: - branches@gitlab-org/gitlab-ce @@ -88,3 +61,16 @@ - master - /^\d+-\d+-auto-deploy-\d+$/ - /(^docs[\/-].+|.+-docs$)/ + +.use-pg: + services: + - name: postgres:9.6.14 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + +.use-pg-10: + image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" + services: + - name: postgres:10.9 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 9923732e587..1936933cca4 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -1,5 +1,12 @@ memory-static: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + variables: + SETUP_DB: "false" script: # Uses two different reports from the 'derailed_benchmars' gem. @@ -23,7 +30,13 @@ memory-static: # The application is booted in `production` environment. # All tests are run without a webserver (directly using Rack::Mock by default). memory-on-boot: - extends: .rspec-metadata-pg-10 + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg-10 + - .except-docs-qa variables: NODE_ENV: "production" RAILS_ENV: "production" diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index f7b18b809b4..3247d7c4bce 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -1,13 +1,15 @@ pages: - extends: .dedicated-no-docs-no-db-pull-cache-job - before_script: [] + extends: + - .default-tags + - .default-retry + - .default-cache + - .except-docs + only: + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee stage: pages - dependencies: - - coverage - - karma - - gitlab:assets:compile - - lint:javascript:report - - jsdoc + dependencies: ["coverage", "karma", "gitlab:assets:compile", "lint:javascript:report", "jsdoc"] script: - mv public/ .public/ - mkdir public/ @@ -21,6 +23,3 @@ pages: artifacts: paths: - public - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 69b932801ad..ac2a70dda0b 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,10 +1,8 @@ .package-and-qa-base: image: ruby:2.6-alpine stage: qa - before_script: [] needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] dependencies: [] - cache: {} variables: GIT_DEPTH: "1" retry: 0 @@ -19,7 +17,9 @@ - master package-and-qa-manual: - extends: .package-and-qa-base + extends: + - .package-and-qa-base + - .except-docs-qa when: manual except: - master diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 50476b43dd6..2e8b197829b 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,52 +1,31 @@ -.use-pg: &use-pg - services: - - name: postgres:9.6.14 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - -.use-pg-10: &use-pg-10 - services: - - name: postgres:10.9 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:alpine - -.only-schedules-master: &only-schedules-master +.only-schedules-master: only: - - schedules@gitlab-org/gitlab-ce - - schedules@gitlab-org/gitlab-ee - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + refs: + - schedules@gitlab-org/gitlab-ce + - schedules@gitlab-org/gitlab-ee + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee -.gitlab-setup: &gitlab-setup +.rake-exec: extends: - - .dedicated-no-docs-and-no-qa-pull-cache-job - - .use-pg + - .default-tags + - .default-retry + - .default-cache + - .default-before_script variables: SETUP_DB: "false" script: - # Manually clone gitlab-test and only seed this project in - # db/fixtures/development/04_project.rb thanks to SIZE=1 below - - git clone https://gitlab.com/gitlab-org/gitlab-test.git - /home/git/repositories/gitlab-org/gitlab-test.git - - scripts/gitaly-test-spawn - - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup - artifacts: - when: on_failure - expire_in: 1d - paths: - - log/development.log - -.rake-exec: &rake-exec - extends: .dedicated-no-docs-no-db-pull-cache-job - script: - bundle exec rake $CI_JOB_NAME -.rspec-metadata: &rspec-metadata +.rspec-base: extends: - - .dedicated-pull-cache-job - - .no-docs-and-no-qa + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs-qa stage: test script: - JOB_NAME=( $CI_JOB_NAME ) @@ -83,52 +62,24 @@ reports: junit: junit_rspec.xml -.rspec-metadata-pg: &rspec-metadata-pg - <<: *rspec-metadata - <<: *use-pg - -.rspec-metadata-pg-10: &rspec-metadata-pg-10 - <<: *rspec-metadata - <<: *use-pg-10 - image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" - -# DB migration, rollback, and seed jobs -.db-migrate-reset: &db-migrate-reset - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - script: - - bundle exec rake db:migrate:reset - dependencies: - - setup-test-env +.rspec-base-pg: + extends: + - .rspec-base + - .use-pg -.migration-paths: &migration-paths - extends: .dedicated-no-docs-and-no-qa-pull-cache-job - variables: - SETUP_DB: "false" - script: - - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0 - - git checkout -f FETCH_HEAD - - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile - - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile - - bundle update google-protobuf grpc bootsnap - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:schema:load db:seed_fu - - date - - git checkout -f $CI_COMMIT_SHA - - bundle install $BUNDLE_INSTALL_FLAGS - - date - - . scripts/prepare_build.sh - - date - - bundle exec rake db:migrate - dependencies: - - setup-test-env +.rspec-base-pg-10: + extends: + - .rspec-base + - .use-pg-10 setup-test-env: extends: - - .dedicated-runner-default-cache - - .no-docs + - .default-tags + - .default-retry + - .default-cache + - .default-before_script - .use-pg + - .except-docs stage: prepare script: - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' @@ -141,67 +92,72 @@ setup-test-env: - vendor/gitaly-ruby rspec unit pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 20 rspec integration pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 6 rspec system pg: - <<: *rspec-metadata-pg + extends: .rspec-base-pg parallel: 24 rspec unit pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 20 rspec integration pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 6 rspec system pg-10: - <<: *rspec-metadata-pg-10 - <<: *only-schedules-master + extends: + - .rspec-base-pg-10 + - .only-schedules-master parallel: 24 rspec-fast-spec-helper: - <<: *rspec-metadata-pg + extends: .rspec-base-pg script: - bundle exec rspec spec/fast_spec_helper.rb -.rspec-quarantine: &rspec-quarantine - <<: *only-schedules-master +rspec quarantine pg: + extends: + - .default-before_script + - .rspec-base-pg + - .only-schedules-master script: - - export CACHE_CLASSES=true + - export NO_KNAPSACK=1 CACHE_CLASSES=true - scripts/gitaly-test-spawn - bin/rspec --color --format documentation --tag quarantine -- spec/ - -rspec quarantine pg: - <<: *rspec-metadata-pg - <<: *rspec-quarantine allow_failure: true static-analysis: - extends: .dedicated-no-docs-no-db-pull-cache-job - dependencies: - - compile-assets - - compile-assets pull-cache - - setup-test-env + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"] + variables: + SETUP_DB: "false" script: - scripts/static-analysis cache: - key: "debian-stretch-ruby-2.6.3-node-12.x-and-rubocop" + key: "debian-stretch-ruby-2.6.3-and-rubocop" paths: - vendor/ruby - - .yarn-cache/ - tmp/rubocop_cache policy: pull-push downtime_check: - <<: *rake-exec + extends: .rake-exec except: refs: - master @@ -209,22 +165,20 @@ downtime_check: - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ - dependencies: - - setup-test-env + dependencies: ["setup-test-env"] ee_compat_check: - <<: *rake-exec + extends: .rake-exec dependencies: [] except: refs: - master - tags - - /[\d-]+-stable(-ee)?/ - - /^security-/ - branches@gitlab-org/gitlab-ee - branches@gitlab/gitlab-ee + - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].+|.+-docs$)/ - retry: 0 + - /^security-/ artifacts: name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" when: always @@ -232,44 +186,106 @@ ee_compat_check: paths: - ee_compat_check/patches/*.patch -db:migrate:reset-pg: - <<: *db-migrate-reset - <<: *use-pg +# DB migration, rollback, and seed jobs +db:migrate:reset: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + script: + - bundle exec rake db:migrate:reset -db:check-schema-pg: - <<: *db-migrate-reset - <<: *use-pg +db:check-schema: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] script: - source scripts/schema_changed.sh -migration:path-pg: - <<: *migration-paths - <<: *use-pg +db:migrate-from-v11.11.0: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + script: + - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v11.11.0 + - git checkout -f FETCH_HEAD + - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile + - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile + - bundle update google-protobuf grpc bootsnap + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - cp config/gitlab.yml.example config/gitlab.yml + - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - date + - git checkout -f $CI_COMMIT_SHA + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - . scripts/prepare_build.sh + - date + - bundle exec rake db:migrate -.db-rollback: &db-rollback - extends: .dedicated-no-docs-and-no-qa-pull-cache-job +db:rollback: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] script: - bundle exec rake db:migrate VERSION=20180101160629 - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true - dependencies: - - setup-test-env - -db:rollback-pg: - <<: *db-rollback - <<: *use-pg -gitlab:setup-pg: - <<: *gitlab-setup - <<: *use-pg - dependencies: - - setup-test-env +gitlab:setup: + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .use-pg + - .except-docs-qa + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" + script: + # Manually clone gitlab-test and only seed this project in + # db/fixtures/development/04_project.rb thanks to SIZE=1 below + - git clone https://gitlab.com/gitlab-org/gitlab-test.git + /home/git/repositories/gitlab-org/gitlab-test.git + - scripts/gitaly-test-spawn + - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup + artifacts: + when: on_failure + expire_in: 1d + paths: + - log/development.log coverage: # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to # download artifacts from all the rspec jobs instead of from setup-test-env only extends: - - .dedicated-runner-default-cache - - .no-docs-and-no-qa + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs-qa cache: policy: pull variables: diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index e3768ecf2a2..905f1ede2c4 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -4,36 +4,29 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml +.reports: + extends: + - .default-retry + - .except-docs + tags: + - gitlab-org + - docker + code_quality: - extends: .dedicated-no-docs - # gitlab-org runners set `privileged: false` but we need to have it set to true - # since we're using Docker in Docker - tags: [] - before_script: [] - cache: {} + extends: .reports sast: - extends: .dedicated-no-docs - tags: [] - before_script: [] - cache: {} + extends: .reports variables: SAST_BRAKEMAN_LEVEL: 2 SAST_EXCLUDED_PATHS: qa,spec,doc dependency_scanning: - extends: .dedicated-no-docs - tags: [] - before_script: [] - cache: {} + extends: .reports dast: - extends: - - .dedicated-runner - - .review-only + extends: .reports stage: qa - dependencies: - - review-deploy + dependencies: ["review-deploy"] before_script: - export DAST_WEBSITE="$(cat review_app_url.txt)" - cache: {} diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index beb049c0b3b..3415f1b6ab4 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,4 +1,4 @@ -.review-schedules-only: &review-schedules-only +.review-schedules-only: only: refs: - schedules@gitlab-org/gitlab-ce @@ -11,39 +11,39 @@ - tags - /(^docs[\/-].+|.+-docs$)/ -.review-base: &review-base +.review-base: extends: - - .dedicated-runner + - .default-tags + - .default-retry - .review-only image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - cache: {} dependencies: [] before_script: - source scripts/utils.sh -.review-docker: &review-docker - <<: *review-base +.review-docker: + extends: .review-base image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine services: - docker:19.03.0-dind tags: - gitlab-org - docker - variables: &review-docker-variables + variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly" QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}" build-qa-image: - <<: *review-docker + extends: .review-docker stage: test script: - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY} - time docker push ${QA_IMAGE} -.review-build-cng-base: &review-build-cng-base +.review-build-cng-base: image: ruby:2.6-alpine stage: test when: manual @@ -52,20 +52,21 @@ build-qa-image: - install_api_client_dependencies_with_apk - install_gitlab_gem dependencies: [] - cache: {} script: - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng review-build-cng: - extends: .review-only - <<: *review-build-cng-base + extends: + - .review-build-cng-base + - .review-only schedule:review-build-cng: - <<: *review-schedules-only - <<: *review-build-cng-base + extends: + - .review-build-cng-base + - .review-schedules-only -.review-deploy-base: &review-deploy-base - <<: *review-base +review-deploy: + extends: .review-base allow_failure: true retry: 1 stage: review @@ -73,7 +74,7 @@ schedule:review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "master" - environment: &review-environment + environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop @@ -98,50 +99,45 @@ schedule:review-build-cng: expire_in: 2 days when: always -review-deploy: - <<: *review-deploy-base - schedule:review-deploy: - <<: *review-deploy-base - <<: *review-schedules-only + extends: + - review-deploy + - .review-schedules-only review-stop: - extends: - - .single-script-job-dedicated-runner - - .review-only - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - stage: review + extends: review-deploy when: manual - allow_failure: true - variables: - SCRIPT_NAME: review_apps/review-apps.sh environment: - <<: *review-environment action: stop - script: + variables: + GIT_STRATEGY: none + before_script: + # We don't clone the repo by using GIT_STRATEGY: none and only download the + # single script we need here so it's much faster than cloning. + - apk add --update openssl + - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/review_apps/review-apps.sh - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/utils.sh - source utils.sh - - source $(basename $SCRIPT_NAME) + - source review-apps.sh + script: - delete + artifacts: {} -.review-qa-base: &review-qa-base - <<: *review-docker - allow_failure: true +.review-qa-base: + extends: .review-docker retry: 2 stage: qa variables: - <<: *review-docker-variables QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + QA_DEBUG: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_ADMIN_USERNAME: "root" GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" - QA_DEBUG: "true" - dependencies: - - review-deploy + dependencies: ["review-deploy"] artifacts: paths: - ./qa/gitlab-qa-run-* @@ -156,12 +152,13 @@ review-stop: - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} review-qa-smoke: - <<: *review-qa-base + extends: .review-qa-base + allow_failure: true script: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" review-qa-all: - <<: *review-qa-base + extends: .review-qa-base allow_failure: true when: manual parallel: 5 @@ -172,21 +169,16 @@ review-qa-all: parallel-spec-reports: extends: - - .dedicated-runner - - .no-docs - dependencies: - - review-qa-all + - .default-tags + - .except-docs image: ruby:2.6-alpine - services: [] - before_script: [] + stage: post-test + dependencies: ["review-qa-all"] variables: - SETUP_DB: "false" NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" - stage: post-test allow_failure: true when: manual - retry: 0 artifacts: when: always paths: @@ -196,15 +188,15 @@ parallel-spec-reports: junit: qa/gitlab-qa-run-*/**/rspec-*.xml script: - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* - - gem install nokogiri + - gem install nokogiri --no-document - cd qa/gitlab-qa-run-*/gitlab-* - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) - - cd ../../.. + - cd - - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm -.review-performance-base: &review-performance-base - <<: *review-qa-base +review-performance: + extends: .review-qa-base allow_failure: true before_script: - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" @@ -222,18 +214,16 @@ parallel-spec-reports: reports: performance: performance.json -review-performance: - <<: *review-performance-base - schedule:review-performance: - <<: *review-performance-base - <<: *review-schedules-only - dependencies: - - schedule:review-deploy + extends: + - review-performance + - .review-schedules-only + dependencies: ["schedule:review-deploy"] schedule:review-cleanup: - <<: *review-base - <<: *review-schedules-only + extends: + - .review-base + - .review-schedules-only stage: build allow_failure: true environment: @@ -246,11 +236,13 @@ schedule:review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb danger-review: - extends: .dedicated-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test dependencies: [] - before_script: [] only: variables: - $DANGER_GITLAB_API_TOKEN @@ -259,9 +251,8 @@ danger-review: - master - /^\d+-\d+-auto-deploy-\d+$/ - /^[\d-]+-stable(-ee)?$/ - variables: - - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/ - - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/ + - /^ce-to-ee-.*/ + - /.*-stable(-ee)?-prepare-.*/ script: - git version - node --version diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index c1fc3a893ca..d9384780356 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -1,41 +1,42 @@ # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: + - .default-tags + - .default-retry + - .default-cache + - .default-before_script + - .except-docs + dependencies: ["setup-test-env"] + variables: + SETUP_DB: "false" script: - bundle package --all --all-platforms artifacts: paths: - vendor/cache only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - tags - dependencies: - - setup-test-env + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - tags -gitlab_git_test: +.minimal-job: extends: - - .dedicated-runner - - .no-docs-and-no-qa - variables: - SETUP_DB: "false" - before_script: [] + - .default-tags + - .default-retry + - .except-docs-qa dependencies: [] - cache: {} + +gitlab_git_test: + extends: .minimal-job script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes no_ee_check: - extends: - - .dedicated-runner - - .no-docs-and-no-qa - variables: - SETUP_DB: "false" - before_script: [] - dependencies: [] - cache: {} + extends: .minimal-job script: - scripts/no-ee-check only: - - /.+/@gitlab-org/gitlab-ce + refs: + - branches@gitlab-org/gitlab-ce diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 4c97a4feb18..b9dac64957e 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,5 +1,4 @@ -.tests-metadata-state: &tests-metadata-state - extends: .dedicated-runner +.tests-metadata-state: variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: @@ -14,7 +13,7 @@ retrieve-tests-metadata: extends: - .tests-metadata-state - - .no-docs-and-no-qa + - .except-docs-qa stage: prepare cache: key: tests_metadata @@ -29,7 +28,7 @@ retrieve-tests-metadata: - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' update-tests-metadata: - <<: *tests-metadata-state + extends: .tests-metadata-state stage: post-test cache: key: tests_metadata @@ -49,25 +48,24 @@ update-tests-metadata: - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json - scripts/insert-rspec-profiling-data only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - master@gitlab/gitlabhq - - master@gitlab/gitlab-ee + refs: + - master@gitlab-org/gitlab-ce + - master@gitlab-org/gitlab-ee + - master@gitlab/gitlabhq + - master@gitlab/gitlab-ee flaky-examples-check: - extends: .dedicated-runner + extends: + - .default-tags + - .default-retry image: ruby:2.6-alpine - services: [] - before_script: [] + stage: post-test variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "false" NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json - stage: post-test allow_failure: true - retry: 0 only: - - branches + refs: + - branches except: refs: - master diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index b7aa418d8f7..3e107b475c9 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -1,9 +1,10 @@ # Yamllint of *.yml for .gitlab-ci.yml. # This uses rules from project root `.yamllint`. lint-ci-gitlab: - extends: .dedicated-runner - before_script: [] - dependencies: [] + extends: + - .default-tags + - .default-retry image: sdesbure/yamllint:latest + dependencies: [] script: - yamllint .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates changelogs |