From 211da6778d65ec47ff090171699b983bb8ae8fdb Mon Sep 17 00:00:00 2001 From: Jason Wray Date: Fri, 2 Aug 2019 13:17:38 -0700 Subject: [build] Skip CI jobs if there are no relevant changed files --- circle.yml | 67 +++++++++++++++------- package.json | 1 + scripts/check-ci-job-skippability.js | 107 +++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 19 deletions(-) create mode 100755 scripts/check-ci-job-skippability.js diff --git a/circle.yml b/circle.yml index e7d50421c4..0e3feda38b 100644 --- a/circle.yml +++ b/circle.yml @@ -465,6 +465,15 @@ commands: scripts/notify-slack.sh fi + check-if-this-job-can-be-skipped: + steps: + - run: + name: Check if this job can be skipped + command: | + if [[ $CIRCLE_BRANCH != master ]] && [[ $CIRCLE_BRANCH != release-* ]]; then + scripts/check-ci-job-skippability.js + fi + jobs: nitpick: docker: @@ -555,6 +564,7 @@ jobs: MBGL_ANDROID_STL: << parameters.stl >> steps: - install-dependencies: { gradle: true } + - check-if-this-job-can-be-skipped - run: name: Initialize vendor submodules command: git submodule update --init platform/android/vendor @@ -631,6 +641,7 @@ jobs: IS_LOCAL_DEVELOPMENT: false steps: - install-dependencies: { gradle: true } + - check-if-this-job-can-be-skipped - run: name: Initialize vendor submodules command: git submodule update --init platform/android/vendor @@ -721,6 +732,9 @@ jobs: ANDROID_NDK: /android/sdk/ndk-bundle steps: - checkout + - npm-install + - prepare-environment + - check-if-this-job-can-be-skipped - run: name: Checkout submodules command: | @@ -764,6 +778,7 @@ jobs: WITH_EGL: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-node - save-dependencies - publish-node-package @@ -780,6 +795,7 @@ jobs: - install-macos-dependencies - install-node-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - build-node - save-dependencies - run-node-macos-tests @@ -801,6 +817,7 @@ jobs: WITH_CXX11ABI: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-linux - save-dependencies @@ -822,6 +839,7 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1:color=always:print_summary=1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - setup-llvm-symbolizer - build-test - save-dependencies @@ -844,6 +862,7 @@ jobs: TSAN_OPTIONS: color=always:print_summary=1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - setup-llvm-symbolizer - build-test - save-dependencies @@ -864,6 +883,7 @@ jobs: DISPLAY: :0 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-linux - build-benchmark - build-test @@ -887,6 +907,7 @@ jobs: WITH_COVERAGE: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-linux - build-benchmark - build-test @@ -917,6 +938,7 @@ jobs: WITH_COVERAGE: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - run: name: Install doxygen command: apt update && apt install -y doxygen @@ -941,6 +963,7 @@ jobs: WITH_EGL: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - configure-cmake - build-mbgl-render-test - run-linux-render-tests @@ -958,14 +981,15 @@ jobs: steps: - install-macos-dependencies - install-dependencies - - build-ios-test - check-public-symbols - - run: - name: Check symbol namespacing for mapbox-events-ios - command: make ios-check-events-symbols - run: name: Lint podspecs and plist files command: make ios-lint + - check-if-this-job-can-be-skipped + - build-ios-test + - run: + name: Check symbol namespacing for mapbox-events-ios + command: make ios-check-events-symbols - run: name: Nitpick Darwin code generation command: scripts/nitpick/generated-code.js darwin @@ -973,21 +997,6 @@ jobs: - collect-xcode-build-logs - upload-xcode-build-logs -# ------------------------------------------------------------------------------ - metrics-nightly: - docker: - - image: mbgl/linux-gcc-5:54f59e3ac5 - working_directory: /src - environment: - LIBSYSCONFCPUS: 2 - JOBS: 2 - steps: - - install-dependencies - - run: - name: Collect GitHub statistics - command: | - scripts/publish_github_stats.js - # ------------------------------------------------------------------------------ ios-sanitize-nightly: macos: @@ -1064,6 +1073,7 @@ jobs: steps: - install-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - install-ios-packaging-dependencies - run: name: Build dynamic framework for device and simulator @@ -1162,6 +1172,7 @@ jobs: steps: - install-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - build-macos-test - check-public-symbols - run: @@ -1189,12 +1200,28 @@ jobs: steps: - install-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - configure-cmake - build-mbgl-render-test - save-dependencies - run-macos-render-tests - upload-render-tests +# ------------------------------------------------------------------------------ + metrics-nightly: + docker: + - image: mbgl/linux-gcc-5:54f59e3ac5 + working_directory: /src + environment: + LIBSYSCONFCPUS: 2 + JOBS: 2 + steps: + - install-dependencies + - run: + name: Collect GitHub statistics + command: | + scripts/publish_github_stats.js + # ------------------------------------------------------------------------------ qt5-linux-gcc5-release: docker: @@ -1208,6 +1235,7 @@ jobs: WITH_QT_I18N: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-qt-app - build-qt-test - run: @@ -1234,6 +1262,7 @@ jobs: - install-macos-dependencies - install-qt-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - build-qt-app - build-qt-test - run: diff --git a/package.json b/package.json index e239c60864..acd2eb80bf 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "license": "BSD-2-Clause", "dependencies": { "@mapbox/cmake-node-module": "^1.2.0", + "minimatch": "^3.0.4", "node-pre-gyp": "^0.10.2", "npm-run-all": "^4.0.2" }, diff --git a/scripts/check-ci-job-skippability.js b/scripts/check-ci-job-skippability.js new file mode 100755 index 0000000000..bac24148d5 --- /dev/null +++ b/scripts/check-ci-job-skippability.js @@ -0,0 +1,107 @@ +#!/usr/bin/env node + +/* + +This script takes two parameters or three environment variables: + + {CI job name|$CIRCLE_JOB} {SHA1...SHA1|$CIRCLE_MERGE_BASE...$CIRCLE_SHA1} + +The flow of this script: + + - List of files changed in PR: `git diff --name-only start...end` + - Extract list into array. + - Remove items from array if they match: + - Global ignore list. + - Other platform ignore lists. + - If array still contains items, do not authorize skipping CI job. + - Else if array is empty, skip CI job. + +*/ + +const execSync = require('child_process').execSync; +const _ = require('lodash'); +const minimatch = require("minimatch"); + +console.step = _.partial(console.log, '\n\033[1m\033[36m*', _, '\033[0m'); + +// BUILD PARAMETERS +const CI_JOB_NAME = process.argv[2] || process.env.CIRCLE_JOB; +if (!CI_JOB_NAME) { console.error('You must specify the name of a CI job as a parameter or environment variable.'); process.exit(1); } + +const COMMIT_RANGE = process.argv[3] || `${process.env.CIRCLE_MERGE_BASE}...${process.env.CIRCLE_SHA1}`; +if (!COMMIT_RANGE || COMMIT_RANGE.includes('undefined')) { console.error('You must specify a range of commits to check for changed files in as a parameter or environment variable.'); process.exit(1); } + +// FILE SKIPPING PATTERNS +const ALWAYS_SKIPPABLE_FILES = [ + '**/CHANGELOG.md', + '**/README.md', + '**/CONTRIBUTING.md', + '**/DEVELOPING.md', + '**/INSTALL.md', + '**/*.podspec' +] + +const ANDROID_FILES = [ + 'platform/android/**' +] + +const IOS_FILES = [ + 'platform/ios/**', + 'platform/darwin/**' +] + +const MACOS_FILES = [ + 'platform/macos/**', + 'platform/darwin/**' +] + +const OTHER_PLATFORMS = [ + 'platform/glfw/**', + 'platform/linux/**', + 'platform/node/**', + 'platform/qt/**', + 'benchmark/**', + 'test/**' +] + +function skippableFilePatternsForCIJob(job) { + if (job.startsWith('ios')) { + return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, MACOS_FILES, OTHER_PLATFORMS); + } else if (job.startsWith('android')) { + return _.concat(ALWAYS_SKIPPABLE_FILES, IOS_FILES, MACOS_FILES, OTHER_PLATFORMS); + } else if (job.startsWith('macos')) { + return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, IOS_FILES, OTHER_PLATFORMS); + } else { + return ALWAYS_SKIPPABLE_FILES; + } +} + +console.step(`Getting list of files changed between ${COMMIT_RANGE}`); + +let changedFiles = execSync(`git diff --name-only ${COMMIT_RANGE}`) + .toString() + .trim() + .split('\n'); +changedFiles = _.compact(changedFiles); + +console.log(`${changedFiles.length} total files changed.`); + +// Filter the changed files array to remove files that are irrelevant to the specified CI job. +_.remove(changedFiles, function(changedFile) { + for (const skippableFilePattern of skippableFilePatternsForCIJob(CI_JOB_NAME)) { + if (minimatch(changedFile, skippableFilePattern, { dot: true, nocase: true })) { + return true; + } + } + return false; +}); + +if (changedFiles.length) { + console.log(`Found ${changedFiles.length} unskippable changed file${changedFiles.length > 1 ? 's':''} for ${CI_JOB_NAME}:\n`, changedFiles); +} else { + console.log(`Found no relevant changed files, so it's safe to skip the remainder of this CI job.`) + if (process.env.CIRCLECI) { + console.step('Aborting CI job'); + execSync(`circleci step halt`, {stdio: 'inherit'}); + } +} -- cgit v1.2.1