diff options
-rw-r--r-- | circle.yml | 97 | ||||
-rw-r--r-- | package.json | 1 | ||||
-rwxr-xr-x | scripts/check-ci-job-skippability.js | 121 |
3 files changed, 196 insertions, 23 deletions
diff --git a/circle.yml b/circle.yml index 464237b0b2..68c49dd3f6 100644 --- a/circle.yml +++ b/circle.yml @@ -4,6 +4,14 @@ workflows: version: 2 default: jobs: + # + # Naming convention: {platform}-{additional description}-{build type} + # - {platform} is the client platform/framework, which may differ from + # the build platform. Specify both if applicable, e.g., "qt5-macos". + # - {additional description} optionally describes the compiler or other + # unique aspect of the build environment. + # - {build type} is typically "debug" or "release". + # - nitpick - clang-tidy: filters: @@ -467,6 +475,28 @@ commands: scripts/notify-slack.sh fi + # + # Add this step to all regular jobs to enable skipping of certain non-code-related changes. + # + # Do not include this step in nightly or release deployment jobs. + # + # To make a job potentially skippable on changes unrelated to its platform, it must: + # - Target one of the skippable platforms: Android, iOS, or macOS. + # - Have a job name that begins with a supported platform name. + # - Not be related to core functionality or rendering tests. Job names that + # contain "render-tests" cannot be skipped by platform changes. + # + # See the script in the following step for how to implement support for other platforms. + # + check-if-this-job-can-be-skipped: + steps: + - run: + name: Check if this job can be skipped + command: | + if [[ $CIRCLE_BRANCH != master ]] && [[ $CIRCLE_BRANCH != release-* ]]; then + scripts/check-ci-job-skippability.js + fi + jobs: nitpick: docker: @@ -557,6 +587,7 @@ jobs: MBGL_ANDROID_STL: << parameters.stl >> steps: - install-dependencies: { gradle: true } + - check-if-this-job-can-be-skipped - run: name: Initialize vendor submodules command: git submodule update --init platform/android/vendor @@ -633,6 +664,7 @@ jobs: IS_LOCAL_DEVELOPMENT: false steps: - install-dependencies: { gradle: true } + - check-if-this-job-can-be-skipped - run: name: Initialize vendor submodules command: git submodule update --init platform/android/vendor @@ -723,6 +755,9 @@ jobs: ANDROID_NDK: /android/sdk/ndk-bundle steps: - checkout + - npm-install + - prepare-environment + - check-if-this-job-can-be-skipped - run: name: Checkout submodules command: | @@ -766,6 +801,7 @@ jobs: WITH_EGL: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-node - save-dependencies - publish-node-package @@ -782,6 +818,7 @@ jobs: - install-macos-dependencies - install-node-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - build-node - save-dependencies - run-node-macos-tests @@ -803,6 +840,7 @@ jobs: WITH_CXX11ABI: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-linux - save-dependencies @@ -824,6 +862,7 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1:color=always:print_summary=1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - setup-llvm-symbolizer - build-test - save-dependencies @@ -846,6 +885,7 @@ jobs: TSAN_OPTIONS: color=always:print_summary=1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - setup-llvm-symbolizer - build-test - save-dependencies @@ -866,6 +906,7 @@ jobs: DISPLAY: :0 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-linux - build-benchmark - build-test @@ -889,6 +930,7 @@ jobs: WITH_COVERAGE: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-linux - build-benchmark - build-test @@ -919,6 +961,7 @@ jobs: WITH_COVERAGE: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - run: name: Install doxygen command: apt update && apt install -y doxygen @@ -943,6 +986,7 @@ jobs: WITH_EGL: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - configure-cmake - build-mbgl-render-test - run-linux-render-tests @@ -960,14 +1004,15 @@ jobs: steps: - install-macos-dependencies - install-dependencies - - build-ios-test - check-public-symbols - run: - name: Check symbol namespacing for mapbox-events-ios - command: make ios-check-events-symbols - - run: name: Lint podspecs and plist files command: make ios-lint + - check-if-this-job-can-be-skipped + - build-ios-test + - run: + name: Check symbol namespacing for mapbox-events-ios + command: make ios-check-events-symbols - run: name: Nitpick Darwin code generation command: scripts/nitpick/generated-code.js darwin @@ -986,14 +1031,15 @@ jobs: steps: - install-macos-dependencies - install-dependencies - - build-ios-test - check-public-symbols - run: - name: Check symbol namespacing for mapbox-events-ios - command: make ios-check-events-symbols - - run: name: Lint podspecs and plist files command: make ios-lint + - check-if-this-job-can-be-skipped + - build-ios-test + - run: + name: Check symbol namespacing for mapbox-events-ios + command: make ios-check-events-symbols - run: name: Nitpick Darwin code generation command: scripts/nitpick/generated-code.js darwin @@ -1002,21 +1048,6 @@ jobs: - upload-xcode-build-logs # ------------------------------------------------------------------------------ - metrics-nightly: - docker: - - image: mbgl/linux-gcc-5:54f59e3ac5 - working_directory: /src - environment: - LIBSYSCONFCPUS: 2 - JOBS: 2 - steps: - - install-dependencies - - run: - name: Collect GitHub statistics - command: | - scripts/publish_github_stats.js - -# ------------------------------------------------------------------------------ ios-sanitize-nightly: macos: xcode: "10.3.0" @@ -1112,6 +1143,7 @@ jobs: steps: - install-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - install-ios-packaging-dependencies - run: name: Build dynamic framework for device and simulator @@ -1210,6 +1242,7 @@ jobs: steps: - install-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - build-macos-test - check-public-symbols - run: @@ -1237,6 +1270,7 @@ jobs: steps: - install-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - configure-cmake - build-mbgl-render-test - save-dependencies @@ -1244,6 +1278,21 @@ jobs: - upload-render-tests # ------------------------------------------------------------------------------ + metrics-nightly: + docker: + - image: mbgl/linux-gcc-5:54f59e3ac5 + working_directory: /src + environment: + LIBSYSCONFCPUS: 2 + JOBS: 2 + steps: + - install-dependencies + - run: + name: Collect GitHub statistics + command: | + scripts/publish_github_stats.js + +# ------------------------------------------------------------------------------ qt5-linux-gcc5-release: docker: - image: mbgl/linux-gcc-5-qt-5.9:5132cfd29f @@ -1256,6 +1305,7 @@ jobs: WITH_QT_I18N: 1 steps: - install-dependencies + - check-if-this-job-can-be-skipped - build-qt-app - build-qt-test - run: @@ -1282,6 +1332,7 @@ jobs: - install-macos-dependencies - install-qt-macos-dependencies - install-dependencies + - check-if-this-job-can-be-skipped - build-qt-app - build-qt-test - run: diff --git a/package.json b/package.json index e239c60864..acd2eb80bf 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "license": "BSD-2-Clause", "dependencies": { "@mapbox/cmake-node-module": "^1.2.0", + "minimatch": "^3.0.4", "node-pre-gyp": "^0.10.2", "npm-run-all": "^4.0.2" }, diff --git a/scripts/check-ci-job-skippability.js b/scripts/check-ci-job-skippability.js new file mode 100755 index 0000000000..55f940db31 --- /dev/null +++ b/scripts/check-ci-job-skippability.js @@ -0,0 +1,121 @@ +#!/usr/bin/env node + +/* +This script takes two parameters or three environment variables: + + {CI job name|$CIRCLE_JOB} {SHA1...SHA1|$CIRCLE_MERGE_BASE...$CIRCLE_SHA1} + +The flow of this script: + + - List of files changed in PR: `git diff --name-only start...end` + - Extract list into array. + - Remove items from array if they match: + - Global ignore list. + - Other platform ignore lists. + - If array still contains items, do not authorize skipping CI job. + - Else if array is empty, skip CI job. + +To add support for a new platform: + + - Add a new PLATFORM_FILES array with a list of unique paths for the + files belonging to the platform. + - Add a case to `skippableFilePatternsForCIJob()` with: + - The prefix for the platform's CI job names. + - A concatenated array of other platform paths that are safe to ignore + for the new platform. These should be files that can change without + having any effect on the output of the new platform. + - Update other cases in `skippableFilePatternsForCIJob()` to include the + paths for the new platform. +*/ + +const execSync = require('child_process').execSync; +const _ = require('lodash'); +const minimatch = require("minimatch"); + +console.step = _.partial(console.log, '\n\033[1m\033[36m*', _, '\033[0m'); + +// BUILD PARAMETERS +const CI_JOB_NAME = process.argv[2] || process.env.CIRCLE_JOB; +if (!CI_JOB_NAME) { console.error('You must specify the name of a CI job as a parameter or environment variable.'); process.exit(1); } + +const COMMIT_RANGE = process.argv[3] || `${process.env.CIRCLE_MERGE_BASE}...${process.env.CIRCLE_SHA1}`; +if (!COMMIT_RANGE || COMMIT_RANGE.includes('undefined')) { console.error('You must specify a range of commits to check for changed files in as a parameter or environment variable.'); process.exit(1); } + +// FILE SKIPPING PATTERNS +const ALWAYS_SKIPPABLE_FILES = [ + '**/CHANGELOG.md', + '**/README.md', + '**/CONTRIBUTING.md', + '**/DEVELOPING.md', + '**/INSTALL.md', + '**/*.podspec' +] + +const ANDROID_FILES = [ + 'platform/android/**' +] + +const DARWIN_FILES = [ + 'platform/darwin/**' +] + +const IOS_FILES = [ + 'platform/ios/**' +] + +const MACOS_FILES = [ + 'platform/macos/**' +] + +const OTHER_PLATFORMS = [ + 'platform/glfw/**', + 'platform/linux/**', + 'platform/node/**', + 'platform/qt/**', + 'benchmark/**', + 'test/**' +] + +function skippableFilePatternsForCIJob(job) { + if (job.includes('render-tests')) { + // Render tests run on specific platforms are effectively unskippable core jobs, not platform jobs. + return ALWAYS_SKIPPABLE_FILES; + } else if (job.startsWith('ios')) { + return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, MACOS_FILES, OTHER_PLATFORMS); + } else if (job.startsWith('android')) { + return _.concat(ALWAYS_SKIPPABLE_FILES, DARWIN_FILES, IOS_FILES, MACOS_FILES, OTHER_PLATFORMS); + } else if (job.startsWith('macos')) { + return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, IOS_FILES, OTHER_PLATFORMS); + } else { + return ALWAYS_SKIPPABLE_FILES; + } +} + +console.step(`Getting list of files changed between ${COMMIT_RANGE}`); + +let changedFiles = execSync(`git diff --name-only ${COMMIT_RANGE}`).toString().split('\n'); +changedFiles = _.compact(changedFiles); + +console.log(`${changedFiles.length} total files changed.`); + +console.step(`Checking changed files for paths relevant to ${CI_JOB_NAME} job`); + +// Filter the changed files array to remove files that are irrelevant to the specified CI job. +_.remove(changedFiles, function(changedFile) { + for (const skippableFilePattern of skippableFilePatternsForCIJob(CI_JOB_NAME)) { + if (minimatch(changedFile, skippableFilePattern, { dot: true, nocase: true })) { + return true; + } + } + return false; +}); + +if (changedFiles.length) { + console.log(`Found ${changedFiles.length} unskippable changed file${changedFiles.length > 1 ? 's':''}:\n`, changedFiles); +} else { + console.log(`Found no relevant changed files, so it's safe to skip the remainder of this CI job.`) + if (process.env.CIRCLECI) { + console.step('Aborting CI job'); + execSync(`circleci step halt`, {stdio: 'inherit'}); + } +} |