[build] Skip CI jobs if there are no relevant changed files

3 files changed, 156 insertions, 19 deletions

diff --git a/circle.yml b/circle.yml
index e7d50421c4..0e3feda38b 100644
--- a/circle.yml
+++ b/circle.yml
@@ -465,6 +465,15 @@ commands:
             scripts/notify-slack.sh
           fi
 
+  check-if-this-job-can-be-skipped:
+    steps:
+    - run:
+        name: Check if this job can be skipped
+        command: |
+          if [[ $CIRCLE_BRANCH != master ]] && [[ $CIRCLE_BRANCH != release-* ]]; then
+            scripts/check-ci-job-skippability.js
+          fi
+
 jobs:
   nitpick:
     docker:
@@ -555,6 +564,7 @@ jobs:
       MBGL_ANDROID_STL: << parameters.stl >>
     steps:
       - install-dependencies: { gradle: true }
+      - check-if-this-job-can-be-skipped
       - run:
           name: Initialize vendor submodules
           command: git submodule update --init platform/android/vendor
@@ -631,6 +641,7 @@ jobs:
       IS_LOCAL_DEVELOPMENT: false
     steps:
       - install-dependencies: { gradle: true }
+      - check-if-this-job-can-be-skipped
       - run:
           name: Initialize vendor submodules
           command: git submodule update --init platform/android/vendor
@@ -721,6 +732,9 @@ jobs:
       ANDROID_NDK: /android/sdk/ndk-bundle
     steps:
       - checkout
+      - npm-install
+      - prepare-environment
+      - check-if-this-job-can-be-skipped
       - run:
           name: Checkout submodules
           command: |
@@ -764,6 +778,7 @@ jobs:
       WITH_EGL: 1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-node
       - save-dependencies
       - publish-node-package
@@ -780,6 +795,7 @@ jobs:
       - install-macos-dependencies
       - install-node-macos-dependencies
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-node
       - save-dependencies
       - run-node-macos-tests
@@ -801,6 +817,7 @@ jobs:
       WITH_CXX11ABI: 1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-linux
       - save-dependencies
 
@@ -822,6 +839,7 @@ jobs:
       UBSAN_OPTIONS: print_stacktrace=1:color=always:print_summary=1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - setup-llvm-symbolizer
       - build-test
       - save-dependencies
@@ -844,6 +862,7 @@ jobs:
       TSAN_OPTIONS: color=always:print_summary=1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - setup-llvm-symbolizer
       - build-test
       - save-dependencies
@@ -864,6 +883,7 @@ jobs:
       DISPLAY: :0
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-linux
       - build-benchmark
       - build-test
@@ -887,6 +907,7 @@ jobs:
       WITH_COVERAGE: 1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-linux
       - build-benchmark
       - build-test
@@ -917,6 +938,7 @@ jobs:
       WITH_COVERAGE: 1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - run:
           name: Install doxygen
           command: apt update && apt install -y doxygen
@@ -941,6 +963,7 @@ jobs:
       WITH_EGL: 1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - configure-cmake
       - build-mbgl-render-test
       - run-linux-render-tests
@@ -958,14 +981,15 @@ jobs:
     steps:
       - install-macos-dependencies
       - install-dependencies
-      - build-ios-test
       - check-public-symbols
       - run:
-          name: Check symbol namespacing for mapbox-events-ios
-          command: make ios-check-events-symbols
-      - run:
           name: Lint podspecs and plist files
           command: make ios-lint
+      - check-if-this-job-can-be-skipped
+      - build-ios-test
+      - run:
+          name: Check symbol namespacing for mapbox-events-ios
+          command: make ios-check-events-symbols
       - run:
           name: Nitpick Darwin code generation
           command: scripts/nitpick/generated-code.js darwin
@@ -974,21 +998,6 @@ jobs:
       - upload-xcode-build-logs
 
 # ------------------------------------------------------------------------------
-  metrics-nightly:
-    docker:
-      - image: mbgl/linux-gcc-5:54f59e3ac5
-    working_directory: /src
-    environment:
-      LIBSYSCONFCPUS: 2
-      JOBS: 2
-    steps:
-      - install-dependencies
-      - run:
-          name: Collect GitHub statistics
-          command: |
-              scripts/publish_github_stats.js
-
-# ------------------------------------------------------------------------------
   ios-sanitize-nightly:
     macos:
       xcode: "10.3.0"
@@ -1064,6 +1073,7 @@ jobs:
     steps:
       - install-macos-dependencies
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - install-ios-packaging-dependencies
       - run:
           name: Build dynamic framework for device and simulator
@@ -1162,6 +1172,7 @@ jobs:
     steps:
       - install-macos-dependencies
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-macos-test
       - check-public-symbols
       - run:
@@ -1189,6 +1200,7 @@ jobs:
     steps:
       - install-macos-dependencies
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - configure-cmake
       - build-mbgl-render-test
       - save-dependencies
@@ -1196,6 +1208,21 @@ jobs:
       - upload-render-tests
 
 # ------------------------------------------------------------------------------
+  metrics-nightly:
+    docker:
+      - image: mbgl/linux-gcc-5:54f59e3ac5
+    working_directory: /src
+    environment:
+      LIBSYSCONFCPUS: 2
+      JOBS: 2
+    steps:
+      - install-dependencies
+      - run:
+          name: Collect GitHub statistics
+          command: |
+              scripts/publish_github_stats.js
+
+# ------------------------------------------------------------------------------
   qt5-linux-gcc5-release:
     docker:
       - image: mbgl/linux-gcc-5-qt-5.9:5132cfd29f
@@ -1208,6 +1235,7 @@ jobs:
       WITH_QT_I18N: 1
     steps:
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-qt-app
       - build-qt-test
       - run:
@@ -1234,6 +1262,7 @@ jobs:
       - install-macos-dependencies
       - install-qt-macos-dependencies
       - install-dependencies
+      - check-if-this-job-can-be-skipped
       - build-qt-app
       - build-qt-test
       - run:
diff --git a/package.json b/package.json
index e239c60864..acd2eb80bf 100644
--- a/package.json
+++ b/package.json
@@ -14,6 +14,7 @@
   "license": "BSD-2-Clause",
   "dependencies": {
     "@mapbox/cmake-node-module": "^1.2.0",
+    "minimatch": "^3.0.4",
     "node-pre-gyp": "^0.10.2",
     "npm-run-all": "^4.0.2"
   },
diff --git a/scripts/check-ci-job-skippability.js b/scripts/check-ci-job-skippability.js
new file mode 100755
index 0000000000..bac24148d5
--- /dev/null
+++ b/scripts/check-ci-job-skippability.js
@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+
+/*
+
+This script takes two parameters or three environment variables:
+
+  {CI job name|$CIRCLE_JOB} {SHA1...SHA1|$CIRCLE_MERGE_BASE...$CIRCLE_SHA1}
+
+The flow of this script:
+
+  - List of files changed in PR: `git diff --name-only start...end`
+  - Extract list into array.
+  - Remove items from array if they match:
+    - Global ignore list.
+    - Other platform ignore lists.
+  - If array still contains items, do not authorize skipping CI job.
+  - Else if array is empty, skip CI job.
+
+*/
+
+const execSync = require('child_process').execSync;
+const _ = require('lodash');
+const minimatch = require("minimatch");
+
+console.step = _.partial(console.log, '\n\033[1m\033[36m*', _, '\033[0m');
+
+// BUILD PARAMETERS
+const CI_JOB_NAME = process.argv[2] || process.env.CIRCLE_JOB;
+if (!CI_JOB_NAME) { console.error('You must specify the name of a CI job as a parameter or environment variable.'); process.exit(1); }
+
+const COMMIT_RANGE = process.argv[3] || `${process.env.CIRCLE_MERGE_BASE}...${process.env.CIRCLE_SHA1}`;
+if (!COMMIT_RANGE || COMMIT_RANGE.includes('undefined')) { console.error('You must specify a range of commits to check for changed files in as a parameter or environment variable.'); process.exit(1); }
+
+// FILE SKIPPING PATTERNS
+const ALWAYS_SKIPPABLE_FILES = [
+  '**/CHANGELOG.md',
+  '**/README.md',
+  '**/CONTRIBUTING.md',
+  '**/DEVELOPING.md',
+  '**/INSTALL.md',
+  '**/*.podspec'
+]
+
+const ANDROID_FILES = [
+  'platform/android/**'
+]
+
+const IOS_FILES = [
+  'platform/ios/**',
+  'platform/darwin/**'
+]
+
+const MACOS_FILES = [
+  'platform/macos/**',
+  'platform/darwin/**'
+]
+
+const OTHER_PLATFORMS = [
+  'platform/glfw/**',
+  'platform/linux/**',
+  'platform/node/**',
+  'platform/qt/**',
+  'benchmark/**',
+  'test/**'
+]
+
+function skippableFilePatternsForCIJob(job) {
+  if (job.startsWith('ios')) {
+      return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, MACOS_FILES, OTHER_PLATFORMS);
+  } else if (job.startsWith('android')) {
+      return _.concat(ALWAYS_SKIPPABLE_FILES, IOS_FILES, MACOS_FILES, OTHER_PLATFORMS);
+  } else if (job.startsWith('macos')) {
+      return _.concat(ALWAYS_SKIPPABLE_FILES, ANDROID_FILES, IOS_FILES, OTHER_PLATFORMS);
+  } else {
+      return ALWAYS_SKIPPABLE_FILES;
+  }
+}
+
+console.step(`Getting list of files changed between ${COMMIT_RANGE}`);
+
+let changedFiles = execSync(`git diff --name-only ${COMMIT_RANGE}`)
+  .toString()
+  .trim()
+  .split('\n');
+changedFiles = _.compact(changedFiles);
+
+console.log(`${changedFiles.length} total files changed.`);
+
+// Filter the changed files array to remove files that are irrelevant to the specified CI job.
+_.remove(changedFiles, function(changedFile) {
+  for (const skippableFilePattern of skippableFilePatternsForCIJob(CI_JOB_NAME)) {
+    if (minimatch(changedFile, skippableFilePattern, { dot: true, nocase: true })) {
+      return true;
+    }
+  }
+  return false;
+});
+
+if (changedFiles.length) {
+  console.log(`Found ${changedFiles.length} unskippable changed file${changedFiles.length > 1 ? 's':''} for ${CI_JOB_NAME}:\n`, changedFiles);
+} else {
+  console.log(`Found no relevant changed files, so it's safe to skip the remainder of this CI job.`)
+  if (process.env.CIRCLECI) {
+    console.step('Aborting CI job');
+    execSync(`circleci step halt`, {stdio: 'inherit'});
+  }
+}