From 74ef93ab4f5e88d2d111d6400bc8d80642075544 Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Thu, 5 Jul 2018 13:08:28 -0700 Subject: [build] Improve ccache freshness Save a fresh ccache on every build by including (a checksum of) the current revision in the cache key. In the worst case, saving the cache appears to take ~1:30 for android-release; other builds appear to take a small amount of time compared to the time saved by having a fresh cache, versus a cache that's up to a week old. Restore caches in the following priority order (all scoped to current architecture and job): 1. The cache for the current branch and revision. 2. If that doesn't exist, the most recent cache for the current branch. 3. If that doesn't exist, the cache for the parent branch and merge base, where "parent branch" means the base branch of the pull request specified by $CIRCLE_PULL_REQUEST (obtained from the GitHub API), and "merge base" means the revision returned by `git merge-base` when given the parent branch and $CIRCLE_BRANCH (obtained by executing that shell command). The intent here is to start a newly created branch off with the most relevant cache from its parent branch. 4. If that doesn't exist, the most recent cache for the parent branch. 5. If that doesn't exist, the most recent cache for master. 6. If that doesn't exist, use the most recent cache for any build. We cannot use the branch names or revisions themselves as key segments, because CircleCI lacks a way to interpolate arbitrary environment variables into the cache key name. Instead, we write the desired value into a file and use the `{{ checksum "filename" }}` syntax. --- circle.yml | 124 +++++++++++++++++++++++++++++++++++++------------ scripts/environment.js | 24 ++++++++++ 2 files changed, 118 insertions(+), 30 deletions(-) create mode 100755 scripts/environment.js diff --git a/circle.yml b/circle.yml index 351889e4b6..b99aa4efbe 100644 --- a/circle.yml +++ b/circle.yml @@ -59,11 +59,29 @@ workflows: - ios-release step-library: + - &npm-install + run: + name: npm install + command: npm install --ignore-scripts + - &prepare-environment + run: + name: Prepare environment + command: touch "$BASH_ENV" && ./scripts/environment.js | tee -a "$BASH_ENV" - &prepare-ccache run: name: Prepare ccache command: | - echo "$(date +"%Y-%V")" > .circle-week + # CircleCI doesn't have any way to embed arbitrary environment variables or command output + # into cache keys, so we're using the workaround of writing the desired content to a file, + # and then using `{{ checksum "filename" }}` in the cache key. + echo "$CIRCLE_BRANCH" + echo "$CIRCLE_BRANCH" > .circle-branch + echo "$CIRCLE_SHA1" + echo "$CIRCLE_SHA1" > .circle-sha1 + echo "$CIRCLE_TARGET_BRANCH" + echo "${CIRCLE_TARGET_BRANCH:master}" > .circle-target-branch + echo "$CIRCLE_MERGE_BASE" + echo "${CIRCLE_MERGE_BASE:master}" > .circle-merge-base ccache --clear ccache --max-size=5G - &reset-ccache-stats @@ -103,14 +121,17 @@ step-library: - &save-ccache save_cache: name: Save ccache - key: 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ .Branch }}/{{ checksum ".circle-week" }}' - paths: [ "/root/.ccache" ] + key: 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ checksum ".circle-branch" }}/{{ checksum ".circle-sha1" }}' + paths: [ "~/.ccache" ] - &restore-ccache restore_cache: name: Restore ccache keys: - - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ .Branch }}/{{ checksum ".circle-week" }}' - - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/master/{{ checksum ".circle-week" }}' + - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ checksum ".circle-branch" }}/{{ checksum ".circle-sha1" }}' + - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ checksum ".circle-branch" }}' + - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ checksum ".circle-target-branch" }}/{{ checksum ".circle-merge-base" }}' + - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}/{{ checksum ".circle-target-branch" }}' + - 'ccache/v1/{{ arch }}/{{ .Environment.CIRCLE_JOB }}' - &save-gradle-cache save_cache: name: Save gradle cache @@ -276,12 +297,11 @@ jobs: steps: - checkout - *restore-node_modules-cache + - *npm-install + - *prepare-environment - run: name: Initialize submodule command: git submodule update --init mapbox-gl-js - - run: - name: npm install - command: npm install --ignore-scripts - run: name: Verify submodule pin command: scripts/nitpick/submodule-pin.js @@ -315,8 +335,10 @@ jobs: BUILDTYPE: Debug steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -346,8 +368,10 @@ jobs: IS_LOCAL_DEVELOPMENT: false steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *restore-gradle-cache @@ -423,8 +447,10 @@ jobs: IS_LOCAL_DEVELOPMENT: false steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *restore-gradle-cache @@ -482,8 +508,10 @@ jobs: WITH_CXX11ABI: 0 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -509,8 +537,10 @@ jobs: WITH_EGL: 1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -532,10 +562,12 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *install-node-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -563,8 +595,10 @@ jobs: WITH_CXX11ABI: 1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -590,8 +624,10 @@ jobs: ASAN_OPTIONS: detect_leaks=0:color=always:print_summary=1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -624,8 +660,10 @@ jobs: UBSAN_OPTIONS: print_stacktrace=1:color=always:print_summary=1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -658,8 +696,10 @@ jobs: TSAN_OPTIONS: color=always:print_summary=1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -691,8 +731,10 @@ jobs: DISPLAY: :0 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -719,8 +761,10 @@ jobs: WITH_COVERAGE: 1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -746,9 +790,11 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -777,9 +823,11 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -802,9 +850,11 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -827,9 +877,11 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -852,10 +904,12 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *install-ios-packaging-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -887,10 +941,12 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *install-ios-packaging-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -916,9 +972,11 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -954,8 +1012,10 @@ jobs: LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libjemalloc.so steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -985,8 +1045,10 @@ jobs: WITH_QT_I18N: 1 steps: - checkout - - *prepare-ccache - *restore-node_modules-cache + - *npm-install + - *prepare-environment + - *prepare-ccache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats @@ -1016,10 +1078,12 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 steps: - checkout + - *restore-node_modules-cache + - *npm-install + - *prepare-environment - *install-macos-dependencies - *install-qt-macos-dependencies - *prepare-ccache - - *restore-node_modules-cache - *restore-mason_packages-cache - *restore-ccache - *reset-ccache-stats diff --git a/scripts/environment.js b/scripts/environment.js new file mode 100755 index 0000000000..b071a484e7 --- /dev/null +++ b/scripts/environment.js @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +// Output some `export` commands with a few extra environment variables. They'll be evaluated +// into the build environment and available for use in later steps. + +const github = require('@octokit/rest')(); +const {execSync} = require('child_process'); + +const pr = process.env['CIRCLE_PULL_REQUEST']; +if (pr) { + const number = +pr.match(/\/(\d+)\/?$/)[1]; + return github.pullRequests.get({ + owner: 'mapbox', + repo: 'mapbox-gl-native', + number + }).then(({data}) => { + const base = data.base.ref; + const head = process.env['CIRCLE_SHA1']; + const mergeBase = execSync(`git merge-base origin/${base} ${head}`).toString().trim(); + + console.log(`export CIRCLE_TARGET_BRANCH=${base}`); + console.log(`export CIRCLE_MERGE_BASE=${mergeBase}`); + }); +} -- cgit v1.2.1