diff options
author | Juha Alanen <juha.alanen@mapbox.com> | 2020-04-11 16:15:42 +0300 |
---|---|---|
committer | Juha Alanen <juha.alanen@mapbox.com> | 2020-04-15 19:11:28 +0300 |
commit | 3a91597c5d218235d57b3c822374a572d0339d41 (patch) | |
tree | fb03e6d94431002bbea91deecf8d0fdc9444009d | |
parent | 36175d57b4a54e3127a4ac26d31267ef1494b88d (diff) | |
download | qtlocation-mapboxgl-3a91597c5d218235d57b3c822374a572d0339d41.tar.gz |
[benchmark] Add script for comparing benchmark results to baseline
Benchmark will fail if the average time is more than 5% higher
than the baseline.
-rwxr-xr-x | scripts/check_benchmark_results.sh | 41 |
1 files changed, 41 insertions, 0 deletions
diff --git a/scripts/check_benchmark_results.sh b/scripts/check_benchmark_results.sh new file mode 100755 index 0000000000..9ea1ca2c1e --- /dev/null +++ b/scripts/check_benchmark_results.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Usage: +# check_benchmark_results.sh <baseline> <results> <comparison report in txt> <comparison report in html> + +# Benchmark comparison script uses a hard-coded 5% threshold by default, but it causes random failures +# Increase the threshold to 10% + +if [ "$(uname)" == "Darwin" ]; then + sed -i '' s/"if res > 0.05"/"if res > 0.1"/ $PWD/vendor/benchmark/tools/gbench/report.py +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + sed -i s/"if res > 0.05"/"if res > 0.1"/ $PWD/vendor/benchmark/tools/gbench/report.py +fi + +# Run the bencmark comparison +vendor/benchmark/tools/compare.py benchmarks $1 $2 > $3 + +# Find cases where the average is above threshold +tests=`fgrep mean $3 | cut -c1-85 | grep 91m` + +if [ -z "$tests" ] +then + echo "" >> $3 + echo "PASSED, all benchmarks within 10% threshold" >> $3 + + ansi2html < $3 > $4 + exit 0 +else + test_names=`fgrep mean $3 | cut -c1-85 | grep 91m | cut -f1 -d' '` + + echo "" >> $3 + echo "FAILED, following benchmarks are not within 10% threshold:" >> $3 + echo "" >> $3 + for t in $test_names; + do + echo $t >> $3 + done + + ansi2html < $3 > $4 + exit 1 +fi |