diff options
| -rwxr-xr-x | scripts/check_benchmark_results.sh | 41 |
1 files changed, 41 insertions, 0 deletions
diff --git a/scripts/check_benchmark_results.sh b/scripts/check_benchmark_results.sh new file mode 100755 index 0000000000..9ea1ca2c1e --- /dev/null +++ b/scripts/check_benchmark_results.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Usage: +# check_benchmark_results.sh <baseline> <results> <comparison report in txt> <comparison report in html> + +# Benchmark comparison script uses a hard-coded 5% threshold by default, but it causes random failures +# Increase the threshold to 10% + +if [ "$(uname)" == "Darwin" ]; then + sed -i '' s/"if res > 0.05"/"if res > 0.1"/ $PWD/vendor/benchmark/tools/gbench/report.py +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + sed -i s/"if res > 0.05"/"if res > 0.1"/ $PWD/vendor/benchmark/tools/gbench/report.py +fi + +# Run the bencmark comparison +vendor/benchmark/tools/compare.py benchmarks $1 $2 > $3 + +# Find cases where the average is above threshold +tests=`fgrep mean $3 | cut -c1-85 | grep 91m` + +if [ -z "$tests" ] +then + echo "" >> $3 + echo "PASSED, all benchmarks within 10% threshold" >> $3 + + ansi2html < $3 > $4 + exit 0 +else + test_names=`fgrep mean $3 | cut -c1-85 | grep 91m | cut -f1 -d' '` + + echo "" >> $3 + echo "FAILED, following benchmarks are not within 10% threshold:" >> $3 + echo "" >> $3 + for t in $test_names; + do + echo $t >> $3 + done + + ansi2html < $3 > $4 + exit 1 +fi |