blob: 0637dc1898eee541efc0ee416d4331a8c7f72582 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
|
#!/bin/bash
# Usage:
# check_benchmark_results.sh <baseline> <results> <comparison report in txt> <comparison report in html>
# Benchmark comparison script uses a hard-coded 5% threshold by default, but it causes random failures
# Increase the threshold to 15%
if [ "$(uname)" == "Darwin" ]; then
sed -i '' s/"if res > 0.05"/"if res > 0.15"/ $PWD/vendor/benchmark/tools/gbench/report.py
elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
sed -i s/"if res > 0.05"/"if res > 0.15"/ $PWD/vendor/benchmark/tools/gbench/report.py
fi
# Run the bencmark comparison
vendor/benchmark/tools/compare.py benchmarks $1 $2 > $3
# Find cases where the average is above threshold
tests=`fgrep mean $3 | cut -c1-85 | grep 91m`
if [ -z "$tests" ]
then
echo "" >> $3
echo "PASSED, all benchmarks within 15% threshold" >> $3
ansi2html < $3 > $4
exit 0
else
test_names=`fgrep mean $3 | cut -c1-85 | grep 91m | cut -f1 -d' '`
echo "" >> $3
echo "FAILED, following benchmarks are not within 15% threshold:" >> $3
echo "" >> $3
for t in $test_names;
do
echo $t >> $3
done
ansi2html < $3 > $4
exit 1
fi
|
