From 3a91597c5d218235d57b3c822374a572d0339d41 Mon Sep 17 00:00:00 2001
From: Juha Alanen <juha.alanen@mapbox.com>
Date: Sat, 11 Apr 2020 16:15:42 +0300
Subject: [benchmark] Add script for comparing benchmark results to baseline

Benchmark will fail if the average time is more than 5% higher
than the baseline.
---
 scripts/check_benchmark_results.sh | 41 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100755 scripts/check_benchmark_results.sh

(limited to 'scripts')

diff --git a/scripts/check_benchmark_results.sh b/scripts/check_benchmark_results.sh
new file mode 100755
index 0000000000..9ea1ca2c1e
--- /dev/null
+++ b/scripts/check_benchmark_results.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Usage:
+# check_benchmark_results.sh <baseline> <results> <comparison report in txt> <comparison report in html>
+
+# Benchmark comparison script uses a hard-coded 5% threshold by default, but it causes random failures
+# Increase the threshold to 10%
+
+if [ "$(uname)" == "Darwin" ]; then
+  sed -i '' s/"if res > 0.05"/"if res > 0.1"/ $PWD/vendor/benchmark/tools/gbench/report.py
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+  sed -i s/"if res > 0.05"/"if res > 0.1"/ $PWD/vendor/benchmark/tools/gbench/report.py
+fi
+
+# Run the bencmark comparison
+vendor/benchmark/tools/compare.py benchmarks $1 $2 > $3
+
+# Find cases where the average is above threshold
+tests=`fgrep mean $3 | cut -c1-85 | grep 91m`
+
+if [ -z "$tests" ]
+then
+      echo "" >> $3
+      echo "PASSED, all benchmarks within 10% threshold" >> $3
+
+      ansi2html < $3 > $4
+      exit 0
+else
+      test_names=`fgrep mean $3 | cut -c1-85 | grep 91m | cut -f1 -d' '`
+
+      echo "" >> $3
+      echo "FAILED, following benchmarks are not within 10% threshold:" >> $3
+      echo "" >> $3
+      for t in $test_names;
+      do
+        echo $t >> $3
+      done
+
+      ansi2html < $3 > $4
+      exit 1
+fi
-- 
cgit v1.2.1