Android retry

1 files changed, 52 insertions, 37 deletions

diff --git a/circle.yml b/circle.yml
index 896c167622..7c435f01df 100644
--- a/circle.yml
+++ b/circle.yml
@@ -276,12 +276,15 @@ commands:
               if ls benchmark-app/TestLogs/*.json 1> /dev/null 2>&1; then cp benchmark-app/TestLogs/*.json /tmp/tests/benchmark/results.json; fi
             fi
   check-benchmark-results:
+    parameters:
+      directory:
+        type: string
     steps:
       - run:
           name: Compare result to baseline
           when: on_success
           command: |
-            scripts/check_benchmark_results.sh metrics/benchmark/ios/results.json /tmp/tests/benchmark/results.json /tmp/tests/benchmark/results_comparison.txt /tmp/tests/benchmark/results_comparison.html
+            scripts/check_benchmark_results.sh metrics/benchmark/<< parameters.directory >>/results.json /tmp/tests/benchmark/results.json /tmp/tests/benchmark/results_comparison.txt /tmp/tests/benchmark/results_comparison.html
 
   retry-ios-benchmark:
     steps:
@@ -314,6 +317,48 @@ commands:
             fi
             scripts/check_benchmark_results.sh metrics/benchmark/ios/results.json /tmp/tests/benchmark/results.json /tmp/tests/benchmark/results_comparison.txt /tmp/tests/benchmark/results_comparison.html
 
+  run-android-benchmark:
+    steps:
+      - run:
+          name: Run BenchmarkRunner on Firebase
+          no_output_timeout: 30m
+          command: |
+            gcloud firebase test android models list
+            gcloud firebase test android run --type instrumentation \
+              --app benchmark/android/app/build/outputs/apk/release/app-release.apk \
+              --test benchmark/android/app/build/outputs/apk/androidTest/release/app-release-androidTest.apk \
+              --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 30m \
+              --directories-to-pull /sdcard/benchmark/results --results-dir benchmark-${CIRCLE_BUILD_NUM} \
+              --no-record-video --no-performance-metrics
+
+  get-android-benchmark-results:
+    steps:
+      - run:
+          name: Retrieve TestResult from gcloud
+          when: always
+          command: |
+            testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/benchmark-${CIRCLE_BUILD_NUM}/**/results/results.json)
+            mkdir -p /tmp/tests/benchmark
+            gsutil cp $testResult /tmp/tests/benchmark
+
+  retry-android-benchmark:
+    steps:
+      - run:
+          when: on_fail
+          no_output_timeout: 30m
+          name: Retry Android benchmark on failure
+          command: |
+            echo "Benchmark failed, retrying"
+            gcloud firebase test android models list
+            gcloud firebase test android run --type instrumentation \
+              --app benchmark/android/app/build/outputs/apk/release/app-release.apk \
+              --test benchmark/android/app/build/outputs/apk/androidTest/release/app-release-androidTest.apk \
+              --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 30m \
+              --directories-to-pull /sdcard/benchmark/results --results-dir benchmark-${CIRCLE_BUILD_NUM} \
+              --no-record-video --no-performance-metrics
+            scripts/check_benchmark_results.sh metrics/benchmark/android/results.json /tmp/tests/benchmark/results.json /tmp/tests/benchmark/results_comparison.txt /tmp/tests/benchmark/results_comparison.html
+
+
   upload-coverage-results:
     steps:
     - run:
@@ -437,6 +482,7 @@ jobs:
           directory: benchmark/ios
       - get-ios-benchmark-results
       - check-benchmark-results
+          directory: ios
       - retry-ios-benchmark
       - save
   android-benchmark-runner:
@@ -450,42 +496,11 @@ jobs:
             cd benchmark/android
             ./gradlew --parallel --max-workers=8 -Pmapbox.abis=arm64-v8a app:assembleRelease app:assembleAndroidTest
       - login-google-cloud-platform
-      - run:
-          name: Run BenchmarkRunner on Firebase
-          no_output_timeout: 30m
-          command: |
-            gcloud firebase test android models list
-            gcloud firebase test android run --type instrumentation \
-              --app benchmark/android/app/build/outputs/apk/release/app-release.apk \
-              --test benchmark/android/app/build/outputs/apk/androidTest/release/app-release-androidTest.apk \
-              --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 30m \
-              --directories-to-pull /sdcard/benchmark/results --results-dir benchmark-${CIRCLE_BUILD_NUM} \
-              --no-record-video --no-performance-metrics
-      - run:
-          name: Retrieve TestResult from gcloud
-          when: always
-          command: |
-            testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/benchmark-${CIRCLE_BUILD_NUM}/**/results/results.json)
-            mkdir -p /tmp/tests/benchmark
-            gsutil cp $testResult /tmp/tests/benchmark
-      - run:
-          name: Compare result to baseline
-          when: always
-          command: |
-            scripts/check_benchmark_results.sh metrics/benchmark/android/results.json /tmp/tests/benchmark/results.json /tmp/tests/benchmark/results_comparison.txt /tmp/tests/benchmark/results_comparison.html
-      - run:
-          name: Re-run benchmark if first attempt failed
-          when: on_fail
-          no_output_timeout: 30m
-          command: |
-            gcloud firebase test android models list
-            gcloud firebase test android run --type instrumentation \
-              --app benchmark/android/app/build/outputs/apk/release/app-release.apk \
-              --test benchmark/android/app/build/outputs/apk/androidTest/release/app-release-androidTest.apk \
-              --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 30m \
-              --directories-to-pull /sdcard/benchmark/results --results-dir benchmark-${CIRCLE_BUILD_NUM} \
-              --no-record-video --no-performance-metrics
-            scripts/check_benchmark_results.sh metrics/benchmark/android/results.json /tmp/tests/benchmark/results.json /tmp/tests/benchmark/results_comparison.txt /tmp/tests/benchmark/results_comparison.html
+      - run-android-benchmark
+      - get-android-benchmark-results
+      - check-benchmark-results
+          directory: android
+      - retry-android-benchmark
       - save
   android-api-breakage:
     executor: ubuntu-eoan