[build] Collect and publish new baselines

Also fail the bot in case there are new baselines.

28 files changed, 110 insertions, 116 deletions

diff --git a/metrics/binary-size.json b/metrics/binary-size.json
deleted file mode 100644
index f874991a2e..0000000000
--- a/metrics/binary-size.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "base_test_path":"tests/binary-size"
-}
diff --git a/metrics/ignores/platform-all.json b/metrics/ignores/platform-all.json
index e1cf323afa..c89b14ef95 100644
--- a/metrics/ignores/platform-all.json
+++ b/metrics/ignores/platform-all.json
@@ -1,12 +1,4 @@
 {
-    "probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing",
-    "binary-size/android-arm64-v8a": "skip - only makes sense for the binary size bot.",
-    "binary-size/android-armeabi-v7a": "skip - only makes sense for the binary size bot.",
-    "binary-size/android-x86": "skip - only makes sense for the binary size bot.",
-    "binary-size/android-x86_64": "skip - only makes sense for the binary size bot.",
-    "binary-size/linux-clang8": "skip - only makes sense for the binary size bot.",
-    "binary-size/linux-gcc8": "skip - only makes sense for the binary size bot.",
-    "binary-size/macos-xcode11": "skip - only makes sense for the binary size bot.",
     "expression-tests/collator/accent-equals-de": "Locale-specific behavior changes based on platform.",
     "expression-tests/in/assert-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
     "expression-tests/in/assert-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893",
@@ -43,6 +35,7 @@
     "probes/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.",
     "probes/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.",
     "probes/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.",
+    "probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing",
     "probes/network/fail-requests": "Should fail, number of requests higher than expected.",
     "probes/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.",
     "probes/network/fail-transferred": "Should fail, amount of transferred data higher than expected.",
diff --git a/metrics/next-binary-size.json b/metrics/next-binary-size.json
new file mode 100644
index 0000000000..05f9d2a7fe
--- /dev/null
+++ b/metrics/next-binary-size.json
@@ -0,0 +1,4 @@
+{
+    "base_test_path": "next-binary-size",
+    "metric_path": "next-binary-size"
+}
diff --git a/metrics/tests/binary-size/android-arm64-v8a/expected.png b/metrics/next-binary-size/android-arm64-v8a/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/android-arm64-v8a/expected.png
+++ b/metrics/next-binary-size/android-arm64-v8a/expected.png
diff --git a/metrics/tests/binary-size/android-arm64-v8a/metrics.json b/metrics/next-binary-size/android-arm64-v8a/metrics.json
index 960fa41232..5529b66e35 100644
--- a/metrics/tests/binary-size/android-arm64-v8a/metrics.json
+++ b/metrics/next-binary-size/android-arm64-v8a/metrics.json
@@ -2,7 +2,7 @@
     "file-size": [
         [
             "android-arm64-v8a",
-            "/src/workspace/next-android-arm64-v8a-release/lib/libmapbox-gl.so",
+            "/tmp/attach/install/next-android-arm64-v8a-release/lib/libmapbox-gl.so",
             1907875
         ]
     ]
diff --git a/metrics/tests/binary-size/android-armeabi-v7a/style.json b/metrics/next-binary-size/android-arm64-v8a/style.json
index 942d198bc0..77b7d38860 100644
--- a/metrics/tests/binary-size/android-armeabi-v7a/style.json
+++ b/metrics/next-binary-size/android-arm64-v8a/style.json
@@ -3,7 +3,7 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "android-armeabi-v7a", "/src/workspace/next-android-armeabi-v7a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
+        ["probeFileSize", "android-arm64-v8a", "/tmp/attach/install/next-android-arm64-v8a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
       ],
       "width": 64,
       "height": 64
diff --git a/metrics/tests/binary-size/android-armeabi-v7a/expected.png b/metrics/next-binary-size/android-armeabi-v7a/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/android-armeabi-v7a/expected.png
+++ b/metrics/next-binary-size/android-armeabi-v7a/expected.png
diff --git a/metrics/tests/binary-size/android-armeabi-v7a/metrics.json b/metrics/next-binary-size/android-armeabi-v7a/metrics.json
index ab110e706a..e8b686180a 100644
--- a/metrics/tests/binary-size/android-armeabi-v7a/metrics.json
+++ b/metrics/next-binary-size/android-armeabi-v7a/metrics.json
@@ -2,7 +2,7 @@
     "file-size": [
         [
             "android-armeabi-v7a",
-            "/src/workspace/next-android-armeabi-v7a-release/lib/libmapbox-gl.so",
+            "/tmp/attach/install/next-android-armeabi-v7a-release/lib/libmapbox-gl.so",
             1607847
         ]
     ]
diff --git a/metrics/tests/binary-size/android-arm64-v8a/style.json b/metrics/next-binary-size/android-armeabi-v7a/style.json
index 57839a6d18..1c737724f2 100644
--- a/metrics/tests/binary-size/android-arm64-v8a/style.json
+++ b/metrics/next-binary-size/android-armeabi-v7a/style.json
@@ -3,7 +3,7 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "android-arm64-v8a", "/src/workspace/next-android-arm64-v8a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
+        ["probeFileSize", "android-armeabi-v7a", "/tmp/attach/install/next-android-armeabi-v7a-release/lib/libmapbox-gl.so", 0.01, "compressed"]
       ],
       "width": 64,
       "height": 64
diff --git a/metrics/tests/binary-size/android-x86/expected.png b/metrics/next-binary-size/android-x86/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/android-x86/expected.png
+++ b/metrics/next-binary-size/android-x86/expected.png
diff --git a/metrics/tests/binary-size/android-x86/metrics.json b/metrics/next-binary-size/android-x86/metrics.json
index aec4320ed4..f19481ea33 100644
--- a/metrics/tests/binary-size/android-x86/metrics.json
+++ b/metrics/next-binary-size/android-x86/metrics.json
@@ -2,7 +2,7 @@
     "file-size": [
         [
             "android-x86",
-            "/src/workspace/next-android-x86-release/lib/libmapbox-gl.so",
+            "/tmp/attach/install/next-android-x86-release/lib/libmapbox-gl.so",
             1934186
         ]
     ]
diff --git a/metrics/tests/binary-size/android-x86/style.json b/metrics/next-binary-size/android-x86/style.json
index c113583ef2..16c74ea763 100644
--- a/metrics/tests/binary-size/android-x86/style.json
+++ b/metrics/next-binary-size/android-x86/style.json
@@ -3,7 +3,7 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "android-x86", "/src/workspace/next-android-x86-release/lib/libmapbox-gl.so", 0.01, "compressed"]
+        ["probeFileSize", "android-x86", "/tmp/attach/install/next-android-x86-release/lib/libmapbox-gl.so", 0.01, "compressed"]
       ],
       "width": 64,
       "height": 64
diff --git a/metrics/tests/binary-size/android-x86_64/expected.png b/metrics/next-binary-size/android-x86_64/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/android-x86_64/expected.png
+++ b/metrics/next-binary-size/android-x86_64/expected.png
diff --git a/metrics/tests/binary-size/android-x86_64/metrics.json b/metrics/next-binary-size/android-x86_64/metrics.json
index 4c093613ad..5bfe5ecb67 100644
--- a/metrics/tests/binary-size/android-x86_64/metrics.json
+++ b/metrics/next-binary-size/android-x86_64/metrics.json
@@ -2,7 +2,7 @@
     "file-size": [
         [
             "android-x86_64",
-            "/src/workspace/next-android-x86_64-release/lib/libmapbox-gl.so",
+            "/tmp/attach/install/next-android-x86_64-release/lib/libmapbox-gl.so",
             1946541
         ]
     ]
diff --git a/metrics/tests/binary-size/android-x86_64/style.json b/metrics/next-binary-size/android-x86_64/style.json
index 8d05da3714..5aeca52013 100644
--- a/metrics/tests/binary-size/android-x86_64/style.json
+++ b/metrics/next-binary-size/android-x86_64/style.json
@@ -3,7 +3,7 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "android-x86_64", "/src/workspace/next-android-x86_64-release/lib/libmapbox-gl.so", 0.01, "compressed"]
+        ["probeFileSize", "android-x86_64", "/tmp/attach/install/next-android-x86_64-release/lib/libmapbox-gl.so", 0.01, "compressed"]
       ],
       "width": 64,
       "height": 64
diff --git a/metrics/tests/binary-size/linux-clang8/expected.png b/metrics/next-binary-size/linux-clang8/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/linux-clang8/expected.png
+++ b/metrics/next-binary-size/linux-clang8/expected.png
diff --git a/metrics/tests/binary-size/linux-clang8/metrics.json b/metrics/next-binary-size/linux-clang8/metrics.json
index c7ca8943d0..9ffd3db016 100644
--- a/metrics/tests/binary-size/linux-clang8/metrics.json
+++ b/metrics/next-binary-size/linux-clang8/metrics.json
@@ -2,17 +2,17 @@
     "file-size": [
         [
             "mbgl-glfw",
-            "/src/workspace/next-linux-clang8-release/bin/mbgl-glfw",
+            "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-glfw",
             6290568
         ],
         [
             "mbgl-offline",
-            "/src/workspace/next-linux-clang8-release/bin/mbgl-offline",
+            "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-offline",
             5597912
         ],
         [
             "mbgl-render",
-            "/src/workspace/next-linux-clang8-release/bin/mbgl-render",
+            "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-render",
             6220728
         ]
     ]
diff --git a/metrics/next-binary-size/linux-clang8/style.json b/metrics/next-binary-size/linux-clang8/style.json
new file mode 100644
index 0000000000..f6d23490e6
--- /dev/null
+++ b/metrics/next-binary-size/linux-clang8/style.json
@@ -0,0 +1,24 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "mbgl-glfw", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-glfw", 0.01],
+        ["probeFileSize", "mbgl-offline", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-offline", 0.01],
+        ["probeFileSize", "mbgl-render", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-render", 0.01]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {},
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "green"
+      }
+    }
+  ]
+}
diff --git a/metrics/tests/binary-size/linux-gcc8/expected.png b/metrics/next-binary-size/linux-gcc8/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/linux-gcc8/expected.png
+++ b/metrics/next-binary-size/linux-gcc8/expected.png
diff --git a/metrics/tests/binary-size/linux-gcc8/metrics.json b/metrics/next-binary-size/linux-gcc8/metrics.json
index 5b8e1f5bec..f568a9cc3d 100644
--- a/metrics/tests/binary-size/linux-gcc8/metrics.json
+++ b/metrics/next-binary-size/linux-gcc8/metrics.json
@@ -2,17 +2,17 @@
     "file-size": [
         [
             "mbgl-glfw",
-            "/src/workspace/next-linux-gcc8-release/bin/mbgl-glfw",
+            "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-glfw",
             7323976
         ],
         [
             "mbgl-offline",
-            "/src/workspace/next-linux-gcc8-release/bin/mbgl-offline",
+            "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-offline",
             6422760
         ],
         [
             "mbgl-render",
-            "/src/workspace/next-linux-gcc8-release/bin/mbgl-render",
+            "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-render",
             7143752
         ]
     ]
diff --git a/metrics/next-binary-size/linux-gcc8/style.json b/metrics/next-binary-size/linux-gcc8/style.json
new file mode 100644
index 0000000000..abe8d6b779
--- /dev/null
+++ b/metrics/next-binary-size/linux-gcc8/style.json
@@ -0,0 +1,24 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "mbgl-glfw", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-glfw", 0.01],
+        ["probeFileSize", "mbgl-offline", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-offline", 0.01],
+        ["probeFileSize", "mbgl-render", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-render", 0.01]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {},
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "green"
+      }
+    }
+  ]
+}
diff --git a/metrics/tests/binary-size/macos-xcode11/expected.png b/metrics/next-binary-size/macos-xcode11/expected.png
index 63740e7305..63740e7305 100644
--- a/metrics/tests/binary-size/macos-xcode11/expected.png
+++ b/metrics/next-binary-size/macos-xcode11/expected.png
diff --git a/metrics/next-binary-size/macos-xcode11/metrics.json b/metrics/next-binary-size/macos-xcode11/metrics.json
new file mode 100644
index 0000000000..4e2298ac30
--- /dev/null
+++ b/metrics/next-binary-size/macos-xcode11/metrics.json
@@ -0,0 +1,19 @@
+{
+    "file-size": [
+        [
+            "mbgl-glfw",
+            "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-glfw",
+            5502420
+        ],
+        [
+            "mbgl-offline",
+            "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-offline",
+            5334136
+        ],
+        [
+            "mbgl-render",
+            "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-render",
+            5384420
+        ]
+    ]
+}
diff --git a/metrics/next-binary-size/macos-xcode11/style.json b/metrics/next-binary-size/macos-xcode11/style.json
new file mode 100644
index 0000000000..cd2db87c48
--- /dev/null
+++ b/metrics/next-binary-size/macos-xcode11/style.json
@@ -0,0 +1,24 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "mbgl-glfw", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-glfw", 0.01],
+        ["probeFileSize", "mbgl-offline", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-offline", 0.01],
+        ["probeFileSize", "mbgl-render", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-render", 0.01]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {},
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "green"
+      }
+    }
+  ]
+}
diff --git a/metrics/tests/binary-size/linux-clang8/style.json b/metrics/tests/binary-size/linux-clang8/style.json
deleted file mode 100644
index 7af68d5359..0000000000
--- a/metrics/tests/binary-size/linux-clang8/style.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-  "version": 8,
-  "metadata": {
-    "test": {
-      "operations": [
-        ["probeFileSize", "mbgl-glfw", "/src/workspace/next-linux-clang8-release/bin/mbgl-glfw", 0.01],
-        ["probeFileSize", "mbgl-offline", "/src/workspace/next-linux-clang8-release/bin/mbgl-offline", 0.01],
-        ["probeFileSize", "mbgl-render", "/src/workspace/next-linux-clang8-release/bin/mbgl-render", 0.01]
-      ],
-      "width": 64,
-      "height": 64
-    }
-  },
-  "sources": {},
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "paint": {
-        "background-color": "green"
-      }
-    }
-  ]
-}
diff --git a/metrics/tests/binary-size/linux-gcc8/style.json b/metrics/tests/binary-size/linux-gcc8/style.json
deleted file mode 100644
index 8ace1f82ab..0000000000
--- a/metrics/tests/binary-size/linux-gcc8/style.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-  "version": 8,
-  "metadata": {
-    "test": {
-      "operations": [
-        ["probeFileSize", "mbgl-glfw", "/src/workspace/next-linux-gcc8-release/bin/mbgl-glfw", 0.01],
-        ["probeFileSize", "mbgl-offline", "/src/workspace/next-linux-gcc8-release/bin/mbgl-offline", 0.01],
-        ["probeFileSize", "mbgl-render", "/src/workspace/next-linux-gcc8-release/bin/mbgl-render", 0.01]
-      ],
-      "width": 64,
-      "height": 64
-    }
-  },
-  "sources": {},
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "paint": {
-        "background-color": "green"
-      }
-    }
-  ]
-}
diff --git a/metrics/tests/binary-size/macos-xcode11/metrics.json b/metrics/tests/binary-size/macos-xcode11/metrics.json
deleted file mode 100644
index 41886e8caf..0000000000
--- a/metrics/tests/binary-size/macos-xcode11/metrics.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-    "file-size": [
-        [
-            "mbgl-glfw",
-            "/src/workspace/next-macos-xcode11-release/bin/mbgl-glfw",
-            5502420
-        ],
-        [
-            "mbgl-offline",
-            "/src/workspace/next-macos-xcode11-release/bin/mbgl-offline",
-            5334136
-        ],
-        [
-            "mbgl-render",
-            "/src/workspace/next-macos-xcode11-release/bin/mbgl-render",
-            5384420
-        ]
-    ]
-}
diff --git a/metrics/tests/binary-size/macos-xcode11/style.json b/metrics/tests/binary-size/macos-xcode11/style.json
deleted file mode 100644
index 1c2e8123f9..0000000000
--- a/metrics/tests/binary-size/macos-xcode11/style.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-  "version": 8,
-  "metadata": {
-    "test": {
-      "operations": [
-        ["probeFileSize", "mbgl-glfw", "/src/workspace/next-macos-xcode11-release/bin/mbgl-glfw", 0.01],
-        ["probeFileSize", "mbgl-offline", "/src/workspace/next-macos-xcode11-release/bin/mbgl-offline", 0.01],
-        ["probeFileSize", "mbgl-render", "/src/workspace/next-macos-xcode11-release/bin/mbgl-render", 0.01]
-      ],
-      "width": 64,
-      "height": 64
-    }
-  },
-  "sources": {},
-  "layers": [
-    {
-      "id": "background",
-      "type": "background",
-      "paint": {
-        "background-color": "green"
-      }
-    }
-  ]
-}