[render-test] Add tests for filesize probeupstream/tmpsantos-binary_size

And make sure we run it on the bots

15 files changed, 173 insertions, 7 deletions

diff --git a/.gitignore b/.gitignore
index bf6b13e007..538870edf7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,9 @@
 *.gcno
 *.gcda
 *~
+**/actual.png
+**/diff.png
+render-test/index.html
 offline.db
 new_offline.db
 xcuserdata
@@ -19,8 +22,6 @@ xcuserdata
 /test/fixtures/api/2.png
 /test/fixtures/offline_database/offline.db
 /test/fixtures/offline_database/offline.db-*
-/test/fixtures/**/actual.png
-/test/fixtures/**/diff.png
 /test/output
 /node_modules
 /platform/ios/benchmark/assets/glyphs/DIN*
diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt
index 5aadd0f859..80709d2b89 100644
--- a/next/render-test/CMakeLists.txt
+++ b/next/render-test/CMakeLists.txt
@@ -47,8 +47,19 @@ add_test(
         render-tests
         --recycle-map
         --shuffle
-        --seed
-        ${MBGL_RENDER_TEST_SEED}
+        --seed=${MBGL_RENDER_TEST_SEED}
+    WORKING_DIRECTORY ${MBGL_ROOT}
+)
+
+add_test(
+    NAME mbgl-render-test-probes
+    COMMAND
+        mbgl-render-test
+        render-tests
+        --recycle-map
+        --shuffle
+        --seed=${MBGL_RENDER_TEST_SEED}
+        --rootPath=render-test
     WORKING_DIRECTORY ${MBGL_ROOT}
 )
 
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index f1af0a1dae..e1c7a14028 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -421,15 +421,17 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
     auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
 
     mbgl::filesystem::path platformSpecificIgnores;
+    mbgl::filesystem::path ownTestsIgnores =
+        mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/tests/should-fail.json");
 
 #ifdef __APPLE__
     platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json");
 #elif __linux__
     platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json");
 #endif
-    
-    std::vector<mbgl::filesystem::path> ignoresPaths = { mainIgnoresPath, platformSpecificIgnores };
-    for (auto path: ignoresPaths) {
+
+    std::vector<mbgl::filesystem::path> ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores};
+    for (auto path : ignoresPaths) {
         auto maybeIgnores = readJson(path);
         if (!maybeIgnores.is<mbgl::JSDocument>()) {
             continue;
diff --git a/render-test/tests/filesize/fail-file-doesnt-match/expected.png b/render-test/tests/filesize/fail-file-doesnt-match/expected.png
new file mode 100644
index 0000000000..feed2e457f
--- /dev/null
+++ b/render-test/tests/filesize/fail-file-doesnt-match/expected.png
diff --git a/render-test/tests/filesize/fail-file-doesnt-match/style.json b/render-test/tests/filesize/fail-file-doesnt-match/style.json
new file mode 100644
index 0000000000..91e90ffbe8
--- /dev/null
+++ b/render-test/tests/filesize/fail-file-doesnt-match/style.json
@@ -0,0 +1,36 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "aaaaa.json"],
+        ["probeFileSize", "image", "bbbbb.png"]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "sprite": "local://sprites/sprite",
+  "layers": [
+    {
+      "id": "symbol",
+      "type": "symbol",
+      "source": "geojson",
+      "layout": {
+        "icon-image": "circle-12"
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/filesize/fail-size-is-over/expected.png b/render-test/tests/filesize/fail-size-is-over/expected.png
new file mode 100644
index 0000000000..feed2e457f
--- /dev/null
+++ b/render-test/tests/filesize/fail-size-is-over/expected.png
diff --git a/render-test/tests/filesize/fail-size-is-over/metrics.json b/render-test/tests/filesize/fail-size-is-over/metrics.json
new file mode 100644
index 0000000000..6b25355235
--- /dev/null
+++ b/render-test/tests/filesize/fail-size-is-over/metrics.json
@@ -0,0 +1 @@
+{"fileSize":[["image","expected.png",111],["style","style.json",111]],"memory":[]}
diff --git a/render-test/tests/filesize/fail-size-is-over/style.json b/render-test/tests/filesize/fail-size-is-over/style.json
new file mode 100644
index 0000000000..ce2069a372
--- /dev/null
+++ b/render-test/tests/filesize/fail-size-is-over/style.json
@@ -0,0 +1,36 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json"],
+        ["probeFileSize", "image", "expected.png"]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "sprite": "local://sprites/sprite",
+  "layers": [
+    {
+      "id": "symbol",
+      "type": "symbol",
+      "source": "geojson",
+      "layout": {
+        "icon-image": "circle-12"
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/filesize/fail-size-is-under/expected.png b/render-test/tests/filesize/fail-size-is-under/expected.png
new file mode 100644
index 0000000000..feed2e457f
--- /dev/null
+++ b/render-test/tests/filesize/fail-size-is-under/expected.png
diff --git a/render-test/tests/filesize/fail-size-is-under/metrics.json b/render-test/tests/filesize/fail-size-is-under/metrics.json
new file mode 100644
index 0000000000..d04a797ca1
--- /dev/null
+++ b/render-test/tests/filesize/fail-size-is-under/metrics.json
@@ -0,0 +1 @@
+{"fileSize":[["image","expected.png",999],["style","style.json",999]],"memory":[]}
diff --git a/render-test/tests/filesize/fail-size-is-under/style.json b/render-test/tests/filesize/fail-size-is-under/style.json
new file mode 100644
index 0000000000..ce2069a372
--- /dev/null
+++ b/render-test/tests/filesize/fail-size-is-under/style.json
@@ -0,0 +1,36 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json"],
+        ["probeFileSize", "image", "expected.png"]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "sprite": "local://sprites/sprite",
+  "layers": [
+    {
+      "id": "symbol",
+      "type": "symbol",
+      "source": "geojson",
+      "layout": {
+        "icon-image": "circle-12"
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/filesize/pass-size-is-same/expected.png b/render-test/tests/filesize/pass-size-is-same/expected.png
new file mode 100644
index 0000000000..feed2e457f
--- /dev/null
+++ b/render-test/tests/filesize/pass-size-is-same/expected.png
diff --git a/render-test/tests/filesize/pass-size-is-same/metrics.json b/render-test/tests/filesize/pass-size-is-same/metrics.json
new file mode 100644
index 0000000000..44f299ac3b
--- /dev/null
+++ b/render-test/tests/filesize/pass-size-is-same/metrics.json
@@ -0,0 +1 @@
+{"fileSize":[["image","expected.png",686],["style","style.json",609]],"memory":[]}
\ No newline at end of file
diff --git a/render-test/tests/filesize/pass-size-is-same/style.json b/render-test/tests/filesize/pass-size-is-same/style.json
new file mode 100644
index 0000000000..ce2069a372
--- /dev/null
+++ b/render-test/tests/filesize/pass-size-is-same/style.json
@@ -0,0 +1,36 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json"],
+        ["probeFileSize", "image", "expected.png"]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "sprite": "local://sprites/sprite",
+  "layers": [
+    {
+      "id": "symbol",
+      "type": "symbol",
+      "source": "geojson",
+      "layout": {
+        "icon-image": "circle-12"
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json
new file mode 100644
index 0000000000..28859d4151
--- /dev/null
+++ b/render-test/tests/should-fail.json
@@ -0,0 +1,5 @@
+{
+  "tests/filesize/fail-size-is-over": "Should fail, size is bigger than expected.",
+  "tests/filesize/fail-size-is-under": "Should fail, size is smaller than expected.",
+  "tests/filesize/fail-file-doesnt-match": "Should fail, file not found."
+}