From 422195de9068e9288e09f928969af7de3e830312 Mon Sep 17 00:00:00 2001 From: "Thiago Marcos P. Santos" Date: Wed, 9 Oct 2019 22:00:30 +0300 Subject: [render-test] Add tests for filesize probe And make sure we run it on the bots --- .gitignore | 5 +-- next/render-test/CMakeLists.txt | 15 +++++++-- render-test/parser.cpp | 8 +++-- .../filesize/fail-file-doesnt-match/expected.png | Bin 0 -> 686 bytes .../filesize/fail-file-doesnt-match/style.json | 36 +++++++++++++++++++++ .../tests/filesize/fail-size-is-over/expected.png | Bin 0 -> 686 bytes .../tests/filesize/fail-size-is-over/metrics.json | 1 + .../tests/filesize/fail-size-is-over/style.json | 36 +++++++++++++++++++++ .../tests/filesize/fail-size-is-under/expected.png | Bin 0 -> 686 bytes .../tests/filesize/fail-size-is-under/metrics.json | 1 + .../tests/filesize/fail-size-is-under/style.json | 36 +++++++++++++++++++++ .../tests/filesize/pass-size-is-same/expected.png | Bin 0 -> 686 bytes .../tests/filesize/pass-size-is-same/metrics.json | 1 + .../tests/filesize/pass-size-is-same/style.json | 36 +++++++++++++++++++++ render-test/tests/should-fail.json | 5 +++ 15 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 render-test/tests/filesize/fail-file-doesnt-match/expected.png create mode 100644 render-test/tests/filesize/fail-file-doesnt-match/style.json create mode 100644 render-test/tests/filesize/fail-size-is-over/expected.png create mode 100644 render-test/tests/filesize/fail-size-is-over/metrics.json create mode 100644 render-test/tests/filesize/fail-size-is-over/style.json create mode 100644 render-test/tests/filesize/fail-size-is-under/expected.png create mode 100644 render-test/tests/filesize/fail-size-is-under/metrics.json create mode 100644 render-test/tests/filesize/fail-size-is-under/style.json create mode 100644 render-test/tests/filesize/pass-size-is-same/expected.png create mode 100644 render-test/tests/filesize/pass-size-is-same/metrics.json create mode 100644 render-test/tests/filesize/pass-size-is-same/style.json create mode 100644 render-test/tests/should-fail.json diff --git a/.gitignore b/.gitignore index bf6b13e007..538870edf7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.gcno *.gcda *~ +**/actual.png +**/diff.png +render-test/index.html offline.db new_offline.db xcuserdata @@ -19,8 +22,6 @@ xcuserdata /test/fixtures/api/2.png /test/fixtures/offline_database/offline.db /test/fixtures/offline_database/offline.db-* -/test/fixtures/**/actual.png -/test/fixtures/**/diff.png /test/output /node_modules /platform/ios/benchmark/assets/glyphs/DIN* diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt index 5aadd0f859..80709d2b89 100644 --- a/next/render-test/CMakeLists.txt +++ b/next/render-test/CMakeLists.txt @@ -47,8 +47,19 @@ add_test( render-tests --recycle-map --shuffle - --seed - ${MBGL_RENDER_TEST_SEED} + --seed=${MBGL_RENDER_TEST_SEED} + WORKING_DIRECTORY ${MBGL_ROOT} +) + +add_test( + NAME mbgl-render-test-probes + COMMAND + mbgl-render-test + render-tests + --recycle-map + --shuffle + --seed=${MBGL_RENDER_TEST_SEED} + --rootPath=render-test WORKING_DIRECTORY ${MBGL_ROOT} ) diff --git a/render-test/parser.cpp b/render-test/parser.cpp index f1af0a1dae..e1c7a14028 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -421,15 +421,17 @@ std::vector> parseIgnores() { auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); mbgl::filesystem::path platformSpecificIgnores; + mbgl::filesystem::path ownTestsIgnores = + mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/tests/should-fail.json"); #ifdef __APPLE__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json"); #elif __linux__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json"); #endif - - std::vector ignoresPaths = { mainIgnoresPath, platformSpecificIgnores }; - for (auto path: ignoresPaths) { + + std::vector ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores}; + for (auto path : ignoresPaths) { auto maybeIgnores = readJson(path); if (!maybeIgnores.is()) { continue; diff --git a/render-test/tests/filesize/fail-file-doesnt-match/expected.png b/render-test/tests/filesize/fail-file-doesnt-match/expected.png new file mode 100644 index 0000000000..feed2e457f Binary files /dev/null and b/render-test/tests/filesize/fail-file-doesnt-match/expected.png differ diff --git a/render-test/tests/filesize/fail-file-doesnt-match/style.json b/render-test/tests/filesize/fail-file-doesnt-match/style.json new file mode 100644 index 0000000000..91e90ffbe8 --- /dev/null +++ b/render-test/tests/filesize/fail-file-doesnt-match/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "aaaaa.json"], + ["probeFileSize", "image", "bbbbb.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/fail-size-is-over/expected.png b/render-test/tests/filesize/fail-size-is-over/expected.png new file mode 100644 index 0000000000..feed2e457f Binary files /dev/null and b/render-test/tests/filesize/fail-size-is-over/expected.png differ diff --git a/render-test/tests/filesize/fail-size-is-over/metrics.json b/render-test/tests/filesize/fail-size-is-over/metrics.json new file mode 100644 index 0000000000..6b25355235 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-over/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","expected.png",111],["style","style.json",111]],"memory":[]} diff --git a/render-test/tests/filesize/fail-size-is-over/style.json b/render-test/tests/filesize/fail-size-is-over/style.json new file mode 100644 index 0000000000..ce2069a372 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-over/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/fail-size-is-under/expected.png b/render-test/tests/filesize/fail-size-is-under/expected.png new file mode 100644 index 0000000000..feed2e457f Binary files /dev/null and b/render-test/tests/filesize/fail-size-is-under/expected.png differ diff --git a/render-test/tests/filesize/fail-size-is-under/metrics.json b/render-test/tests/filesize/fail-size-is-under/metrics.json new file mode 100644 index 0000000000..d04a797ca1 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-under/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","expected.png",999],["style","style.json",999]],"memory":[]} diff --git a/render-test/tests/filesize/fail-size-is-under/style.json b/render-test/tests/filesize/fail-size-is-under/style.json new file mode 100644 index 0000000000..ce2069a372 --- /dev/null +++ b/render-test/tests/filesize/fail-size-is-under/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/filesize/pass-size-is-same/expected.png b/render-test/tests/filesize/pass-size-is-same/expected.png new file mode 100644 index 0000000000..feed2e457f Binary files /dev/null and b/render-test/tests/filesize/pass-size-is-same/expected.png differ diff --git a/render-test/tests/filesize/pass-size-is-same/metrics.json b/render-test/tests/filesize/pass-size-is-same/metrics.json new file mode 100644 index 0000000000..44f299ac3b --- /dev/null +++ b/render-test/tests/filesize/pass-size-is-same/metrics.json @@ -0,0 +1 @@ +{"fileSize":[["image","expected.png",686],["style","style.json",609]],"memory":[]} \ No newline at end of file diff --git a/render-test/tests/filesize/pass-size-is-same/style.json b/render-test/tests/filesize/pass-size-is-same/style.json new file mode 100644 index 0000000000..ce2069a372 --- /dev/null +++ b/render-test/tests/filesize/pass-size-is-same/style.json @@ -0,0 +1,36 @@ +{ + "version": 8, + "metadata": { + "test": { + "operations": [ + ["probeFileSize", "style", "style.json"], + ["probeFileSize", "image", "expected.png"] + ], + "width": 64, + "height": 64 + } + }, + "sources": { + "geojson": { + "type": "geojson", + "data": { + "type": "Point", + "coordinates": [ + 0, + 0 + ] + } + } + }, + "sprite": "local://sprites/sprite", + "layers": [ + { + "id": "symbol", + "type": "symbol", + "source": "geojson", + "layout": { + "icon-image": "circle-12" + } + } + ] +} diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json new file mode 100644 index 0000000000..28859d4151 --- /dev/null +++ b/render-test/tests/should-fail.json @@ -0,0 +1,5 @@ +{ + "tests/filesize/fail-size-is-over": "Should fail, size is bigger than expected.", + "tests/filesize/fail-size-is-under": "Should fail, size is smaller than expected.", + "tests/filesize/fail-file-doesnt-match": "Should fail, file not found." +} -- cgit v1.2.1