diff options
Diffstat (limited to 'metrics/ignores')
| -rw-r--r-- | metrics/ignores/platform-all.json | 133 | ||||
| -rw-r--r-- | metrics/ignores/platform-android.json | 13 | ||||
| -rw-r--r-- | metrics/ignores/platform-linux.json | 10 | ||||
| -rw-r--r-- | metrics/ignores/platform-macos.json | 4 |
4 files changed, 160 insertions, 0 deletions
diff --git a/metrics/ignores/platform-all.json b/metrics/ignores/platform-all.json new file mode 100644 index 0000000000..e1cf323afa --- /dev/null +++ b/metrics/ignores/platform-all.json @@ -0,0 +1,133 @@ +{ + "probes/memory/pass-memory-size-is-same": "TODO: Check with Mikhail why is this failing", + "binary-size/android-arm64-v8a": "skip - only makes sense for the binary size bot.", + "binary-size/android-armeabi-v7a": "skip - only makes sense for the binary size bot.", + "binary-size/android-x86": "skip - only makes sense for the binary size bot.", + "binary-size/android-x86_64": "skip - only makes sense for the binary size bot.", + "binary-size/linux-clang8": "skip - only makes sense for the binary size bot.", + "binary-size/linux-gcc8": "skip - only makes sense for the binary size bot.", + "binary-size/macos-xcode11": "skip - only makes sense for the binary size bot.", + "expression-tests/collator/accent-equals-de": "Locale-specific behavior changes based on platform.", + "expression-tests/in/assert-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/assert-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/basic-array": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/basic-string": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/invalid-haystack": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/in/invalid-needle": "https://github.com/mapbox/mapbox-gl-native/issues/15893", + "expression-tests/interpolate-hcl/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720", + "expression-tests/interpolate-lab/linear": "https://github.com/mapbox/mapbox-gl-native/issues/8720", + "expression-tests/is-supported-script/default": "This tests RTL text plugin behavior specific to GL JS", + "expression-tests/legacy/categorical/array": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/boolean": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/color": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/number": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/categorical/string": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/exponential/color-hcl": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/exponential/color-lab": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/identity/color-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/identity/enum-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/interval/composite": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/interval/composite-default": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/legacy/interval/tokens-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/12747", + "expression-tests/resolved-locale/basic": "Even the 'en' locale may not be present on some test systems.", + "probes/file-size/fail-file-doesnt-match": "Should fail, doesn't match the expectation.", + "probes/file-size/fail-file-not-found": "Should fail, file not found.", + "probes/file-size/fail-size-is-over": "Should fail, size is bigger than expected.", + "probes/file-size/fail-size-is-under": "Should fail, size is smaller than expected.", + "probes/gfx/fail-ib-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.", + "probes/gfx/fail-negative-framebuffer-count": "Should fail, number of frame buffers is higher than expected.", + "probes/gfx/fail-texture-mem-mismatch": "Should fail, combined byte size of textures doesn't match the expectation.", + "probes/gfx/fail-too-few-buffers": "Should fail, number of vertex and index buffers is smaller than expected.", + "probes/gfx/fail-too-few-textures": "Should fail, number of textures is smaller than expected.", + "probes/gfx/fail-too-many-drawcalls": "Should fail, number of draw calls higher than expected.", + "probes/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.", + "probes/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.", + "probes/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.", + "probes/network/fail-requests": "Should fail, number of requests higher than expected.", + "probes/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.", + "probes/network/fail-transferred": "Should fail, amount of transferred data higher than expected.", + "query-tests/fill-extrusion-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701", + "query-tests/fill-extrusion/base-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/box-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/side-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort-concave-inner": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort-concave-outer": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/sort-rotated": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-extrusion/top-in": "https://github.com/mapbox/mapbox-gl-native/issues/13139", + "query-tests/fill-translate/multiple-layers": "https://github.com/mapbox/mapbox-gl-native/issues/12701", + "query-tests/geometry/multilinestring": "needs investigation", + "query-tests/geometry/multipolygon": "needs investigation", + "query-tests/geometry/polygon": "needs investigation", + "query-tests/regressions/mapbox-gl-js#6555": "skip - no querySourceFeatures in mbgl-node; needs issue", + "query-tests/regressions/mapbox-gl-js#7883": "https://github.com/mapbox/mapbox-gl-native/issues/14585", + "query-tests/world-wrapping/box": "skip - needs issue", + "render-tests/background-color/colorSpace-hcl": "needs issue", + "render-tests/background-color/transition": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/canvas/default": "skip - js specific", + "render-tests/canvas/update": "skip - js specific", + "render-tests/circle-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", + "render-tests/collator/default": "Some test platforms don't resolve 'en' locale", + "render-tests/collator/resolved-locale": "Some test platforms don't resolve 'en' locale", + "render-tests/custom-layer-js/depth": "skip - js specific", + "render-tests/custom-layer-js/null-island": "skip - js specific", + "render-tests/custom-layer-js/tent-3d": "skip - js specific", + "render-tests/debug/collision": "https://github.com/mapbox/mapbox-gl-native/issues/3841", + "render-tests/debug/overdraw": "https://github.com/mapbox/mapbox-gl-native/issues/15638", + "render-tests/debug/raster": "https://github.com/mapbox/mapbox-gl-native/issues/15510", + "render-tests/debug/tile": "https://github.com/mapbox/mapbox-gl-native/issues/3841", + "render-tests/debug/tile-overscaled": "https://github.com/mapbox/mapbox-gl-native/issues/3841", + "render-tests/extent/1024-circle": "needs investigation", + "render-tests/feature-state/promote-id": "https://github.com/mapbox/mapbox-gl-native/issues/16016", + "render-tests/fill-extrusion-geometry/linestring": "https://github.com/mapbox/mapbox-gl-native/pull/14240", + "render-tests/fill-extrusion-pattern/@2x": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/feature-expression": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/function": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/function-2": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/literal": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/opacity": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-extrusion-pattern/tile-buffer": "https://github.com/mapbox/mapbox-gl-js/issues/3327", + "render-tests/fill-pattern/update-feature-state": "https://github.com/mapbox/mapbox-gl-native/issues/15895", + "render-tests/fill-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", + "render-tests/geojson/inline-linestring-fill": "current behavior is arbitrary", + "render-tests/icon-text-fit/stretch-fifteen-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-@2x": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-content": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-content-collision": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-just-height": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-nine-part-just-width": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-three-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-two-part": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/stretch-underscale": "https://github.com/mapbox/mapbox-gl-native/issues/16017", + "render-tests/icon-text-fit/text-variable-anchor-overlap": "https://github.com/mapbox/mapbox-gl-native/issues/15809", + "render-tests/line-sort-key/literal": "https://github.com/mapbox/mapbox-gl-native/issues/15008", + "render-tests/mixed-zoom/z10-z11": "https://github.com/mapbox/mapbox-gl-native/issues/10397", + "render-tests/raster-masking/overlapping-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/10195", + "render-tests/real-world/bangkok": "https://github.com/mapbox/mapbox-gl-native/issues/10412", + "render-tests/real-world/chicago": "https://github.com/mapbox/mapbox-gl-native/issues/10412", + "render-tests/real-world/sanfrancisco": "https://github.com/mapbox/mapbox-gl-native/issues/10412", + "render-tests/regressions/mapbox-gl-js#2467": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#2762": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#2769": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#5740": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#5982": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#6655": "skip - port https://github.com/mapbox/mapbox-gl-js/pull/6263 - needs issue", + "render-tests/regressions/mapbox-gl-js#6706": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/regressions/mapbox-gl-js#6806": "pending https://github.com/mapbox/mapbox-gl-js/pull/6812", + "render-tests/regressions/mapbox-gl-js#7271": "https://github.com/mapbox/mapbox-gl-native/issues/12888", + "render-tests/regressions/mapbox-gl-js#7302": "skip - js specific", + "render-tests/regressions/mapbox-gl-js#7708": "skip - js specific", + "render-tests/regressions/mapbox-gl-js#8026": "skip - js specific", + "render-tests/regressions/mapbox-gl-js#8817": "skip - https://github.com/mapbox/mapbox-gl-native/issues/15737", + "render-tests/regressions/mapbox-gl-js#9009": "https://github.com/mapbox/mapbox-gl-native/issues/16018", + "render-tests/regressions/mapbox-gl-native#7357": "https://github.com/mapbox/mapbox-gl-native/issues/7357", + "render-tests/runtime-styling/image-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021", + "render-tests/runtime-styling/pattern-add-remove-add": "skip - https://github.com/mapbox/mapbox-gl-native/issues/16021", + "render-tests/symbol-cross-fade/chinese": "https://github.com/mapbox/mapbox-gl-native/issues/10619", + "render-tests/text-max-width/zero-width-point-placement": "https://github.com/mapbox/mapbox-gl-native/issues/15648", + "render-tests/text-size/nan": "https://github.com/mapbox/mapbox-gl-native/issues/16020", + "render-tests/text-variable-anchor/remember-last-placement": "skip - fails on gl-native, as symbol index is not functional at static map mode - needs issue", + "render-tests/video/default": "skip - https://github.com/mapbox/mapbox-gl-native/issues/601", + "render-tests/zoomed-fill/negative-zoom": "https://github.com/mapbox/mapbox-gl-native/issues/16019" +} diff --git a/metrics/ignores/platform-android.json b/metrics/ignores/platform-android.json new file mode 100644 index 0000000000..ffd8fc9e15 --- /dev/null +++ b/metrics/ignores/platform-android.json @@ -0,0 +1,13 @@ +{ + "probes/gfx/pass": "Needs to be investigated and fixed.", + "probes/gfx/pass-double-probe": "Needs to be investigated and fixed.", + "probes/gfx/pass-probe-reset": "Needs to be investigated and fixed.", + "probes/network/pass": "Needs to be investigated and fixed.", + "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", + "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", + "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.", + "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.", + "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.", + "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.", + "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed." +} diff --git a/metrics/ignores/platform-linux.json b/metrics/ignores/platform-linux.json new file mode 100644 index 0000000000..0988aff5a2 --- /dev/null +++ b/metrics/ignores/platform-linux.json @@ -0,0 +1,10 @@ +{ + "render-tests/fill-opacity/zoom-and-property-function-pattern": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15322", + "render-tests/fill-pattern/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14423", + "render-tests/fill-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14870", + "render-tests/fill-pattern/zoomed": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14768", + "render-tests/line-pattern/opacity": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/15320", + "render-tests/line-translate/literal": "Flaky on Linux: https://github.com/mapbox/mapbox-gl-native/issues/14859", + "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test", + "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test" +} diff --git a/metrics/ignores/platform-macos.json b/metrics/ignores/platform-macos.json new file mode 100644 index 0000000000..bc5e67820a --- /dev/null +++ b/metrics/ignores/platform-macos.json @@ -0,0 +1,4 @@ +{ + "render-tests/regressions/mapbox-gl-js#5642": "Failing with mbgl-render-test", + "render-tests/regressions/mapbox-gl-js#7066": "Failing with mbgl-render-test" +} |