From 8748aa00dcdcd4e842a6fd086d731a890353afa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20K=C3=A4fer?= Date: Tue, 19 Jul 2016 17:25:07 +0200 Subject: [build] add facilities for using american fuzzy loop --- .gitignore | 1 + .mason | 2 +- Makefile | 10 + platform/linux/platform.gyp | 19 ++ platform/macos/platform.gyp | 36 +++ scripts/export-xcconfig.py | 7 +- scripts/fuzz.sh | 28 +++ src/mbgl/tile/vector_tile.cpp | 271 +-------------------- src/mbgl/tile/vector_tile_data.cpp | 218 +++++++++++++++++ src/mbgl/tile/vector_tile_data.hpp | 79 ++++++ .../id:000000,sig:06,src:000000,op:havoc,rep:16 | 1 + .../id:000001,sig:06,src:000000,op:havoc,rep:8 | 1 + .../id:000002,sig:06,src:000000,op:havoc,rep:16 | 1 + .../id:000003,sig:06,src:000000,op:havoc,rep:32 | 1 + .../id:000004,sig:06,src:000000,op:havoc,rep:16 | 1 + .../id:000005,sig:06,src:000000,op:havoc,rep:2 | 1 + .../id:000006,sig:06,src:000000,op:havoc,rep:64 | 1 + .../id:000007,sig:06,src:000000,op:havoc,rep:2 | 1 + .../id:000008,sig:06,src:000000,op:havoc,rep:16 | Bin 0 -> 11 bytes .../id:000009,sig:06,src:000000,op:havoc,rep:8 | Bin 0 -> 11 bytes .../id:000010,sig:06,src:000000,op:havoc,rep:32 | 1 + .../id:000011,sig:06,src:000000,op:havoc,rep:8 | Bin 0 -> 4 bytes .../id:000012,sig:06,src:000000,op:havoc,rep:32 | 1 + .../id:000013,sig:06,src:000000,op:havoc,rep:8 | 1 + .../id:000014,sig:06,src:000000,op:havoc,rep:128 | 1 + .../id:000015,sig:06,src:000000,op:havoc,rep:32 | Bin 0 -> 12 bytes .../id:000016,sig:06,src:000000,op:havoc,rep:2 | Bin 0 -> 4 bytes .../id:000017,sig:06,src:000000,op:havoc,rep:8 | 1 + .../id:000018,sig:06,src:000000,op:havoc,rep:128 | 1 + .../id:000019,sig:06,src:000000,op:havoc,rep:8 | Bin 0 -> 8 bytes .../id:000020,sig:06,src:000000,op:havoc,rep:16 | Bin 0 -> 14 bytes .../id:000021,sig:06,src:000000,op:havoc,rep:32 | Bin 0 -> 26 bytes .../id:000022,sig:06,src:000000,op:havoc,rep:32 | Bin 0 -> 32 bytes ...000023,sig:06,src:000010,op:arith8,pos:2,val:-2 | 1 + ...000024,sig:06,src:000010,op:arith8,pos:2,val:-3 | 1 + ...000025,sig:06,src:000010,op:arith8,pos:2,val:-4 | 1 + ...000026,sig:06,src:000010,op:arith8,pos:2,val:+9 | 1 + ...00027,sig:06,src:000010,op:arith8,pos:2,val:+17 | 1 + ...00028,sig:06,src:000010,op:arith8,pos:2,val:+25 | 1 + ...00029,sig:06,src:000010,op:arith8,pos:2,val:+33 | 1 + .../id:000030,sig:06,src:000019,op:havoc,rep:8 | Bin 0 -> 18 bytes .../id:000031,sig:06,src:000018,op:havoc,rep:32 | 1 + .../id:000032,sig:06,src:000026,op:havoc,rep:128 | Bin 0 -> 40 bytes .../id:000033,sig:06,src:000001,op:havoc,rep:32 | Bin 0 -> 16 bytes ...00034,sig:06,src:000018+000004,op:splice,rep:32 | Bin 0 -> 24 bytes ...000035,sig:06,src:000007+000004,op:splice,rep:4 | 1 + ...000036,sig:06,src:000040+000043,op:splice,rep:8 | Bin 0 -> 23 bytes .../id:000037,sig:06,src:000044,op:havoc,rep:2 | Bin 0 -> 7 bytes ...000038,sig:06,src:000002+000010,op:splice,rep:2 | Bin 0 -> 5 bytes test/fuzz-input/canonicalize-tile-url/access_token | 1 + test/fuzz-input/canonicalize-tile-url/image | 1 + test/fuzz-input/glyphs-url/boxmap | 1 + test/fuzz-input/glyphs-url/comic-sans | 1 + test/fuzz-input/parse-filter/all-in-class.json | 1 + test/fuzz-input/parse-filter/f-gte-array.json | 1 + test/fuzz-input/parse-filter/has-foo.json | 1 + test/fuzz-input/parse-filter/id-lte-4.json | 1 + test/fuzz-input/parse-style/empty.json | 31 +++ test/fuzz-input/parse-tile/0-0-0.vector.pbf | Bin 0 -> 482553 bytes test/fuzz-input/source-url/user.map | 1 + test/fuzz-input/sprite-url/streets-draft-png | 1 + test/fuzz-input/sprite-url/streets-json | 1 + test/fuzz-input/sprite-url/streets-png | 1 + test/src/fuzz.cpp | 90 +++++++ test/test.gypi | 1 + test/tile/vector_tile_data.cpp | 39 +++ 66 files changed, 595 insertions(+), 272 deletions(-) create mode 100755 scripts/fuzz.sh create mode 100644 src/mbgl/tile/vector_tile_data.cpp create mode 100644 src/mbgl/tile/vector_tile_data.hpp create mode 100644 test/fixtures/tile_parser/id:000000,sig:06,src:000000,op:havoc,rep:16 create mode 100644 test/fixtures/tile_parser/id:000001,sig:06,src:000000,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000002,sig:06,src:000000,op:havoc,rep:16 create mode 100644 test/fixtures/tile_parser/id:000003,sig:06,src:000000,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000004,sig:06,src:000000,op:havoc,rep:16 create mode 100644 test/fixtures/tile_parser/id:000005,sig:06,src:000000,op:havoc,rep:2 create mode 100644 test/fixtures/tile_parser/id:000006,sig:06,src:000000,op:havoc,rep:64 create mode 100644 test/fixtures/tile_parser/id:000007,sig:06,src:000000,op:havoc,rep:2 create mode 100644 test/fixtures/tile_parser/id:000008,sig:06,src:000000,op:havoc,rep:16 create mode 100644 test/fixtures/tile_parser/id:000009,sig:06,src:000000,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000010,sig:06,src:000000,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000011,sig:06,src:000000,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000012,sig:06,src:000000,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000013,sig:06,src:000000,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000014,sig:06,src:000000,op:havoc,rep:128 create mode 100644 test/fixtures/tile_parser/id:000015,sig:06,src:000000,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000016,sig:06,src:000000,op:havoc,rep:2 create mode 100644 test/fixtures/tile_parser/id:000017,sig:06,src:000000,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000018,sig:06,src:000000,op:havoc,rep:128 create mode 100644 test/fixtures/tile_parser/id:000019,sig:06,src:000000,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000020,sig:06,src:000000,op:havoc,rep:16 create mode 100644 test/fixtures/tile_parser/id:000021,sig:06,src:000000,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000022,sig:06,src:000000,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000023,sig:06,src:000010,op:arith8,pos:2,val:-2 create mode 100644 test/fixtures/tile_parser/id:000024,sig:06,src:000010,op:arith8,pos:2,val:-3 create mode 100644 test/fixtures/tile_parser/id:000025,sig:06,src:000010,op:arith8,pos:2,val:-4 create mode 100644 test/fixtures/tile_parser/id:000026,sig:06,src:000010,op:arith8,pos:2,val:+9 create mode 100644 test/fixtures/tile_parser/id:000027,sig:06,src:000010,op:arith8,pos:2,val:+17 create mode 100644 test/fixtures/tile_parser/id:000028,sig:06,src:000010,op:arith8,pos:2,val:+25 create mode 100644 test/fixtures/tile_parser/id:000029,sig:06,src:000010,op:arith8,pos:2,val:+33 create mode 100644 test/fixtures/tile_parser/id:000030,sig:06,src:000019,op:havoc,rep:8 create mode 100644 test/fixtures/tile_parser/id:000031,sig:06,src:000018,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000032,sig:06,src:000026,op:havoc,rep:128 create mode 100644 test/fixtures/tile_parser/id:000033,sig:06,src:000001,op:havoc,rep:32 create mode 100644 test/fixtures/tile_parser/id:000034,sig:06,src:000018+000004,op:splice,rep:32 create mode 100644 test/fixtures/tile_parser/id:000035,sig:06,src:000007+000004,op:splice,rep:4 create mode 100644 test/fixtures/tile_parser/id:000036,sig:06,src:000040+000043,op:splice,rep:8 create mode 100644 test/fixtures/tile_parser/id:000037,sig:06,src:000044,op:havoc,rep:2 create mode 100644 test/fixtures/tile_parser/id:000038,sig:06,src:000002+000010,op:splice,rep:2 create mode 100644 test/fuzz-input/canonicalize-tile-url/access_token create mode 100644 test/fuzz-input/canonicalize-tile-url/image create mode 100644 test/fuzz-input/glyphs-url/boxmap create mode 100644 test/fuzz-input/glyphs-url/comic-sans create mode 100644 test/fuzz-input/parse-filter/all-in-class.json create mode 100644 test/fuzz-input/parse-filter/f-gte-array.json create mode 100644 test/fuzz-input/parse-filter/has-foo.json create mode 100644 test/fuzz-input/parse-filter/id-lte-4.json create mode 100644 test/fuzz-input/parse-style/empty.json create mode 100644 test/fuzz-input/parse-tile/0-0-0.vector.pbf create mode 100644 test/fuzz-input/source-url/user.map create mode 100644 test/fuzz-input/sprite-url/streets-draft-png create mode 100644 test/fuzz-input/sprite-url/streets-json create mode 100644 test/fuzz-input/sprite-url/streets-png create mode 100644 test/src/fuzz.cpp create mode 100644 test/tile/vector_tile_data.cpp diff --git a/.gitignore b/.gitignore index 16fd245107..275c296a26 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ xcuserdata /test/fixtures/**/actual.png /test/fixtures/**/diff.png /test/output +/test/fuzz-result /node_modules /platform/ios/benchmark/assets/glyphs/DIN* /platform/ios/benchmark/assets/tiles/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6 diff --git a/.mason b/.mason index 39ed569b3a..c40d5468f8 160000 --- a/.mason +++ b/.mason @@ -1 +1 @@ -Subproject commit 39ed569b3a3dc1197bc00d423c63800c25dc17f6 +Subproject commit c40d5468f8c7e0b037e0c46b78c2b88642d3e310 diff --git a/Makefile b/Makefile index 4b10a77233..2a980336e8 100644 --- a/Makefile +++ b/Makefile @@ -363,6 +363,16 @@ check-linux: clang-tools-linux check-macos: clang-tools-macos scripts/clang-tools.sh $(MACOS_OUTPUT_PATH)/$(BUILDTYPE) --diff +fuzz-%: platform/macos/platform.gyp $(MACOS_OUTPUT_PATH)/config.gypi + .mason/mason install afl 2.19b + .mason/mason install clang 3.8.0 + CXX=afl-clang-fast++ CC=afl-clang-fast $(GYP) -f ninja -I $(MACOS_OUTPUT_PATH)/config.gypi \ + --generator-output=$(MACOS_OUTPUT_PATH)/fuzz $< + PATH="`.mason/mason prefix afl 2.19b`/bin:`.mason/mason prefix clang 3.8.0`/bin:${PATH}" \ + AFL_PATH=`.mason/mason prefix afl 2.19b`/lib/afl deps/ninja/ninja-macos -C $(MACOS_OUTPUT_PATH)/fuzz/$(BUILDTYPE) fuzz + scripts/fuzz.sh $(MACOS_OUTPUT_PATH)/fuzz/$(BUILDTYPE) $* + + #### Miscellaneous targets ##################################################### style-code: diff --git a/platform/linux/platform.gyp b/platform/linux/platform.gyp index 7adf5337c4..5d3ab8b592 100644 --- a/platform/linux/platform.gyp +++ b/platform/linux/platform.gyp @@ -34,6 +34,25 @@ '../../test/src/main.cpp', ], }, + { + 'target_name': 'fuzz', + 'type': 'executable', + + 'dependencies': [ + 'test-lib', + 'platform-lib', + 'copy_certificate_bundle', + ], + + 'include_dirs': [ + '../../include', + '../../src', + ], + + 'sources': [ + '../../test/src/fuzz.cpp', + ], + }, { 'target_name': 'benchmark', 'type': 'executable', diff --git a/platform/macos/platform.gyp b/platform/macos/platform.gyp index 68ab4ce467..9d64c956cf 100644 --- a/platform/macos/platform.gyp +++ b/platform/macos/platform.gyp @@ -26,6 +26,42 @@ 'sources': [ '../../test/src/main.cpp', ], + }, + { + 'target_name': 'fuzz', + 'type': 'executable', + + 'dependencies': [ + 'platform-lib', + ], + + 'include_dirs': [ + '../../include', + '../../src', + ], + + 'sources': [ + '../../test/src/fuzz.cpp', + ], + + 'variables': { + 'cflags_cc': [ + '<@(rapidjson_cflags)', + '<@(variant_cflags)', + '<@(geometry_cflags)', + '<@(protozero_cflags)', + ], + }, + + 'conditions': [ + ['OS == "mac"', { + 'xcode_settings': { + 'OTHER_CPLUSPLUSFLAGS': [ '<@(cflags_cc)' ], + }, + }, { + 'cflags_cc': [ '<@(cflags_cc)' ], + }], + ], }, { 'target_name': 'benchmark', diff --git a/scripts/export-xcconfig.py b/scripts/export-xcconfig.py index 75fe933e24..ae9fa9c6bb 100755 --- a/scripts/export-xcconfig.py +++ b/scripts/export-xcconfig.py @@ -1,13 +1,18 @@ #!/usr/bin/env python -import sys, ast, re +import sys, ast, re, os ILLEGAL_CHAR_RE = re.compile(r"\W") +env_vars = ['CC', 'CXX'] + def main(): with open(sys.argv[1], "r") as in_file, open(sys.argv[2], "w") as out_file: config = ast.literal_eval(in_file.read()) variables = ["// Do not edit -- generated by export-xcconfig.py\n"] + for var in env_vars: + if os.environ.get(var): + variables.append("%s = %s\n" % (var, os.environ.get(var))) assert(type(config) is dict) assert(type(config["variables"]) is dict) for variable, flags in config["variables"].iteritems(): diff --git a/scripts/fuzz.sh b/scripts/fuzz.sh new file mode 100755 index 0000000000..047e55cad7 --- /dev/null +++ b/scripts/fuzz.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +export PATH="`pwd`/.mason:${PATH}" MASON_DIR="`pwd`/.mason" + +AFL_PREFIX=$(mason prefix afl 2.19b) +AFL_FUZZ=${AFL_PREFIX}/bin/afl-fuzz + + +if [ -z "$2" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ ! -d "test/fuzz-input/$2" ]; then + echo "Fuzzing data for '$2' does not exist." + exit 1 +fi + +FUZZ=$1/fuzz +${FUZZ} $2 < /dev/null + +rm -rf test/fuzz-result/$2 +mkdir -p test/fuzz-result/$2 + +${AFL_FUZZ} -i test/fuzz-input/$2 -o test/fuzz-result/$2 ${FUZZ} $2 diff --git a/src/mbgl/tile/vector_tile.cpp b/src/mbgl/tile/vector_tile.cpp index 6a3c51c05d..784b6cb1ab 100644 --- a/src/mbgl/tile/vector_tile.cpp +++ b/src/mbgl/tile/vector_tile.cpp @@ -1,10 +1,8 @@ #include +#include #include -#include #include -#include - #include #include #include @@ -12,62 +10,6 @@ namespace mbgl { -class VectorTileLayer; - -using pbf_iter_type = protozero::pbf_reader::const_uint32_iterator; -using packed_iter_type = std::pair; - -class VectorTileFeature : public GeometryTileFeature { -public: - VectorTileFeature(protozero::pbf_reader, const VectorTileLayer&); - - FeatureType getType() const override { return type; } - optional getValue(const std::string&) const override; - std::unordered_map getProperties() const override; - optional getID() const override; - GeometryCollection getGeometries() const override; - -private: - const VectorTileLayer& layer; - optional id; - FeatureType type = FeatureType::Unknown; - packed_iter_type tags_iter; - packed_iter_type geometry_iter; -}; - -class VectorTileLayer : public GeometryTileLayer { -public: - VectorTileLayer(protozero::pbf_reader); - - std::size_t featureCount() const override { return features.size(); } - util::ptr getFeature(std::size_t) const override; - std::string getName() const override; - -private: - friend class VectorTileData; - friend class VectorTileFeature; - - std::string name; - uint32_t version = 1; - uint32_t extent = 4096; - std::map keysMap; - std::vector> keys; - std::vector values; - std::vector features; -}; - -class VectorTileData : public GeometryTileData { -public: - VectorTileData(std::shared_ptr data); - - util::ptr getLayer(const std::string&) const override; - -private: - std::shared_ptr data; - mutable bool parsed = false; - mutable std::map> layers; -}; - VectorTile::VectorTile(const OverscaledTileID& id_, std::string sourceID_, const style::UpdateParameters& parameters, @@ -89,215 +31,4 @@ void VectorTile::setData(std::shared_ptr data_, GeometryTile::setData(data_ ? std::make_unique(data_) : nullptr); } -Value parseValue(protozero::pbf_reader data) { - while (data.next()) - { - switch (data.tag()) { - case 1: // string_value - return data.get_string(); - case 2: // float_value - return static_cast(data.get_float()); - case 3: // double_value - return data.get_double(); - case 4: // int_value - return data.get_int64(); - case 5: // uint_value - return data.get_uint64(); - case 6: // sint_value - return data.get_sint64(); - case 7: // bool_value - return data.get_bool(); - default: - data.skip(); - break; - } - } - return false; -} - -VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, const VectorTileLayer& layer_) - : layer(layer_) { - while (feature_pbf.next()) { - switch (feature_pbf.tag()) { - case 1: // id - id = { feature_pbf.get_uint64() }; - break; - case 2: // tags - tags_iter = feature_pbf.get_packed_uint32(); - break; - case 3: // type - type = static_cast(feature_pbf.get_enum()); - break; - case 4: // geometry - geometry_iter = feature_pbf.get_packed_uint32(); - break; - default: - feature_pbf.skip(); - break; - } - } -} - -optional VectorTileFeature::getValue(const std::string& key) const { - auto keyIter = layer.keysMap.find(key); - if (keyIter == layer.keysMap.end()) { - return optional(); - } - - auto start_itr = tags_iter.first; - const auto & end_itr = tags_iter.second; - while (start_itr != end_itr) { - uint32_t tag_key = static_cast(*start_itr++); - - if (layer.keysMap.size() <= tag_key) { - throw std::runtime_error("feature referenced out of range key"); - } - - if (start_itr == end_itr) { - throw std::runtime_error("uneven number of feature tag ids"); - } - - uint32_t tag_val = static_cast(*start_itr++);; - if (layer.values.size() <= tag_val) { - throw std::runtime_error("feature referenced out of range value"); - } - - if (tag_key == keyIter->second) { - return layer.values[tag_val]; - } - } - - return optional(); -} - -std::unordered_map VectorTileFeature::getProperties() const { - std::unordered_map properties; - auto start_itr = tags_iter.first; - const auto & end_itr = tags_iter.second; - while (start_itr != end_itr) { - uint32_t tag_key = static_cast(*start_itr++); - if (start_itr == end_itr) { - throw std::runtime_error("uneven number of feature tag ids"); - } - uint32_t tag_val = static_cast(*start_itr++); - properties[layer.keys.at(tag_key)] = layer.values.at(tag_val); - } - return properties; -} - -optional VectorTileFeature::getID() const { - return id; -} - -GeometryCollection VectorTileFeature::getGeometries() const { - uint8_t cmd = 1; - uint32_t length = 0; - int32_t x = 0; - int32_t y = 0; - const float scale = float(util::EXTENT) / layer.extent; - - GeometryCollection lines; - - lines.emplace_back(); - GeometryCoordinates* line = &lines.back(); - - auto g_itr = geometry_iter; - while (g_itr.first != g_itr.second) { - if (length == 0) { - uint32_t cmd_length = static_cast(*g_itr.first++); - cmd = cmd_length & 0x7; - length = cmd_length >> 3; - } - - --length; - - if (cmd == 1 || cmd == 2) { - x += protozero::decode_zigzag32(static_cast(*g_itr.first++)); - y += protozero::decode_zigzag32(static_cast(*g_itr.first++)); - - if (cmd == 1 && !line->empty()) { // moveTo - lines.emplace_back(); - line = &lines.back(); - } - - line->emplace_back(::round(x * scale), ::round(y * scale)); - - } else if (cmd == 7) { // closePolygon - if (!line->empty()) { - line->push_back((*line)[0]); - } - - } else { - throw std::runtime_error("unknown command"); - } - } - - if (layer.version >= 2 || type != FeatureType::Polygon) { - return lines; - } - - return fixupPolygons(lines); -} - -VectorTileData::VectorTileData(std::shared_ptr data_) - : data(std::move(data_)) { -} - -util::ptr VectorTileData::getLayer(const std::string& name) const { - if (!parsed) { - parsed = true; - protozero::pbf_reader tile_pbf(*data); - while (tile_pbf.next(3)) { - util::ptr layer = std::make_shared(tile_pbf.get_message()); - layers.emplace(layer->name, layer); - } - } - - auto layer_it = layers.find(name); - if (layer_it != layers.end()) { - return layer_it->second; - } - - return nullptr; -} - -VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf) { - while (layer_pbf.next()) { - switch (layer_pbf.tag()) { - case 1: // name - name = layer_pbf.get_string(); - break; - case 2: // feature - features.push_back(layer_pbf.get_message()); - break; - case 3: // keys - { - auto iter = keysMap.emplace(layer_pbf.get_string(), keysMap.size()); - keys.emplace_back(std::reference_wrapper(iter.first->first)); - } - break; - case 4: // values - values.emplace_back(parseValue(layer_pbf.get_message())); - break; - case 5: // extent - extent = layer_pbf.get_uint32(); - break; - case 15: // version - version = layer_pbf.get_uint32(); - break; - default: - layer_pbf.skip(); - break; - } - } -} - -util::ptr VectorTileLayer::getFeature(std::size_t i) const { - return std::make_shared(features.at(i), *this); -} - -std::string VectorTileLayer::getName() const { - return name; -} - } // namespace mbgl diff --git a/src/mbgl/tile/vector_tile_data.cpp b/src/mbgl/tile/vector_tile_data.cpp new file mode 100644 index 0000000000..fd15dd939d --- /dev/null +++ b/src/mbgl/tile/vector_tile_data.cpp @@ -0,0 +1,218 @@ +#include + +namespace mbgl { + +class VectorTileLayer; + +Value parseValue(protozero::pbf_reader data) { + while (data.next()) { + switch (data.tag()) { + case 1: // string_value + return data.get_string(); + case 2: // float_value + return static_cast(data.get_float()); + case 3: // double_value + return data.get_double(); + case 4: // int_value + return data.get_int64(); + case 5: // uint_value + return data.get_uint64(); + case 6: // sint_value + return data.get_sint64(); + case 7: // bool_value + return data.get_bool(); + default: + data.skip(); + break; + } + } + return false; +} + +VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, + const VectorTileLayer& layer_) + : layer(layer_) { + while (feature_pbf.next()) { + switch (feature_pbf.tag()) { + case 1: // id + id = { feature_pbf.get_uint64() }; + break; + case 2: // tags + tags_iter = feature_pbf.get_packed_uint32(); + break; + case 3: // type + type = static_cast(feature_pbf.get_enum()); + break; + case 4: // geometry + geometry_iter = feature_pbf.get_packed_uint32(); + break; + default: + feature_pbf.skip(); + break; + } + } +} + +optional VectorTileFeature::getValue(const std::string& key) const { + auto keyIter = layer.keysMap.find(key); + if (keyIter == layer.keysMap.end()) { + return optional(); + } + + auto start_itr = tags_iter.first; + const auto& end_itr = tags_iter.second; + while (start_itr != end_itr) { + uint32_t tag_key = static_cast(*start_itr++); + + if (layer.keysMap.size() <= tag_key) { + throw std::runtime_error("feature referenced out of range key"); + } + + if (start_itr == end_itr) { + throw std::runtime_error("uneven number of feature tag ids"); + } + + uint32_t tag_val = static_cast(*start_itr++); + ; + if (layer.values.size() <= tag_val) { + throw std::runtime_error("feature referenced out of range value"); + } + + if (tag_key == keyIter->second) { + return layer.values[tag_val]; + } + } + + return optional(); +} + +std::unordered_map VectorTileFeature::getProperties() const { + std::unordered_map properties; + auto start_itr = tags_iter.first; + const auto& end_itr = tags_iter.second; + while (start_itr != end_itr) { + uint32_t tag_key = static_cast(*start_itr++); + if (start_itr == end_itr) { + throw std::runtime_error("uneven number of feature tag ids"); + } + uint32_t tag_val = static_cast(*start_itr++); + properties[layer.keys.at(tag_key)] = layer.values.at(tag_val); + } + return properties; +} + +optional VectorTileFeature::getID() const { + return id; +} + +GeometryCollection VectorTileFeature::getGeometries() const { + uint8_t cmd = 1; + uint32_t length = 0; + int32_t x = 0; + int32_t y = 0; + const float scale = float(util::EXTENT) / layer.extent; + + GeometryCollection lines; + + lines.emplace_back(); + GeometryCoordinates* line = &lines.back(); + + auto g_itr = geometry_iter; + while (g_itr.first != g_itr.second) { + if (length == 0) { + uint32_t cmd_length = static_cast(*g_itr.first++); + cmd = cmd_length & 0x7; + length = cmd_length >> 3; + } + + --length; + + if (cmd == 1 || cmd == 2) { + x += protozero::decode_zigzag32(static_cast(*g_itr.first++)); + y += protozero::decode_zigzag32(static_cast(*g_itr.first++)); + + if (cmd == 1 && !line->empty()) { // moveTo + lines.emplace_back(); + line = &lines.back(); + } + + line->emplace_back(::round(x * scale), ::round(y * scale)); + + } else if (cmd == 7) { // closePolygon + if (!line->empty()) { + line->push_back((*line)[0]); + } + + } else { + throw std::runtime_error("unknown command"); + } + } + + if (layer.version >= 2 || type != FeatureType::Polygon) { + return lines; + } + + return fixupPolygons(lines); +} + +VectorTileData::VectorTileData(std::shared_ptr data_) : data(std::move(data_)) { +} + +util::ptr VectorTileData::getLayer(const std::string& name) const { + if (!parsed) { + parsed = true; + protozero::pbf_reader tile_pbf(*data); + while (tile_pbf.next(3)) { + util::ptr layer = + std::make_shared(tile_pbf.get_message()); + layers.emplace(layer->name, layer); + } + } + + auto layer_it = layers.find(name); + if (layer_it != layers.end()) { + return layer_it->second; + } + + return nullptr; +} + +VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf) { + while (layer_pbf.next()) { + switch (layer_pbf.tag()) { + case 1: // name + name = layer_pbf.get_string(); + break; + case 2: // feature + features.push_back(layer_pbf.get_message()); + break; + case 3: // keys + { + auto iter = keysMap.emplace(layer_pbf.get_string(), keysMap.size()); + keys.emplace_back(std::reference_wrapper(iter.first->first)); + } break; + case 4: // values + values.emplace_back(parseValue(layer_pbf.get_message())); + break; + case 5: // extent + extent = layer_pbf.get_uint32(); + break; + case 15: // version + version = layer_pbf.get_uint32(); + break; + default: + layer_pbf.skip(); + break; + } + } +} + +util::ptr VectorTileLayer::getFeature(std::size_t i) const { + return std::make_shared(features.at(i), *this); +} + +std::string VectorTileLayer::getName() const { + return name; +} + +} // namespace mbgl diff --git a/src/mbgl/tile/vector_tile_data.hpp b/src/mbgl/tile/vector_tile_data.hpp new file mode 100644 index 0000000000..8c251289a7 --- /dev/null +++ b/src/mbgl/tile/vector_tile_data.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +namespace mbgl { + +class VectorTileLayer; + +namespace { + +using pbf_iter_type = protozero::pbf_reader::const_uint32_iterator; +using packed_iter_type = std::pair; + +} // namespace + +class VectorTileFeature : public GeometryTileFeature { +public: + VectorTileFeature(protozero::pbf_reader, const VectorTileLayer&); + + FeatureType getType() const override { + return type; + } + optional getValue(const std::string&) const override; + std::unordered_map getProperties() const override; + optional getID() const override; + GeometryCollection getGeometries() const override; + +private: + const VectorTileLayer& layer; + optional id; + FeatureType type = FeatureType::Unknown; + packed_iter_type tags_iter; + packed_iter_type geometry_iter; +}; + +class VectorTileLayer : public GeometryTileLayer { +public: + VectorTileLayer(protozero::pbf_reader); + + std::size_t featureCount() const override { + return features.size(); + } + util::ptr getFeature(std::size_t) const override; + std::string getName() const override; + +private: + friend class VectorTileData; + friend class VectorTileFeature; + + std::string name; + uint32_t version = 1; + uint32_t extent = 4096; + std::map keysMap; + std::vector> keys; + std::vector values; + std::vector features; +}; + +class VectorTileData : public GeometryTileData { +public: + VectorTileData(std::shared_ptr data); + + util::ptr getLayer(const std::string&) const override; + +private: + std::shared_ptr data; + mutable bool parsed = false; + mutable std::map> layers; +}; + +} // namespace mbgl diff --git a/test/fixtures/tile_parser/id:000000,sig:06,src:000000,op:havoc,rep:16 b/test/fixtures/tile_parser/id:000000,sig:06,src:000000,op:havoc,rep:16 new file mode 100644 index 0000000000..ccb7ded269 --- /dev/null +++ b/test/fixtures/tile_parser/id:000000,sig:06,src:000000,op:havoc,rep:16 @@ -0,0 +1 @@ +Ž \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000001,sig:06,src:000000,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000001,sig:06,src:000000,op:havoc,rep:8 new file mode 100644 index 0000000000..72f454b126 --- /dev/null +++ b/test/fixtures/tile_parser/id:000001,sig:06,src:000000,op:havoc,rep:8 @@ -0,0 +1 @@ +ñ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000002,sig:06,src:000000,op:havoc,rep:16 b/test/fixtures/tile_parser/id:000002,sig:06,src:000000,op:havoc,rep:16 new file mode 100644 index 0000000000..2af1d37e32 --- /dev/null +++ b/test/fixtures/tile_parser/id:000002,sig:06,src:000000,op:havoc,rep:16 @@ -0,0 +1 @@ +H½ÿÙƒ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000003,sig:06,src:000000,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000003,sig:06,src:000000,op:havoc,rep:32 new file mode 100644 index 0000000000..41b32e1e23 --- /dev/null +++ b/test/fixtures/tile_parser/id:000003,sig:06,src:000000,op:havoc,rep:32 @@ -0,0 +1 @@ +°Ã \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000004,sig:06,src:000000,op:havoc,rep:16 b/test/fixtures/tile_parser/id:000004,sig:06,src:000000,op:havoc,rep:16 new file mode 100644 index 0000000000..4da31b8a75 --- /dev/null +++ b/test/fixtures/tile_parser/id:000004,sig:06,src:000000,op:havoc,rep:16 @@ -0,0 +1 @@ +òò" \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000005,sig:06,src:000000,op:havoc,rep:2 b/test/fixtures/tile_parser/id:000005,sig:06,src:000000,op:havoc,rep:2 new file mode 100644 index 0000000000..33a597e73f --- /dev/null +++ b/test/fixtures/tile_parser/id:000005,sig:06,src:000000,op:havoc,rep:2 @@ -0,0 +1 @@ +ý Ñ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000006,sig:06,src:000000,op:havoc,rep:64 b/test/fixtures/tile_parser/id:000006,sig:06,src:000000,op:havoc,rep:64 new file mode 100644 index 0000000000..c96ab3cc70 --- /dev/null +++ b/test/fixtures/tile_parser/id:000006,sig:06,src:000000,op:havoc,rep:64 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000007,sig:06,src:000000,op:havoc,rep:2 b/test/fixtures/tile_parser/id:000007,sig:06,src:000000,op:havoc,rep:2 new file mode 100644 index 0000000000..ca0aef5fc6 --- /dev/null +++ b/test/fixtures/tile_parser/id:000007,sig:06,src:000000,op:havoc,rep:2 @@ -0,0 +1 @@ +î/ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000008,sig:06,src:000000,op:havoc,rep:16 b/test/fixtures/tile_parser/id:000008,sig:06,src:000000,op:havoc,rep:16 new file mode 100644 index 0000000000..86d54ed2f7 Binary files /dev/null and b/test/fixtures/tile_parser/id:000008,sig:06,src:000000,op:havoc,rep:16 differ diff --git a/test/fixtures/tile_parser/id:000009,sig:06,src:000000,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000009,sig:06,src:000000,op:havoc,rep:8 new file mode 100644 index 0000000000..f88e12bd45 Binary files /dev/null and b/test/fixtures/tile_parser/id:000009,sig:06,src:000000,op:havoc,rep:8 differ diff --git a/test/fixtures/tile_parser/id:000010,sig:06,src:000000,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000010,sig:06,src:000000,op:havoc,rep:32 new file mode 100644 index 0000000000..501a6bbaf1 --- /dev/null +++ b/test/fixtures/tile_parser/id:000010,sig:06,src:000000,op:havoc,rep:32 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000011,sig:06,src:000000,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000011,sig:06,src:000000,op:havoc,rep:8 new file mode 100644 index 0000000000..34931bd1f2 Binary files /dev/null and b/test/fixtures/tile_parser/id:000011,sig:06,src:000000,op:havoc,rep:8 differ diff --git a/test/fixtures/tile_parser/id:000012,sig:06,src:000000,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000012,sig:06,src:000000,op:havoc,rep:32 new file mode 100644 index 0000000000..241bb5b843 --- /dev/null +++ b/test/fixtures/tile_parser/id:000012,sig:06,src:000000,op:havoc,rep:32 @@ -0,0 +1 @@ +ÿçþ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000013,sig:06,src:000000,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000013,sig:06,src:000000,op:havoc,rep:8 new file mode 100644 index 0000000000..2a6d94d76f --- /dev/null +++ b/test/fixtures/tile_parser/id:000013,sig:06,src:000000,op:havoc,rep:8 @@ -0,0 +1 @@ +ïdÍÍ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000014,sig:06,src:000000,op:havoc,rep:128 b/test/fixtures/tile_parser/id:000014,sig:06,src:000000,op:havoc,rep:128 new file mode 100644 index 0000000000..5cd97c70df --- /dev/null +++ b/test/fixtures/tile_parser/id:000014,sig:06,src:000000,op:havoc,rep:128 @@ -0,0 +1 @@ +ÿÿ=ÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000015,sig:06,src:000000,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000015,sig:06,src:000000,op:havoc,rep:32 new file mode 100644 index 0000000000..ba5099443d Binary files /dev/null and b/test/fixtures/tile_parser/id:000015,sig:06,src:000000,op:havoc,rep:32 differ diff --git a/test/fixtures/tile_parser/id:000016,sig:06,src:000000,op:havoc,rep:2 b/test/fixtures/tile_parser/id:000016,sig:06,src:000000,op:havoc,rep:2 new file mode 100644 index 0000000000..21ade492ec Binary files /dev/null and b/test/fixtures/tile_parser/id:000016,sig:06,src:000000,op:havoc,rep:2 differ diff --git a/test/fixtures/tile_parser/id:000017,sig:06,src:000000,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000017,sig:06,src:000000,op:havoc,rep:8 new file mode 100644 index 0000000000..116343b772 --- /dev/null +++ b/test/fixtures/tile_parser/id:000017,sig:06,src:000000,op:havoc,rep:8 @@ -0,0 +1 @@ + çÑÕÑÑÛÑ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000018,sig:06,src:000000,op:havoc,rep:128 b/test/fixtures/tile_parser/id:000018,sig:06,src:000000,op:havoc,rep:128 new file mode 100644 index 0000000000..343c691276 --- /dev/null +++ b/test/fixtures/tile_parser/id:000018,sig:06,src:000000,op:havoc,rep:128 @@ -0,0 +1 @@ +5*ÌÌÌÌ*ÌÌÌÌÌÌ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000019,sig:06,src:000000,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000019,sig:06,src:000000,op:havoc,rep:8 new file mode 100644 index 0000000000..3d36333ec4 Binary files /dev/null and b/test/fixtures/tile_parser/id:000019,sig:06,src:000000,op:havoc,rep:8 differ diff --git a/test/fixtures/tile_parser/id:000020,sig:06,src:000000,op:havoc,rep:16 b/test/fixtures/tile_parser/id:000020,sig:06,src:000000,op:havoc,rep:16 new file mode 100644 index 0000000000..37bed93971 Binary files /dev/null and b/test/fixtures/tile_parser/id:000020,sig:06,src:000000,op:havoc,rep:16 differ diff --git a/test/fixtures/tile_parser/id:000021,sig:06,src:000000,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000021,sig:06,src:000000,op:havoc,rep:32 new file mode 100644 index 0000000000..f64e344494 Binary files /dev/null and b/test/fixtures/tile_parser/id:000021,sig:06,src:000000,op:havoc,rep:32 differ diff --git a/test/fixtures/tile_parser/id:000022,sig:06,src:000000,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000022,sig:06,src:000000,op:havoc,rep:32 new file mode 100644 index 0000000000..a952b2e8a2 Binary files /dev/null and b/test/fixtures/tile_parser/id:000022,sig:06,src:000000,op:havoc,rep:32 differ diff --git a/test/fixtures/tile_parser/id:000023,sig:06,src:000010,op:arith8,pos:2,val:-2 b/test/fixtures/tile_parser/id:000023,sig:06,src:000010,op:arith8,pos:2,val:-2 new file mode 100644 index 0000000000..d9c34ae208 --- /dev/null +++ b/test/fixtures/tile_parser/id:000023,sig:06,src:000010,op:arith8,pos:2,val:-2 @@ -0,0 +1 @@ +þ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000024,sig:06,src:000010,op:arith8,pos:2,val:-3 b/test/fixtures/tile_parser/id:000024,sig:06,src:000010,op:arith8,pos:2,val:-3 new file mode 100644 index 0000000000..fc3a9a4aac --- /dev/null +++ b/test/fixtures/tile_parser/id:000024,sig:06,src:000010,op:arith8,pos:2,val:-3 @@ -0,0 +1 @@ +ý \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000025,sig:06,src:000010,op:arith8,pos:2,val:-4 b/test/fixtures/tile_parser/id:000025,sig:06,src:000010,op:arith8,pos:2,val:-4 new file mode 100644 index 0000000000..392605a5d3 --- /dev/null +++ b/test/fixtures/tile_parser/id:000025,sig:06,src:000010,op:arith8,pos:2,val:-4 @@ -0,0 +1 @@ +ü \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000026,sig:06,src:000010,op:arith8,pos:2,val:+9 b/test/fixtures/tile_parser/id:000026,sig:06,src:000010,op:arith8,pos:2,val:+9 new file mode 100644 index 0000000000..97a3f5edc5 --- /dev/null +++ b/test/fixtures/tile_parser/id:000026,sig:06,src:000010,op:arith8,pos:2,val:+9 @@ -0,0 +1 @@ +  \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000027,sig:06,src:000010,op:arith8,pos:2,val:+17 b/test/fixtures/tile_parser/id:000027,sig:06,src:000010,op:arith8,pos:2,val:+17 new file mode 100644 index 0000000000..fbe476b76e --- /dev/null +++ b/test/fixtures/tile_parser/id:000027,sig:06,src:000010,op:arith8,pos:2,val:+17 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000028,sig:06,src:000010,op:arith8,pos:2,val:+25 b/test/fixtures/tile_parser/id:000028,sig:06,src:000010,op:arith8,pos:2,val:+25 new file mode 100644 index 0000000000..a5edb71521 --- /dev/null +++ b/test/fixtures/tile_parser/id:000028,sig:06,src:000010,op:arith8,pos:2,val:+25 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000029,sig:06,src:000010,op:arith8,pos:2,val:+33 b/test/fixtures/tile_parser/id:000029,sig:06,src:000010,op:arith8,pos:2,val:+33 new file mode 100644 index 0000000000..0c081621a7 --- /dev/null +++ b/test/fixtures/tile_parser/id:000029,sig:06,src:000010,op:arith8,pos:2,val:+33 @@ -0,0 +1 @@ +! \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000030,sig:06,src:000019,op:havoc,rep:8 b/test/fixtures/tile_parser/id:000030,sig:06,src:000019,op:havoc,rep:8 new file mode 100644 index 0000000000..f85cc81fa2 Binary files /dev/null and b/test/fixtures/tile_parser/id:000030,sig:06,src:000019,op:havoc,rep:8 differ diff --git a/test/fixtures/tile_parser/id:000031,sig:06,src:000018,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000031,sig:06,src:000018,op:havoc,rep:32 new file mode 100644 index 0000000000..0150b53f47 --- /dev/null +++ b/test/fixtures/tile_parser/id:000031,sig:06,src:000018,op:havoc,rep:32 @@ -0,0 +1 @@ +èÓèèèèèèÿú€Øú \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000032,sig:06,src:000026,op:havoc,rep:128 b/test/fixtures/tile_parser/id:000032,sig:06,src:000026,op:havoc,rep:128 new file mode 100644 index 0000000000..9f46944a55 Binary files /dev/null and b/test/fixtures/tile_parser/id:000032,sig:06,src:000026,op:havoc,rep:128 differ diff --git a/test/fixtures/tile_parser/id:000033,sig:06,src:000001,op:havoc,rep:32 b/test/fixtures/tile_parser/id:000033,sig:06,src:000001,op:havoc,rep:32 new file mode 100644 index 0000000000..6a1ad321e3 Binary files /dev/null and b/test/fixtures/tile_parser/id:000033,sig:06,src:000001,op:havoc,rep:32 differ diff --git a/test/fixtures/tile_parser/id:000034,sig:06,src:000018+000004,op:splice,rep:32 b/test/fixtures/tile_parser/id:000034,sig:06,src:000018+000004,op:splice,rep:32 new file mode 100644 index 0000000000..fb9809d832 Binary files /dev/null and b/test/fixtures/tile_parser/id:000034,sig:06,src:000018+000004,op:splice,rep:32 differ diff --git a/test/fixtures/tile_parser/id:000035,sig:06,src:000007+000004,op:splice,rep:4 b/test/fixtures/tile_parser/id:000035,sig:06,src:000007+000004,op:splice,rep:4 new file mode 100644 index 0000000000..2663ab2909 --- /dev/null +++ b/test/fixtures/tile_parser/id:000035,sig:06,src:000007+000004,op:splice,rep:4 @@ -0,0 +1 @@ +@ \ No newline at end of file diff --git a/test/fixtures/tile_parser/id:000036,sig:06,src:000040+000043,op:splice,rep:8 b/test/fixtures/tile_parser/id:000036,sig:06,src:000040+000043,op:splice,rep:8 new file mode 100644 index 0000000000..dd9c1f152b Binary files /dev/null and b/test/fixtures/tile_parser/id:000036,sig:06,src:000040+000043,op:splice,rep:8 differ diff --git a/test/fixtures/tile_parser/id:000037,sig:06,src:000044,op:havoc,rep:2 b/test/fixtures/tile_parser/id:000037,sig:06,src:000044,op:havoc,rep:2 new file mode 100644 index 0000000000..6f8c364dd8 Binary files /dev/null and b/test/fixtures/tile_parser/id:000037,sig:06,src:000044,op:havoc,rep:2 differ diff --git a/test/fixtures/tile_parser/id:000038,sig:06,src:000002+000010,op:splice,rep:2 b/test/fixtures/tile_parser/id:000038,sig:06,src:000002+000010,op:splice,rep:2 new file mode 100644 index 0000000000..3e1b32aa0a Binary files /dev/null and b/test/fixtures/tile_parser/id:000038,sig:06,src:000002+000010,op:splice,rep:2 differ diff --git a/test/fuzz-input/canonicalize-tile-url/access_token b/test/fuzz-input/canonicalize-tile-url/access_token new file mode 100644 index 0000000000..73047054cd --- /dev/null +++ b/test/fuzz-input/canonicalize-tile-url/access_token @@ -0,0 +1 @@ +http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.vector.pbf?access_token=key \ No newline at end of file diff --git a/test/fuzz-input/canonicalize-tile-url/image b/test/fuzz-input/canonicalize-tile-url/image new file mode 100644 index 0000000000..8497f1fd9b --- /dev/null +++ b/test/fuzz-input/canonicalize-tile-url/image @@ -0,0 +1 @@ +http://api.mapbox.com/v4/a.b/{z}/{x}/{y}.png \ No newline at end of file diff --git a/test/fuzz-input/glyphs-url/boxmap b/test/fuzz-input/glyphs-url/boxmap new file mode 100644 index 0000000000..84bafe746c --- /dev/null +++ b/test/fuzz-input/glyphs-url/boxmap @@ -0,0 +1 @@ +mapbox://fonts/boxmap/{fontstack}/{range}.pbf \ No newline at end of file diff --git a/test/fuzz-input/glyphs-url/comic-sans b/test/fuzz-input/glyphs-url/comic-sans new file mode 100644 index 0000000000..7e4b223250 --- /dev/null +++ b/test/fuzz-input/glyphs-url/comic-sans @@ -0,0 +1 @@ +mapbox://fonts/boxmap/Comic%20Sans/0-255.pbf \ No newline at end of file diff --git a/test/fuzz-input/parse-filter/all-in-class.json b/test/fuzz-input/parse-filter/all-in-class.json new file mode 100644 index 0000000000..02a2ca761d --- /dev/null +++ b/test/fuzz-input/parse-filter/all-in-class.json @@ -0,0 +1 @@ +["all",["in","class","path"]] \ No newline at end of file diff --git a/test/fuzz-input/parse-filter/f-gte-array.json b/test/fuzz-input/parse-filter/f-gte-array.json new file mode 100644 index 0000000000..f04d09b867 --- /dev/null +++ b/test/fuzz-input/parse-filter/f-gte-array.json @@ -0,0 +1 @@ +[">=", "f", []] \ No newline at end of file diff --git a/test/fuzz-input/parse-filter/has-foo.json b/test/fuzz-input/parse-filter/has-foo.json new file mode 100644 index 0000000000..37c94d6003 --- /dev/null +++ b/test/fuzz-input/parse-filter/has-foo.json @@ -0,0 +1 @@ +["has", "foo"] \ No newline at end of file diff --git a/test/fuzz-input/parse-filter/id-lte-4.json b/test/fuzz-input/parse-filter/id-lte-4.json new file mode 100644 index 0000000000..23fc29c020 --- /dev/null +++ b/test/fuzz-input/parse-filter/id-lte-4.json @@ -0,0 +1 @@ +["<=", "$id", 4] \ No newline at end of file diff --git a/test/fuzz-input/parse-style/empty.json b/test/fuzz-input/parse-style/empty.json new file mode 100644 index 0000000000..222ac82bf8 --- /dev/null +++ b/test/fuzz-input/parse-style/empty.json @@ -0,0 +1,31 @@ +{ + "version": 8, + "name": "Empty", + "sources": { + "mapbox": { + "type": "vector", + "maxzoom": 15, + "tiles": [ + "./fixtures/tiles/{z}-{x}-{y}.vector.pbf" + ] + } + }, + "layers": [ + { + "id": "background", + "type": "background", + "paint": { + "background-color": "white" + } + }, + { + "id": "water", + "type": "fill", + "source": "mapbox", + "source-layer": "water", + "paint": { + "fill-color": "blue" + } + } + ] +} diff --git a/test/fuzz-input/parse-tile/0-0-0.vector.pbf b/test/fuzz-input/parse-tile/0-0-0.vector.pbf new file mode 100644 index 0000000000..a0f049ad43 Binary files /dev/null and b/test/fuzz-input/parse-tile/0-0-0.vector.pbf differ diff --git a/test/fuzz-input/source-url/user.map b/test/fuzz-input/source-url/user.map new file mode 100644 index 0000000000..33dedc50b8 --- /dev/null +++ b/test/fuzz-input/source-url/user.map @@ -0,0 +1 @@ +mapbox://user.map \ No newline at end of file diff --git a/test/fuzz-input/sprite-url/streets-draft-png b/test/fuzz-input/sprite-url/streets-draft-png new file mode 100644 index 0000000000..f7ab08129b --- /dev/null +++ b/test/fuzz-input/sprite-url/streets-draft-png @@ -0,0 +1 @@ +mapbox://sprites/mapbox/streets-v8/draft.png \ No newline at end of file diff --git a/test/fuzz-input/sprite-url/streets-json b/test/fuzz-input/sprite-url/streets-json new file mode 100644 index 0000000000..b58e7e9e78 --- /dev/null +++ b/test/fuzz-input/sprite-url/streets-json @@ -0,0 +1 @@ +mapbox://sprites/mapbox/streets-v8.json \ No newline at end of file diff --git a/test/fuzz-input/sprite-url/streets-png b/test/fuzz-input/sprite-url/streets-png new file mode 100644 index 0000000000..f05374a98c --- /dev/null +++ b/test/fuzz-input/sprite-url/streets-png @@ -0,0 +1 @@ +mapbox://sprites/mapbox/streets-v8@2x.png \ No newline at end of file diff --git a/test/src/fuzz.cpp b/test/src/fuzz.cpp new file mode 100644 index 0000000000..b498bc5b48 --- /dev/null +++ b/test/src/fuzz.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include + +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +typedef void (*Callback)(std::string); +static const std::unordered_map fuzzers{ + { "source-url", + [](std::string input) { mbgl::util::mapbox::normalizeSourceURL(input, "key"); } }, + { "style-url", [](std::string input) { mbgl::util::mapbox::normalizeStyleURL(input, "key"); } }, + { "sprite-url", + [](std::string input) { mbgl::util::mapbox::normalizeSpriteURL(input, "key"); } }, + { "glyphs-url", + [](std::string input) { mbgl::util::mapbox::normalizeGlyphsURL(input, "key"); } }, + { "tile-url", [](std::string input) { mbgl::util::mapbox::normalizeTileURL(input, "key"); } }, + { "canonicalize-tile-url", + [](std::string input) { + mbgl::util::mapbox::canonicalizeTileURL(input, mbgl::SourceType::Raster, 42); + } }, + { "parse-style", + [](std::string input) { + mbgl::style::Parser parser; + parser.parse(input); + } }, + { "parse-filter", + [](std::string input) { + using namespace mbgl; + using namespace mbgl::style; + rapidjson::GenericDocument, rapidjson::CrtAllocator> document; + document.Parse<0>(input.c_str()); + if (!document.HasParseError()) { + conversion::convert(document); + } + } }, + { "parse-tile", + [](std::string input) { + mbgl::VectorTileData tile(std::make_shared(std::move(input))); + try { + tile.getLayer("foo"); + } catch (protozero::exception& ex) { + } + } }, +}; + +void printValidFuzzers() { + std::cerr << "Valid fuzzers are:" << std::endl; + for (auto& fuzzer : fuzzers) { + std::cerr << " - " << fuzzer.first << std::endl; + } +} + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "Usage: fuzz " << std::endl; + printValidFuzzers(); + return 1; + } + + auto it = fuzzers.find(argv[1]); + if (it == fuzzers.end()) { + std::cerr << "Fuzzer '" << argv[1] << "' does not exist." << std::endl; + printValidFuzzers(); + return 1; + } + + // Disables logging + mbgl::Log::setObserver(std::make_unique()); + + while (__AFL_LOOP(1000)) { + // Pass stdin to the function. + std::cin >> std::noskipws; + it->second({ std::istream_iterator(std::cin), std::istream_iterator() }); + } + + return 0; +} diff --git a/test/test.gypi b/test/test.gypi index 304e0e9730..48646fe97e 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -60,6 +60,7 @@ 'text/quads.cpp', 'tile/geometry_tile_data.cpp', + 'tile/vector_tile_data.cpp', 'tile/tile_id.cpp', 'storage/offline.cpp', diff --git a/test/tile/vector_tile_data.cpp b/test/tile/vector_tile_data.cpp new file mode 100644 index 0000000000..28d0e7d8f3 --- /dev/null +++ b/test/tile/vector_tile_data.cpp @@ -0,0 +1,39 @@ +#include + +#include +#include + +#include +#include + +#include + +using namespace mbgl; + +typedef std::pair Message; +typedef std::vector Messages; + +class TileParserTest : public ::testing::TestWithParam {}; + +TEST_P(TileParserTest, ParseTile) { + const std::string filename = std::string("test/fixtures/tile_parser/") + GetParam(); + + VectorTileData tile(std::make_shared(util::read_file(filename))); + tile.getLayer("foo"); +} + +INSTANTIATE_TEST_CASE_P(TileParser, TileParserTest, ::testing::ValuesIn([] { + std::vector names; + + const std::string tile_directory = "test/fixtures/tile_parser"; + DIR *dir = opendir(tile_directory.c_str()); + if (dir != nullptr) { + for (dirent *dp = nullptr; (dp = readdir(dir)) != nullptr;) { + names.push_back(dp->d_name); + } + closedir(dir); + } + + EXPECT_GT(names.size(), 0u); + return names; +}())); -- cgit v1.2.1