From 631bc4f6a3ff7518787a8a3db76b252e96198a07 Mon Sep 17 00:00:00 2001 From: Alexander Shalamov Date: Mon, 2 Sep 2019 17:46:15 +0300 Subject: [core] Add native expression test runner --- CMakeLists.txt | 1 + cmake/expression-test.cmake | 27 ++ expression-test/expression_test_logger.cpp | 177 ++++++++++ expression-test/expression_test_logger.hpp | 14 + expression-test/expression_test_parser.cpp | 536 +++++++++++++++++++++++++++++ expression-test/expression_test_parser.hpp | 94 +++++ expression-test/expression_test_runner.cpp | 303 ++++++++++++++++ expression-test/expression_test_runner.hpp | 32 ++ expression-test/main.cpp | 75 ++++ 9 files changed, 1259 insertions(+) create mode 100644 cmake/expression-test.cmake create mode 100644 expression-test/expression_test_logger.cpp create mode 100644 expression-test/expression_test_logger.hpp create mode 100644 expression-test/expression_test_parser.cpp create mode 100644 expression-test/expression_test_parser.hpp create mode 100644 expression-test/expression_test_runner.cpp create mode 100644 expression-test/expression_test_runner.hpp create mode 100644 expression-test/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index df2e4940fd..d8881f4a1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -185,6 +185,7 @@ include(cmake/core.cmake) if(COMMAND mbgl_platform_test) include(cmake/test.cmake) include(cmake/render-test.cmake) + include(cmake/expression-test.cmake) endif() if(COMMAND mbgl_platform_benchmark) diff --git a/cmake/expression-test.cmake b/cmake/expression-test.cmake new file mode 100644 index 0000000000..6c5b71a4f6 --- /dev/null +++ b/cmake/expression-test.cmake @@ -0,0 +1,27 @@ +add_executable(mbgl-expression-test + expression-test/main.cpp + expression-test/expression_test_parser.cpp + expression-test/expression_test_runner.cpp + expression-test/expression_test_logger.cpp +) + +if(APPLE) + target_link_libraries(mbgl-expression-test PRIVATE mbgl-loop-darwin) +else() + target_link_libraries(mbgl-expression-test PRIVATE mbgl-loop-uv) +endif() + +target_include_directories(mbgl-expression-test + PRIVATE src + PRIVATE expression-test + PRIVATE render-test +) + +target_link_libraries(mbgl-expression-test PRIVATE + mbgl-core + Mapbox::Base::Extras::args + Mapbox::Base::Extras::filesystem + Mapbox::Base::Extras::rapidjson +) + +add_definitions(-DTEST_RUNNER_ROOT_PATH="${CMAKE_SOURCE_DIR}") diff --git a/expression-test/expression_test_logger.cpp b/expression-test/expression_test_logger.cpp new file mode 100644 index 0000000000..9127a6d7ae --- /dev/null +++ b/expression-test/expression_test_logger.cpp @@ -0,0 +1,177 @@ +#include "expression_test_logger.hpp" +#include "expression_test_runner.hpp" +#include "filesystem.hpp" + +#include +#include + +#include + +using namespace mbgl; +using namespace std::literals; + +namespace { + +const char* resultsStyle = R"HTML( + +)HTML"; + +const char* resultsHeaderButtons = R"HTML( + + + + +)HTML"; + +const char* resultsScript = R"HTML( + +)HTML"; + +std::string createResultItem(const TestRunOutput& result, const std::string& status, bool shouldHide) { + std::ostringstream html; + html << "
\n"; + html << R"(

)" << status << " " << result.id << "

\n"; + + html << "

"s << result.expression << "

\n"s; + if (result.passed) { + html << "Serialized:

"s << result.serialized << "

\n"s; + } else { + html << "

Difference:

" << result.text << "

\n"; + } + html << "
\n"; + + return html.str(); +} + +std::string createResultPage(const TestStats& stats, bool shuffle, uint32_t seed) { + const std::size_t unsuccessfulCount = stats.errored.size() + stats.failed.size(); + const bool unsuccessful = unsuccessfulCount > 0; + std::ostringstream resultsPage; + + // Style + resultsPage << resultsStyle; + + // Header with buttons + if (unsuccessful) { + resultsPage << R"HTML(

)HTML"; + resultsPage << util::toString(unsuccessfulCount) << " tests failed."; + } else { + resultsPage << R"HTML(

)HTML"; + resultsPage << "All tests passed!"; + } + + resultsPage << resultsHeaderButtons; + + // Test sequence + { + resultsPage << "
\n"; + + // Failed tests + if (unsuccessful) { + resultsPage << "

Failed tests:"; + for (const auto& failed : stats.failed) { + resultsPage << failed.id << " "; + } + resultsPage << "

Errored tests:"; + for (const auto& errored : stats.errored) { + resultsPage << errored.id << " "; + } + resultsPage << "

\n"; + } + + // Test sequence + resultsPage << "

Test sequence: "; + for (const auto& id : stats.ids) { + resultsPage << id << " "; + } + resultsPage << "

\n"; + + // Shuffle + if (shuffle) { + resultsPage << "

Shuffle seed: " << util::toString(seed) << "

\n"; + } + + resultsPage << "
\n"; + } + + // Script + resultsPage << resultsScript; + + // Tests + resultsPage << "
\n"; + const auto appendResult = [&] (const auto& results, const std::string& status, bool hide = false) { + for (const auto& result : results) { + resultsPage << createResultItem(result, status, hide); + } + }; + + appendResult(stats.passed, "passed"s, unsuccessful); + appendResult(stats.failed, "failed"s); + appendResult(stats.errored, "errored"s); + appendResult(stats.ignorePassed, "ignored passed"s, unsuccessful); + appendResult(stats.ignoreFailed, "ignored"s, true); + resultsPage << "
\n"; + + return resultsPage.str(); +} + +} // namespace + +void printStats(const TestStats& stats) { + const std::size_t count = stats.testCount(); + if (std::size_t passedTests = stats.passed.size()) { + printf(ANSI_COLOR_GREEN "%zu passed (%.1lf%%)" ANSI_COLOR_RESET "\n", passedTests, 100.0 * passedTests / count); + } + if (std::size_t ignorePassedTests = stats.ignorePassed.size()) { + printf(ANSI_COLOR_YELLOW "%zu passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", ignorePassedTests, 100.0 * ignorePassedTests / count); + } + if (std::size_t ignoreFailedTests = stats.ignoreFailed.size()) { + printf(ANSI_COLOR_LIGHT_GRAY "%zu ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", ignoreFailedTests, 100.0 * ignoreFailedTests / count); + } + if (std::size_t failedTests = stats.failed.size()) { + printf(ANSI_COLOR_RED "%zu failed (%.1lf%%)" ANSI_COLOR_RESET "\n", failedTests, 100.0 * failedTests / count); + } + if (std::size_t erroredTests = stats.errored.size()) { + printf(ANSI_COLOR_RED "%zu errored (%.1lf%%)" ANSI_COLOR_RESET "\n", erroredTests, 100.0 * erroredTests / count); + } +} + +void writeHTMLResults(const TestStats& stats, const std::string& rootPath, bool shuffle, uint32_t seed) { + filesystem::path path = filesystem::path(rootPath) / "index.html"s; + try { + util::write_file(path.string(), createResultPage(stats, shuffle, seed)); + printf("Results at: %s\n", path.string().c_str()); + } catch (std::exception&) { + printf(ANSI_COLOR_RED "* ERROR can't write result page %s" ANSI_COLOR_RESET "\n", path.string().c_str()); + } +} diff --git a/expression-test/expression_test_logger.hpp b/expression-test/expression_test_logger.hpp new file mode 100644 index 0000000000..95b0697cb5 --- /dev/null +++ b/expression-test/expression_test_logger.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +#define ANSI_COLOR_RED "\x1b[31m" +#define ANSI_COLOR_GREEN "\x1b[32m" +#define ANSI_COLOR_YELLOW "\x1b[33m" +#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m" +#define ANSI_COLOR_RESET "\x1b[0m" + +class TestStats; + +void printStats(const TestStats&); +void writeHTMLResults(const TestStats&, const std::string&, bool, uint32_t); diff --git a/expression-test/expression_test_parser.cpp b/expression-test/expression_test_parser.cpp new file mode 100644 index 0000000000..3c194ffee0 --- /dev/null +++ b/expression-test/expression_test_parser.cpp @@ -0,0 +1,536 @@ +#include "expression_test_parser.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +using namespace mbgl; +using namespace mbgl::style; +using namespace mbgl::style::conversion; +using namespace std::literals; + +namespace { + +void writeJSON(rapidjson::PrettyWriter& writer, const Value& value) { + value.match( + [&] (const NullValue&) { writer.Null(); }, + [&] (bool b) { writer.Bool(b); }, + [&] (uint64_t u) { writer.Uint64(u); }, + [&] (int64_t i) { writer.Int64(i); }, + [&] (double d) { d == std::floor(d) ? writer.Int64(d) : writer.Double(d); }, + [&] (const std::string& s) { writer.String(s); }, + [&] (const std::vector& arr) { + writer.StartArray(); + for(const auto& item : arr) { + writeJSON(writer, item); + } + writer.EndArray(); + }, + [&] (const std::unordered_map& obj) { + writer.StartObject(); + for(const auto& entry : obj) { + writer.Key(entry.first.c_str()); + writeJSON(writer, entry.second); + } + writer.EndObject(); + } + ); +} + +using ErrorMessage = std::string; +using JSONReply = variant; +JSONReply readJson(const filesystem::path& jsonPath) { + auto maybeJSON = util::readFile(jsonPath); + if (!maybeJSON) { + return { "Unable to open file "s + jsonPath.string() }; + } + + JSDocument document; + document.Parse(*maybeJSON); + if (document.HasParseError()) { + return { formatJSONParseError(document) }; + } + + return { std::move(document) }; +} + +std::string toString(const JSValue& value) { + assert(value.IsString()); + return { value.GetString(), value.GetStringLength() }; +} + +optional toValue(const JSValue& jsvalue) { + if (jsvalue.IsNull()) { + return Value{}; + } + + if (jsvalue.IsArray()) { + std::vector values; + values.reserve(jsvalue.GetArray().Size()); + for (const auto& v : jsvalue.GetArray()) { + if (auto value = toValue(v)) { + values.emplace_back(std::move(*value)); + } + } + return {std::move(values)}; + } + + if (jsvalue.IsObject()) { + std::unordered_map value_map; + for (const auto& pair : jsvalue.GetObject()) { + if (auto value = toValue(pair.value)) { + value_map.emplace(toString(pair.name), std::move(*value)); + } + } + return {std::move(value_map)}; + } + + if (!jsvalue.IsArray() && !jsvalue.IsObject()) { + return toValue(Convertible(&jsvalue)); + } + + return nullopt; +} + +style::expression::type::Type stringToType(const std::string& type) { + using namespace style::expression; + if (type == "string"s || type == "number-format"s) { + return type::String; + } else if (type == "number"s) { + return type::Number; + } else if (type == "boolean"s) { + return type::Boolean; + } else if (type == "object"s) { + return type::Object; + } else if (type == "color"s) { + return type::Color; + } else if (type == "value"s) { + return type::Value; + } else if (type == "formatted"s) { + return type::Formatted; + } + + // Should not reach. + assert(false); + return type::Null; +} + +optional toExpressionType(const PropertySpec& spec) { + using namespace style::expression; + if (spec.type == "array") { + type::Type itemType = spec.value.empty() ? type::Value : stringToType(spec.value); + if (spec.length) { + return {type::Array(itemType, spec.length)}; + } + return {type::Array(itemType)}; + } + + if (spec.type == "enum") { + return {type::String}; + } + + return spec.type.empty() ? nullopt : optional{stringToType(spec.type)}; +} + +void parseCompiled(const JSValue& compiledValue, TestData& data) { + const auto& compiled = compiledValue.GetObject(); + assert(compiled.HasMember("result")); + assert(compiled["result"].IsString()); + const std::string& result = toString(compiled["result"]); + data.expected.compiled.success = result == "success"; + + if (compiled.HasMember("isFeatureConstant")) { + assert(compiled["isFeatureConstant"].IsBool()); + data.expected.compiled.isFeatureConstant = compiled["isFeatureConstant"].GetBool(); + } + + if (compiled.HasMember("isZoomConstant")) { + assert(compiled["isZoomConstant"].IsBool()); + data.expected.compiled.isZoomConstant = compiled["isZoomConstant"].GetBool(); + } + + if (compiled.HasMember("type")) { + assert(compiled["type"].IsString()); + data.expected.compiled.serializedType = toString(compiled["type"]); + } + + if (compiled.HasMember("errors")) { + assert(compiled["errors"].IsArray()); + for (const auto& errorVal : compiled["errors"].GetArray()) { + assert(errorVal.IsObject()); + const auto& errorObject = errorVal.GetObject(); + assert(errorObject.HasMember("key")); + assert(errorObject.HasMember("error")); + + std::unordered_map errorMap; + errorMap.emplace("key"s, Value{toString(errorObject["key"])}); + errorMap.emplace("error"s, Value{toString(errorObject["error"])}); + data.expected.compiled.errors.emplace_back(Value{std::move(errorMap)}); + } + } +} + +void parseExpected(const JSValue& expectedValue, TestData& data) { + assert(expectedValue.IsObject()); + const auto& expected = expectedValue.GetObject(); + assert(expected.HasMember("compiled")); + parseCompiled(expected["compiled"], data); + + // set outputs + if (expected.HasMember("outputs")) { + data.expected.outputs = toValue(expected["outputs"]); + } + + // set serialized + if (expected.HasMember("serialized")) { + data.expected.serialized = toValue(expected["serialized"]); + } +} + +void parsePropertySpec(const JSValue& value, TestData& data) { + const auto& propertySpec = value.GetObject(); + PropertySpec spec; + + if (propertySpec.HasMember("type")) { + assert(propertySpec["type"].IsString()); + spec.type = toString(propertySpec["type"]); + } + + if (propertySpec.HasMember("value")) { + assert(propertySpec["value"].IsString()); + spec.value = toString(propertySpec["value"]); + } + + if (propertySpec.HasMember("length")) { + assert(propertySpec["length"].IsNumber()); + spec.length = propertySpec["length"].GetDouble(); + } + + if (propertySpec.HasMember("property-type")) { + assert(propertySpec["property-type"].IsString()); + spec.isDataDriven = true; + } + + if (propertySpec.HasMember("expression")) { + assert(propertySpec["expression"].IsObject()); + spec.expression = toValue(propertySpec["expression"]); + } + + data.spec = std::move(spec); +} + +bool parseInputs(const JSValue& inputsValue, TestData& data) { + assert(inputsValue.IsArray()); + for (const auto& input : inputsValue.GetArray()) { + assert(input.IsArray()); + assert(input.Size() == 2); + assert(input[0].IsObject()); + assert(input[1].IsObject()); + + // Parse evaluation context, zoom. + optional zoom; + const auto& evaluationContext = input[0].GetObject(); + if (evaluationContext.HasMember("zoom")) { + assert(evaluationContext["zoom"].IsNumber()); + zoom = evaluationContext["zoom"].GetDouble(); + } + + // Parse heatmap density + optional heatmapDensity; + if (evaluationContext.HasMember("heatmapDensity")) { + assert(evaluationContext["heatmapDensity"].IsNumber()); + heatmapDensity = evaluationContext["heatmapDensity"].GetDouble(); + } + + // Parse feature properties + Feature feature(mapbox::geometry::point(0.0, 0.0)); + const auto& featureObject = input[1].GetObject(); + if (featureObject.HasMember("properties")) { + assert(featureObject["properties"].IsObject()); + feature.properties = mapbox::geojson::convert(featureObject["properties"]); + } + + if (featureObject.HasMember("geometry")) { + assert(featureObject["geometry"].IsObject()); + feature.geometry = mapbox::geojson::convert>(featureObject["geometry"]); + } + + if (featureObject.HasMember("id")) { + assert(featureObject["id"].IsNumber() || featureObject["id"].IsString()); + feature.id = mapbox::geojson::convert(featureObject["id"]); + } + + data.inputs.emplace_back(std::move(zoom), std::move(heatmapDensity), std::move(feature)); + } + return true; +} + +} // namespace + +std::tuple, bool, uint32_t> parseArguments(int argc, char** argv) { + args::ArgumentParser argumentParser("Mapbox GL Expression Test Runner"); + + args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" }); + args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", + { 's', "shuffle" }); + args::ValueFlag seedValue(argumentParser, "seed", "Shuffle seed (default: random)", + { "seed" }); + args::PositionalList testNameValues(argumentParser, "URL", "Test name(s)"); + + try { + argumentParser.ParseCLI(argc, argv); + } catch (const args::Help&) { + std::ostringstream stream; + stream << argumentParser; + Log::Info(Event::General, stream.str()); + exit(0); + } catch (const args::ParseError& e) { + std::ostringstream stream; + stream << argumentParser; + Log::Info(Event::General, stream.str()); + Log::Error(Event::General, e.what()); + exit(1); + } catch (const args::ValidationError& e) { + std::ostringstream stream; + stream << argumentParser; + Log::Info(Event::General, stream.str()); + Log::Error(Event::General, e.what()); + exit(2); + } + + filesystem::path rootPath {std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/expression-tests")}; + if (!filesystem::exists(rootPath)) { + Log::Error(Event::General, "Test path '%s' does not exist.", rootPath.string().c_str()); + exit(3); + } + + std::vector paths; + for (const auto& testName : args::get(testNameValues)) { + paths.emplace_back(rootPath.string() + "/" + testName); + } + + if (paths.empty()) { + paths.emplace_back(rootPath); + } + + // Recursively traverse through the test paths and collect test directories containing "test.json". + std::vector testPaths; + testPaths.reserve(paths.size()); + for (const auto& path : paths) { + if (!filesystem::exists(path)) { + Log::Warning(Event::General, "Provided test folder '%s' does not exist.", path.string().c_str()); + continue; + } + + for (auto& testPath : filesystem::recursive_directory_iterator(path)) { + if (testPath.path().filename() == "test.json") { + testPaths.emplace_back(testPath.path()); + } + } + } + + return Arguments{ std::move(rootPath), + std::move(testPaths), + shuffleFlag ? args::get(shuffleFlag) : false, + seedValue ? args::get(seedValue) : 1u }; +} + +Ignores parseExpressionIgnores() { + Ignores ignores; + const auto mainIgnoresPath = filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); + auto maybeIgnores = readJson(mainIgnoresPath); + if (!maybeIgnores.is()) { // NOLINT + return {}; + } + + for (const auto& property : maybeIgnores.get().GetObject()) { + std::string id{ toString(property.name) }; + // Keep only expression-test ignores + if (id.rfind("expression-tests", 0) != 0) { + continue; + } + std::string reason{ toString(property.value) }; + ignores.emplace_back(std::move(id), std::move(reason)); + } + + return ignores; +} + +optional parseTestData(const filesystem::path& path) { + TestData data; + auto maybeJson = readJson(path.string()); + if (!maybeJson.is()) { // NOLINT + Log::Error(Event::General, "Cannot parse test '%s'.", path.string().c_str()); + return nullopt; + } + + data.document = std::move(maybeJson.get()); + + // Check that mandatory test data members are present. + if (!data.document.HasMember("expression") || !data.document.HasMember("expected")) { + Log::Error(Event::General, "Test fixture '%s' does not contain required data.", path.string().c_str()); + return nullopt; + } + + // Parse propertySpec + if (data.document.HasMember("propertySpec")) { + assert(data.document["propertySpec"].IsObject()); + parsePropertySpec(data.document["propertySpec"], data); + } + + // Parse expected + parseExpected(data.document["expected"], data); + + // Parse inputs + if (data.document.HasMember("inputs") && !parseInputs(data.document["inputs"], data)) { + Log::Error(Event::General,"Can't convert inputs value for '%s'", path.string().c_str()); + return nullopt; + } + + return {std::move(data)}; +} + +std::string toJSON(const Value& value, unsigned indent, bool singleLine) { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + if (singleLine) { + writer.SetFormatOptions(rapidjson::kFormatSingleLineArray); + } + writer.SetIndent(' ', indent); + writeJSON(writer, value); + return buffer.GetString(); +} + +JSDocument toDocument(const Value& value) { + JSDocument document; + document.Parse(toJSON(value)); + return document; +} + +Value toValue(const Compiled& compiled) { + std::unordered_map compiledObject; + compiledObject.emplace("result", compiled.success ? "success"s : "error"s); + if (compiled.success) { + compiledObject.emplace("isFeatureConstant", compiled.isFeatureConstant); + compiledObject.emplace("isZoomConstant", compiled.isZoomConstant); + compiledObject.emplace("type", compiled.serializedType); + } else { + compiledObject.emplace("errors", compiled.errors); + } + return {std::move(compiledObject)}; +} + +// Serializes native expression types to JS format. +// Color and Formatted are exceptions. +// TODO: harmonize serialized format to remove this conversion. +optional toValue(const expression::Value& exprValue) { + return exprValue.match( + [](const Color& c) -> optional { + std::vector color { double(c.r), double(c.g), double(c.b), double(c.a) }; + return {Value{std::move(color)}}; + }, + [](const expression::Formatted& formatted) -> optional { + std::unordered_map serialized; + std::vector sections; + for (const auto& section : formatted.sections) { + std::unordered_map serializedSection; + serializedSection.emplace("text", section.text); + if (section.fontScale) { + serializedSection.emplace("scale", *section.fontScale); + } else { + serializedSection.emplace("scale", NullValue()); + } + if (section.fontStack) { + std::string fontStackString; + serializedSection.emplace("fontStack", fontStackToString(*section.fontStack)); + } else { + serializedSection.emplace("fontStack", NullValue()); + } + if (section.textColor) { + serializedSection.emplace("textColor", section.textColor->toObject()); + } else { + serializedSection.emplace("textColor", NullValue()); + } + sections.emplace_back(serializedSection); + } + serialized.emplace("sections", sections); + return {Value{std::move(serialized)}}; + }, + [](const std::vector& values) -> optional { + std::vector mbglValues; + for (const auto& value : values) { + if (auto converted = expression::fromExpressionValue(value)) { + mbglValues.emplace_back(std::move(*converted)); + } + } + return {Value{std::move(mbglValues)}}; + }, + [](const std::unordered_map& valueMap) -> optional { + std::unordered_map mbglValueMap; + for (const auto& pair : valueMap) { + if (auto converted = expression::fromExpressionValue(pair.second)) { + mbglValueMap.emplace(pair.first, std::move(*converted)); + } + } + return {Value{std::move(mbglValueMap)}}; + }, + [](const auto& v) { return expression::fromExpressionValue(v); }); +} + +std::unique_ptr parseExpression(const JSValue& value, + optional& spec, + TestResult& result) { + optional expected = spec ? toExpressionType(*spec) : nullopt; + expression::ParsingContext ctx = expected ? expression::ParsingContext(*expected) : + expression::ParsingContext(); + Convertible convertible(&value); + expression::ParseResult parsed; + if (value.IsObject() && !value.IsArray() && expected){ + Error error; + parsed = convertFunctionToExpression(*expected, convertible, error, false /*convert tokens*/); + if (!parsed) { + // TODO: should the error message be checked for function conversion? + } + } else { + parsed = ctx.parseLayerPropertyExpression(convertible); + if (!parsed) { + for (const auto& parsingError : ctx.getErrors()) { + std::unordered_map errorMap; + errorMap.emplace("key"s, Value{parsingError.key}); + errorMap.emplace("error"s, Value{parsingError.message}); + result.compiled.errors.emplace_back(Value{std::move(errorMap)}); + } + } + } + + result.expression = toValue(value); + result.compiled.success = bool(parsed); + if (parsed) { + result.compiled.isFeatureConstant = expression::isFeatureConstant(**parsed); + result.compiled.isZoomConstant = expression::isZoomConstant(**parsed); + result.compiled.serializedType = style::expression::type::toString((*parsed)->getType()); + result.serialized = (*parsed)->serialize(); + return std::move(*parsed); + } + + return nullptr; +} + +std::unique_ptr parseExpression(const optional& value, + optional& spec, + TestResult& result) { + assert(value); + auto document = toDocument(*value); + assert(!document.HasParseError()); + return parseExpression(document, spec, result); +} diff --git a/expression-test/expression_test_parser.hpp b/expression-test/expression_test_parser.hpp new file mode 100644 index 0000000000..561ccd9647 --- /dev/null +++ b/expression-test/expression_test_parser.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include "filesystem.hpp" + +#include +#include +#include +#include + +#include +#include + +using namespace mbgl; + +struct Input { + Input(optional zoom_, optional heatmapDensity_, Feature feature_) + : zoom(std::move(zoom_)), + heatmapDensity(std::move(heatmapDensity_)), + feature(std::move(feature_)) {} + optional zoom; + optional heatmapDensity; + Feature feature; +}; + +struct Compiled { + bool operator==(const Compiled& other) const { + bool typeEqual = success == other.success && + isFeatureConstant == other.isFeatureConstant && + isZoomConstant == other.isZoomConstant && + serializedType == other.serializedType && + errors == other.errors; + return typeEqual; + } + + bool success = false; + bool isFeatureConstant = false; + bool isZoomConstant = false; + std::string serializedType; + std::vector errors; +}; + +struct TestResult { + Compiled compiled; + optional expression; + optional outputs; + optional serialized; +}; + +struct PropertySpec { + std::string type; + std::string value; + std::size_t length = 0; + bool isDataDriven = false; + optional expression; +}; + +class TestData { +public: + std::vector inputs; + TestResult expected; + TestResult result; + TestResult recompiled; + optional spec; + JSDocument document; +}; + +struct Ignore { + Ignore(std::string id_, std::string reason_) + : id(std::move(id_)), + reason(std::move(reason_)) {} + + std::string id; + std::string reason; +}; + +using Arguments = std::tuple, bool, uint32_t>; +Arguments parseArguments(int argc, char** argv); + +using Ignores = std::vector; +Ignores parseExpressionIgnores(); +optional parseTestData(const filesystem::path&); + +std::string toJSON(const Value& value, unsigned indent = 0, bool singleLine = false); +JSDocument toDocument(const Value&); +Value toValue(const Compiled&); +optional toValue(const style::expression::Value&); + +std::unique_ptr parseExpression(const JSValue&, + optional&, + TestResult&); +std::unique_ptr parseExpression(const optional&, + optional&, + TestResult&); + diff --git a/expression-test/expression_test_runner.cpp b/expression-test/expression_test_runner.cpp new file mode 100644 index 0000000000..695d129049 --- /dev/null +++ b/expression-test/expression_test_runner.cpp @@ -0,0 +1,303 @@ +#include "expression_test_runner.hpp" +#include "expression_test_parser.hpp" +#include "expression_test_logger.hpp" +#include "filesystem.hpp" + +#include + +#include +#include +#include + +#include +#include + +using namespace std::literals; + +namespace { + +// Strip precision for numbers, so that we can compare evaluated results with fixtures. +// Copied from JS expression harness. +Value stripPrecision(const Value& value) { + const double decimalSigFigs = 6; + if (auto num = numericValue(value)) { + if (*num == 0) { + return *num; + } + + const double multiplier = std::pow(10, + std::max(0.0, decimalSigFigs - std::ceil(std::log10(std::fabs(*num))))); + + // We strip precision twice in a row here to avoid cases where + // stripping an already stripped number will modify its value + // due to bad floating point precision luck + // eg `Math.floor(8.16598 * 100000) / 100000` -> 8.16597 + const double firstStrip = std::floor(*num * multiplier) / multiplier; + return std::floor(firstStrip * multiplier) / multiplier; + } + + if (value.is>()) { + std::vector stripped; + const auto& vec = value.get>(); + stripped.reserve(vec.size()); + for (const auto& val : vec) { + stripped.emplace_back(stripPrecision(val)); + } + return stripped; + } else if (value.is>()) { + std::unordered_map stripped; + const auto& map = value.get>(); + for (const auto& pair : map) { + stripped.emplace(pair.first, stripPrecision(pair.second)); + } + return stripped; + } + + return value; +} + +bool deepEqual(const Value& a, const Value& b) { + const auto& anum = numericValue(a); + const auto& bnum = numericValue(b); + if (anum && bnum) { + return stripPrecision(*anum) == stripPrecision(*bnum); + } + + if (a.which() != b.which()) { + return false; + } + + if (a.is>()) { + const auto& avec = a.get>(); + const auto& bvec = b.get>(); + if (avec.size() != bvec.size()) { + return false; + } + for (std::size_t i = 0; i < avec.size(); ++i) { + if (!deepEqual(avec[i], bvec[i])) { + return false; + } + } + return true; + } + + if (a.is>()) { + const auto& amap = a.get>(); + const auto& bmap = b.get>(); + if (amap.size() != bmap.size()) { + return false; + } + for (const auto& pair : amap) { + auto it = bmap.find(pair.first); + if (it == bmap.end()) { + return false; + } + if (!deepEqual(pair.second, it->second)) { + return false; + } + } + return true; + } + + return a == b; +} + +bool deepEqual(const optional& a, const optional& b) { + if ((a && !b) || (!a && b)) { + return false; + } + + if (a && b) { + return deepEqual(*a, *b); + } + + return true; +} + +std::vector tokenize(std::string str) { + std::vector tokens; + std::regex re("\n"); + std::copy(std::regex_token_iterator(str.begin(), str.end(), re, -1), + std::regex_token_iterator(), + std::back_inserter(tokens)); + return tokens; +} + +std::string simpleDiff(const Value& result, const Value& expected) { + std::vector resultTokens {tokenize(toJSON(result, 2, true))}; + std::vector expectedTokens {tokenize(toJSON(expected, 2, true))}; + std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size()); + std::ostringstream diff; + const auto flush = [] (const std::vector& vec, std::size_t pos, std::ostringstream& out, std::string separator) { + for (std::size_t j = pos; j < vec.size(); ++j) { + out << separator << vec[j] << std::endl; + } + }; + + for (std::size_t i = 0; i < maxLength; ++i) { + if (resultTokens.size() <= i) { + flush(expectedTokens, i, diff, "+"s); + break; + } + + if (expectedTokens.size() <= i) { + flush(expectedTokens, i, diff, "-"s); + break; + } + + if (resultTokens[i] != expectedTokens[i]) { + diff << "-"s << expectedTokens[i] << std::endl; + diff << "+"s << resultTokens[i] << std::endl; + } else { + diff << resultTokens[i] << std::endl; + } + } + return diff.str(); +} + +void rewriteRoundtrippedType(const std::string& expected, std::string& actual) { + if (!expected.rfind("array", 0) && !actual.rfind("array", 0)) { + actual = expected; + } +} + +void updateTest(TestData& data) { + assert(data.document["expected"].IsObject()); + auto& expected = data.document["expected"]; + auto compiled = toDocument(toValue(data.result.compiled)); + assert(!compiled.HasParseError()); + expected["compiled"].Swap(compiled); + + if (data.result.outputs) { + auto outputs = toDocument(stripPrecision(data.result.outputs.value_or(Value{}))); + assert(!outputs.HasParseError()); + expected["outputs"].Swap(outputs); + } else { + expected.RemoveMember("outputs"); + } + + if (data.result.serialized) { + auto serialized = toDocument(*data.result.serialized); + assert(!serialized.HasParseError()); + expected["serialized"].Swap(serialized); + } else { + expected.RemoveMember("serialized"); + } +} + +void writeTestData(const JSDocument& document, const std::string& rootPath, const std::string& id) { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + writer.SetFormatOptions(rapidjson::kFormatSingleLineArray); + writer.SetIndent(' ', 2); + document.Accept(writer); + buffer.Put('\n'); + filesystem::path path = filesystem::path(rootPath) / id / "test.json"s; + try { + util::write_file(path.string(), {buffer.GetString(), buffer.GetSize()}); + } catch (std::exception&) { + printf(ANSI_COLOR_RED "* ERROR can't update '%s' test" ANSI_COLOR_RESET "\n", id.c_str()); + } +} + +} // namespace + +TestRunOutput runExpressionTest(TestData& data, const std::string& rootPath, const std::string& id) { + TestRunOutput output(id); + const auto evaluateExpression = [&data](std::unique_ptr& expression, + TestResult& result) { + assert(expression); + std::vector outputs; + if (!data.inputs.empty()) { + for (const auto& input : data.inputs) { + auto evaluationResult = expression->evaluate(input.zoom, input.feature, input.heatmapDensity); + if (!evaluationResult) { + std::unordered_map error{{"error", Value{evaluationResult.error().message}}}; + outputs.emplace_back(Value{std::move(error)}); + } else { + auto value = toValue(*evaluationResult); + assert(value); + outputs.emplace_back(Value{*value}); + } + } + } + result.outputs = {Value{std::move(outputs)}}; + }; + + // Parse expression + auto parsedExpression = parseExpression(data.document["expression"], data.spec, data.result); + output.expression = toJSON(data.result.expression.value_or(Value{}), 2, true); + + // Evaluate expression + if (parsedExpression) { + evaluateExpression(parsedExpression, data.result); + output.serialized = toJSON(data.result.serialized.value_or(Value{}), 2, true); + + // round trip + auto recompiledExpression = parseExpression(data.result.serialized, data.spec, data.recompiled); + if (recompiledExpression) { + evaluateExpression(recompiledExpression, data.recompiled); + rewriteRoundtrippedType(data.expected.compiled.serializedType, + data.recompiled.compiled.serializedType); + } + } + + if (getenv("UPDATE")) { + output.passed = true; + updateTest(data); + writeTestData(data.document, rootPath, id); + return output; + } + + bool compileOk = data.result.compiled == data.expected.compiled; + bool evalOk = compileOk && deepEqual(data.result.outputs, data.expected.outputs); + + bool recompileOk = true; + bool roundTripOk = true; + bool serializationOk = true; + + if (data.expected.compiled.success) { + serializationOk = compileOk && deepEqual(data.result.serialized, data.expected.serialized); + recompileOk = compileOk && data.recompiled.compiled == data.expected.compiled; + roundTripOk = recompileOk && deepEqual(data.recompiled.outputs, data.expected.outputs); + } + + output.passed = compileOk && evalOk && recompileOk && roundTripOk && serializationOk; + + if (!compileOk) { + auto resultValue = toValue(data.result.compiled); + auto expectedValue = toValue(data.expected.compiled); + output.text += "Compiled expression difference:\n"s + + simpleDiff(resultValue, expectedValue) + + "\n"s; + } + + if (compileOk && !serializationOk) { + auto diff = simpleDiff(data.expected.serialized.value_or(Value{}), + data.result.serialized.value_or(Value{})); + output.text += "Serialized expression difference:\n"s + diff + "\n"s; + } + + if (compileOk && !recompileOk) { + auto recompiledValue = toValue(data.recompiled.compiled); + auto expectedValue = toValue(data.expected.compiled); + output.text += "Recompiled expression difference:\n"s + + simpleDiff(recompiledValue, expectedValue) + + "\n"s; + } + + if (compileOk && !evalOk) { + auto diff = simpleDiff(data.expected.outputs.value_or(Value{}), + data.result.outputs.value_or(Value{})); + output.text += "Expression outputs difference:\n"s + diff + "\n"s; + } + + if (recompileOk && !roundTripOk) { + auto diff = simpleDiff(data.expected.outputs.value_or(Value{}), + data.recompiled.outputs.value_or(Value{})); + output.text += "Roundtripped through serialize expression outputs difference:\n"s + + diff + "\n"s; + } + + return output; +} diff --git a/expression-test/expression_test_runner.hpp b/expression-test/expression_test_runner.hpp new file mode 100644 index 0000000000..596d5c11b6 --- /dev/null +++ b/expression-test/expression_test_runner.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +class TestData; + +struct TestRunOutput { + TestRunOutput(std::string id_) : id(std::move(id_)) {} + std::string id; + bool passed = false; + std::string text; + std::string expression; + std::string serialized; +}; + +class TestStats { +public: + std::size_t testCount() const { + return passed.size() + failed.size() + errored.size() + + ignorePassed.size() + ignoreFailed.size(); + } + + std::vector passed; + std::vector failed; + std::vector errored; + std::vector ignorePassed; + std::vector ignoreFailed; + std::vector ids; +}; + +TestRunOutput runExpressionTest(TestData&, const std::string& rootPath, const std::string& id); diff --git a/expression-test/main.cpp b/expression-test/main.cpp new file mode 100644 index 0000000000..db468dffe0 --- /dev/null +++ b/expression-test/main.cpp @@ -0,0 +1,75 @@ +#include "expression_test_parser.hpp" +#include "expression_test_runner.hpp" +#include "expression_test_logger.hpp" + +#include + +int main(int argc, char** argv) { + // Parse args + std::vector testPaths; + mbgl::filesystem::path rootPath; + bool shuffle; + uint32_t seed; + std::tie(rootPath, testPaths, shuffle, seed) = parseArguments(argc, argv); + + // Parse ignores + const auto ignores = parseExpressionIgnores(); + + if (shuffle) { + printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed); + std::seed_seq sequence { seed }; + std::mt19937 shuffler(sequence); + std::shuffle(testPaths.begin(), testPaths.end(), shuffler); + } + + // Run tests, collect stats and output of each run. + TestStats stats; + for (const auto& path : testPaths) { + const auto& expectation = path.parent_path().string(); + std::string id = expectation.substr(rootPath.string().length() + 1, expectation.length() - rootPath.string().length()); + stats.ids.emplace_back(id); + + bool shouldIgnore = false; + std::string ignoreReason; + const std::string ignoreName = "expression-tests/" + id; + const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName] (const auto& ignore) { return ignore.id == ignoreName; }); + if (it != ignores.end()) { + shouldIgnore = true; + ignoreReason = (*it).reason; + } + + optional testRun; + if (auto testData = parseTestData(path)) { + testRun = runExpressionTest(*testData, rootPath.string(), id); + } + + if (!testRun) { + stats.errored.emplace_back(id); + printf(ANSI_COLOR_RED "* ERROR can't parse '%s' test" ANSI_COLOR_RESET "\n", id.c_str()); + continue; + } + + if (shouldIgnore) { + if (testRun->passed) { + stats.ignorePassed.emplace_back(std::move(*testRun)); + printf(ANSI_COLOR_YELLOW "* PASSED ignored test %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str()); + } else { + stats.ignoreFailed.emplace_back(std::move(*testRun)); + printf(ANSI_COLOR_LIGHT_GRAY "* FAILED ignored test %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str()); + } + } else { + if (testRun->passed) { + stats.passed.emplace_back(std::move(*testRun)); + printf(ANSI_COLOR_GREEN "* PASSED %s" ANSI_COLOR_RESET "\n", id.c_str()); + } else { + printf(ANSI_COLOR_RED "* FAILED %s\n%s" ANSI_COLOR_RESET, id.c_str(), testRun->text.c_str()); + stats.failed.emplace_back(std::move(*testRun)); + } + } + } + + printStats(stats); + writeHTMLResults(stats, rootPath.string(), shuffle, seed); + + return stats.errored.size() + stats.failed.size() == 0 ? 0 : 1; +} -- cgit v1.2.1