diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-07-16 11:45:35 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-07-17 08:59:23 +0000 |
commit | 552906b0f222c5d5dd11b9fd73829d510980461a (patch) | |
tree | 3a11e6ed0538a81dd83b20cf3a4783e297f26d91 /chromium/third_party/snappy | |
parent | 1b05827804eaf047779b597718c03e7d38344261 (diff) | |
download | qtwebengine-chromium-552906b0f222c5d5dd11b9fd73829d510980461a.tar.gz |
BASELINE: Update Chromium to 83.0.4103.122
Change-Id: Ie3a82f5bb0076eec2a7c6a6162326b4301ee291e
Reviewed-by: Michael BrĂ¼ning <michael.bruning@qt.io>
Diffstat (limited to 'chromium/third_party/snappy')
-rw-r--r-- | chromium/third_party/snappy/src/.appveyor.yml | 17 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/.travis.yml | 35 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/CMakeLists.txt | 164 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/cmake/SnappyConfig.cmake | 1 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/cmake/SnappyConfig.cmake.in | 33 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy-stubs-internal.cc | 2 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy-stubs-internal.h | 6 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy-stubs-public.h.in | 2 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy-test.cc | 31 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy-test.h | 20 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy.cc | 13 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy.h | 22 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy_compress_fuzzer.cc | 59 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy_uncompress_fuzzer.cc | 57 | ||||
-rw-r--r-- | chromium/third_party/snappy/src/snappy_unittest.cc | 257 |
15 files changed, 522 insertions, 197 deletions
diff --git a/chromium/third_party/snappy/src/.appveyor.yml b/chromium/third_party/snappy/src/.appveyor.yml index c9ffc8f4121..441a2b66054 100644 --- a/chromium/third_party/snappy/src/.appveyor.yml +++ b/chromium/third_party/snappy/src/.appveyor.yml @@ -8,9 +8,9 @@ environment: matrix: # AppVeyor currently has no custom job name feature. # http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs - - JOB: Visual Studio 2017 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - CMAKE_GENERATOR: Visual Studio 15 2017 + - JOB: Visual Studio 2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + CMAKE_GENERATOR: Visual Studio 16 2019 platform: - x86 @@ -22,14 +22,15 @@ configuration: build_script: - git submodule update --init --recursive - - mkdir out - - cd out - - if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64 + - mkdir build + - cd build + - if "%platform%"=="x86" (set CMAKE_GENERATOR_PLATFORM="Win32") + else (set CMAKE_GENERATOR_PLATFORM="%platform%") - cmake --version - - cmake .. -G "%CMAKE_GENERATOR%" + - cmake .. -G "%CMAKE_GENERATOR%" -A "%CMAKE_GENERATOR_PLATFORM%" -DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%" -DSNAPPY_REQUIRE_AVX2=ON - cmake --build . --config %CONFIGURATION% - cd .. test_script: - - out\%CONFIGURATION%\snappy_unittest + - build\%CONFIGURATION%\snappy_unittest diff --git a/chromium/third_party/snappy/src/.travis.yml b/chromium/third_party/snappy/src/.travis.yml index 717bac876a6..896147cf4a9 100644 --- a/chromium/third_party/snappy/src/.travis.yml +++ b/chromium/third_party/snappy/src/.travis.yml @@ -3,8 +3,8 @@ # This file can be validated on: http://lint.travis-ci.org/ language: cpp -dist: xenial -osx_image: xcode10.2 +dist: bionic +osx_image: xcode10.3 compiler: - gcc @@ -31,10 +31,11 @@ matrix: addons: apt: sources: - - llvm-toolchain-xenial-8 - - ubuntu-toolchain-r-test + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + - sourceline: 'ppa:ubuntu-toolchain-r/test' packages: - - clang-8 + - clang-9 - cmake - gcc-9 - g++-9 @@ -43,16 +44,22 @@ addons: packages: - cmake - gcc@9 - - llvm@8 + - llvm@9 - ninja update: true install: # The following Homebrew packages aren't linked by default, and need to be # prepended to the path explicitly. -- if [ "$TRAVIS_OS_NAME" == "osx" ]; then +- if [ "$TRAVIS_OS_NAME" = "osx" ]; then export PATH="$(brew --prefix llvm)/bin:$PATH"; fi +# Fuzzing is only supported on Clang. Perform fuzzing on Debug builds. +- if [ "$CXX" = "clang++" ] && [ "$BUILD_TYPE" = "Debug" ]; then + export FUZZING=1; + else + export FUZZING=0; + fi # /usr/bin/gcc points to an older compiler on both Linux and macOS. - if [ "$CXX" = "g++" ]; then export CXX="g++-9" CC="gcc-9"; fi # /usr/bin/clang points to an older compiler on both Linux and macOS. @@ -60,8 +67,8 @@ install: # Homebrew's llvm package doesn't ship a versioned clang++ binary, so the values # below don't work on macOS. Fortunately, the path change above makes the # default values (clang and clang++) resolve to the correct compiler on macOS. -- if [ "$TRAVIS_OS_NAME" == "linux" ]; then - if [ "$CXX" = "clang++" ]; then export CXX="clang++-8" CC="clang-8"; fi; +- if [ "$TRAVIS_OS_NAME" = "linux" ]; then + if [ "$CXX" = "clang++" ]; then export CXX="clang++-9" CC="clang-9"; fi; fi - echo ${CC} - echo ${CXX} @@ -71,9 +78,17 @@ install: before_script: - mkdir -p build && cd build - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=$BUILD_TYPE - -DSNAPPY_REQUIRE_${CPU_LEVEL}=ON + -DSNAPPY_REQUIRE_${CPU_LEVEL}=ON -DSNAPPY_FUZZING_BUILD=${FUZZING} + -DCMAKE_INSTALL_PREFIX=$HOME/.local - cmake --build . - cd .. script: - build/snappy_unittest +- if [ -f build/snappy_compress_fuzzer ]; then + build/snappy_compress_fuzzer -runs=1000 -close_fd_mask=3; + fi +- if [ -f build/snappy_uncompress_fuzzer ]; then + build/snappy_uncompress_fuzzer -runs=1000 -close_fd_mask=3; + fi +- cd build && cmake --build . --target install diff --git a/chromium/third_party/snappy/src/CMakeLists.txt b/chromium/third_party/snappy/src/CMakeLists.txt index 6d3a157ddc9..32c1d076c21 100644 --- a/chromium/third_party/snappy/src/CMakeLists.txt +++ b/chromium/third_party/snappy/src/CMakeLists.txt @@ -1,10 +1,41 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + cmake_minimum_required(VERSION 3.1) project(Snappy VERSION 1.1.7 LANGUAGES C CXX) -# This project requires C++11. -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) +# C++ standard can be overridden when this is used as a sub-project. +if(NOT CMAKE_CXX_STANDARD) + # This project requires C++11. + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) +endif(NOT CMAKE_CXX_STANDARD) # BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make # it prominent in the GUI. @@ -12,10 +43,14 @@ option(BUILD_SHARED_LIBS "Build shared libraries(DLLs)." OFF) option(SNAPPY_BUILD_TESTS "Build Snappy's own tests." ON) +option(SNAPPY_FUZZING_BUILD "Build Snappy for fuzzing." OFF) + option(SNAPPY_REQUIRE_AVX "Target processors with AVX support." OFF) option(SNAPPY_REQUIRE_AVX2 "Target processors with AVX2 support." OFF) +option(SNAPPY_INSTALL "Install Snappy's header and library" ON) + include(TestBigEndian) test_big_endian(SNAPPY_IS_BIG_ENDIAN) @@ -101,7 +136,7 @@ if(GFLAGS_FOUND) endif(GFLAGS_FOUND) configure_file( - "${PROJECT_SOURCE_DIR}/cmake/config.h.in" + "cmake/config.h.in" "${PROJECT_BINARY_DIR}/config.h" ) @@ -113,19 +148,33 @@ if(NOT HAVE_SYS_UIO_H_01) set(HAVE_SYS_UIO_H_01 0) endif(NOT HAVE_SYS_UIO_H_01) +if (SNAPPY_FUZZING_BUILD) + if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + message(WARNING "Fuzzing builds are only supported with Clang") + endif (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + + if(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + endif(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=address") + + if(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=fuzzer-no-link") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=fuzzer-no-link") + endif(NOT CMAKE_CXX_FLAGS MATCHES "-fsanitize=fuzzer-no-link") +endif (SNAPPY_FUZZING_BUILD) + configure_file( - "${PROJECT_SOURCE_DIR}/snappy-stubs-public.h.in" + "snappy-stubs-public.h.in" "${PROJECT_BINARY_DIR}/snappy-stubs-public.h") add_library(snappy "") target_sources(snappy PRIVATE - "${PROJECT_SOURCE_DIR}/snappy-internal.h" - "${PROJECT_SOURCE_DIR}/snappy-stubs-internal.h" - "${PROJECT_SOURCE_DIR}/snappy-c.cc" - "${PROJECT_SOURCE_DIR}/snappy-sinksource.cc" - "${PROJECT_SOURCE_DIR}/snappy-stubs-internal.cc" - "${PROJECT_SOURCE_DIR}/snappy.cc" + "snappy-internal.h" + "snappy-stubs-internal.h" + "snappy-c.cc" + "snappy-sinksource.cc" + "snappy-stubs-internal.cc" + "snappy.cc" "${PROJECT_BINARY_DIR}/config.h" # Only CMake 3.3+ supports PUBLIC sources in targets exported by "install". @@ -159,8 +208,8 @@ if(SNAPPY_BUILD_TESTS) add_executable(snappy_unittest "") target_sources(snappy_unittest PRIVATE - "${PROJECT_SOURCE_DIR}/snappy_unittest.cc" - "${PROJECT_SOURCE_DIR}/snappy-test.cc" + "snappy_unittest.cc" + "snappy-test.cc" ) target_compile_definitions(snappy_unittest PRIVATE -DHAVE_CONFIG_H) target_link_libraries(snappy_unittest snappy ${GFLAGS_LIBRARIES}) @@ -185,35 +234,64 @@ if(SNAPPY_BUILD_TESTS) COMMAND "${PROJECT_BINARY_DIR}/snappy_unittest") endif(SNAPPY_BUILD_TESTS) +if(SNAPPY_FUZZING_BUILD) + add_executable(snappy_compress_fuzzer "") + target_sources(snappy_compress_fuzzer + PRIVATE "snappy_compress_fuzzer.cc" + ) + target_link_libraries(snappy_compress_fuzzer snappy) + set_target_properties(snappy_compress_fuzzer + PROPERTIES LINK_FLAGS "-fsanitize=fuzzer" + ) + + add_executable(snappy_uncompress_fuzzer "") + target_sources(snappy_uncompress_fuzzer + PRIVATE "snappy_uncompress_fuzzer.cc" + ) + target_link_libraries(snappy_uncompress_fuzzer snappy) + set_target_properties(snappy_uncompress_fuzzer + PROPERTIES LINK_FLAGS "-fsanitize=fuzzer" + ) +endif(SNAPPY_FUZZING_BUILD) + +# Must be included before CMAKE_INSTALL_INCLUDEDIR is used. include(GNUInstallDirs) -install(TARGETS snappy - EXPORT SnappyTargets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -) -install( - FILES - "${PROJECT_SOURCE_DIR}/snappy-c.h" - "${PROJECT_SOURCE_DIR}/snappy-sinksource.h" - "${PROJECT_SOURCE_DIR}/snappy.h" - "${PROJECT_BINARY_DIR}/snappy-stubs-public.h" - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) -include(CMakePackageConfigHelpers) -write_basic_package_version_file( - "${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake" +if(SNAPPY_INSTALL) + install(TARGETS snappy + EXPORT SnappyTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + install( + FILES + "snappy-c.h" + "snappy-sinksource.h" + "snappy.h" + "${PROJECT_BINARY_DIR}/snappy-stubs-public.h" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + ) + + include(CMakePackageConfigHelpers) + configure_package_config_file( + "cmake/${PROJECT_NAME}Config.cmake.in" + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) + write_basic_package_version_file( + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake" COMPATIBILITY SameMajorVersion -) -install( - EXPORT SnappyTargets - NAMESPACE Snappy:: - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy" -) -install( - FILES - "${PROJECT_SOURCE_DIR}/cmake/SnappyConfig.cmake" - "${PROJECT_BINARY_DIR}/SnappyConfigVersion.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/Snappy" -) + ) + install( + EXPORT SnappyTargets + NAMESPACE Snappy:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) + install( + FILES + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake" + "${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" + ) +endif(SNAPPY_INSTALL) diff --git a/chromium/third_party/snappy/src/cmake/SnappyConfig.cmake b/chromium/third_party/snappy/src/cmake/SnappyConfig.cmake deleted file mode 100644 index dc308ae265f..00000000000 --- a/chromium/third_party/snappy/src/cmake/SnappyConfig.cmake +++ /dev/null @@ -1 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake")
\ No newline at end of file diff --git a/chromium/third_party/snappy/src/cmake/SnappyConfig.cmake.in b/chromium/third_party/snappy/src/cmake/SnappyConfig.cmake.in new file mode 100644 index 00000000000..9e7d134628a --- /dev/null +++ b/chromium/third_party/snappy/src/cmake/SnappyConfig.cmake.in @@ -0,0 +1,33 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/SnappyTargets.cmake") + +check_required_components(Snappy)
\ No newline at end of file diff --git a/chromium/third_party/snappy/src/snappy-stubs-internal.cc b/chromium/third_party/snappy/src/snappy-stubs-internal.cc index 6ed334371f1..66ed2e90393 100644 --- a/chromium/third_party/snappy/src/snappy-stubs-internal.cc +++ b/chromium/third_party/snappy/src/snappy-stubs-internal.cc @@ -33,7 +33,7 @@ namespace snappy { -void Varint::Append32(string* s, uint32 value) { +void Varint::Append32(std::string* s, uint32 value) { char buf[Varint::kMax32]; const char* p = Varint::Encode32(buf, value); s->append(buf, p - buf); diff --git a/chromium/third_party/snappy/src/snappy-stubs-internal.h b/chromium/third_party/snappy/src/snappy-stubs-internal.h index 716a949c711..4854689d177 100644 --- a/chromium/third_party/snappy/src/snappy-stubs-internal.h +++ b/chromium/third_party/snappy/src/snappy-stubs-internal.h @@ -524,7 +524,7 @@ class Varint { static char* Encode32(char* ptr, uint32 v); // EFFECTS Appends the varint representation of "value" to "*s". - static void Append32(string* s, uint32 value); + static void Append32(std::string* s, uint32 value); }; inline const char* Varint::Parse32WithLimit(const char* p, @@ -581,7 +581,7 @@ inline char* Varint::Encode32(char* sptr, uint32 v) { // replace this function with one that resizes the string without // filling the new space with zeros (if applicable) -- // it will be non-portable but faster. -inline void STLStringResizeUninitialized(string* s, size_t new_size) { +inline void STLStringResizeUninitialized(std::string* s, size_t new_size) { s->resize(new_size); } @@ -597,7 +597,7 @@ inline void STLStringResizeUninitialized(string* s, size_t new_size) { // (http://www.open-std.org/JTC1/SC22/WG21/docs/lwg-defects.html#530) // proposes this as the method. It will officially be part of the standard // for C++0x. This should already work on all current implementations. -inline char* string_as_array(string* str) { +inline char* string_as_array(std::string* str) { return str->empty() ? NULL : &*str->begin(); } diff --git a/chromium/third_party/snappy/src/snappy-stubs-public.h.in b/chromium/third_party/snappy/src/snappy-stubs-public.h.in index 2fd82ad676b..416ab9978c2 100644 --- a/chromium/third_party/snappy/src/snappy-stubs-public.h.in +++ b/chromium/third_party/snappy/src/snappy-stubs-public.h.in @@ -60,8 +60,6 @@ using uint32 = std::uint32_t; using int64 = std::int64_t; using uint64 = std::uint64_t; -using string = std::string; - #if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H // Windows does not have an iovec type, yet the concept is universally useful. // It is simple to define it ourselves, so we put it inside our own namespace. diff --git a/chromium/third_party/snappy/src/snappy-test.cc b/chromium/third_party/snappy/src/snappy-test.cc index 01d5541fd2c..83be2d36311 100644 --- a/chromium/third_party/snappy/src/snappy-test.cc +++ b/chromium/third_party/snappy/src/snappy-test.cc @@ -48,12 +48,12 @@ DEFINE_bool(run_microbenchmarks, true, namespace snappy { -string ReadTestDataFile(const string& base, size_t size_limit) { - string contents; +std::string ReadTestDataFile(const std::string& base, size_t size_limit) { + std::string contents; const char* srcdir = getenv("srcdir"); // This is set by Automake. - string prefix; + std::string prefix; if (srcdir) { - prefix = string(srcdir) + "/"; + prefix = std::string(srcdir) + "/"; } file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults() ).CheckSuccess(); @@ -63,11 +63,11 @@ string ReadTestDataFile(const string& base, size_t size_limit) { return contents; } -string ReadTestDataFile(const string& base) { +std::string ReadTestDataFile(const std::string& base) { return ReadTestDataFile(base, 0); } -string StringPrintf(const char* format, ...) { +std::string StrFormat(const char* format, ...) { char buf[4096]; va_list ap; va_start(ap, format); @@ -79,7 +79,7 @@ string StringPrintf(const char* format, ...) { bool benchmark_running = false; int64 benchmark_real_time_us = 0; int64 benchmark_cpu_time_us = 0; -string *benchmark_label = NULL; +std::string* benchmark_label = nullptr; int64 benchmark_bytes_processed = 0; void ResetBenchmarkTiming() { @@ -163,11 +163,11 @@ void StopBenchmarkTiming() { benchmark_running = false; } -void SetBenchmarkLabel(const string& str) { +void SetBenchmarkLabel(const std::string& str) { if (benchmark_label) { delete benchmark_label; } - benchmark_label = new string(str); + benchmark_label = new std::string(str); } void SetBenchmarkBytesProcessed(int64 bytes) { @@ -217,8 +217,8 @@ void Benchmark::Run() { benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us; } - string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num); - string human_readable_speed; + std::string heading = StrFormat("%s/%d", name_.c_str(), test_case_num); + std::string human_readable_speed; std::nth_element(benchmark_runs, benchmark_runs + kMedianPos, @@ -232,15 +232,16 @@ void Benchmark::Run() { int64 bytes_per_second = benchmark_bytes_processed * 1000000 / cpu_time_us; if (bytes_per_second < 1024) { - human_readable_speed = StringPrintf("%dB/s", bytes_per_second); + human_readable_speed = + StrFormat("%dB/s", static_cast<int>(bytes_per_second)); } else if (bytes_per_second < 1024 * 1024) { - human_readable_speed = StringPrintf( + human_readable_speed = StrFormat( "%.1fkB/s", bytes_per_second / 1024.0f); } else if (bytes_per_second < 1024 * 1024 * 1024) { - human_readable_speed = StringPrintf( + human_readable_speed = StrFormat( "%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f)); } else { - human_readable_speed = StringPrintf( + human_readable_speed = StrFormat( "%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f)); } } diff --git a/chromium/third_party/snappy/src/snappy-test.h b/chromium/third_party/snappy/src/snappy-test.h index e31a78d6a1b..c8b7d388607 100644 --- a/chromium/third_party/snappy/src/snappy-test.h +++ b/chromium/third_party/snappy/src/snappy-test.h @@ -167,7 +167,7 @@ namespace file { namespace snappy { #define FLAGS_test_random_seed 301 -typedef string TypeParam; +using TypeParam = std::string; void Test_CorruptedTest_VerifyCorrupted(); void Test_Snappy_SimpleTests(); @@ -181,13 +181,13 @@ void Test_Snappy_ReadPastEndOfBuffer(); void Test_Snappy_FindMatchLength(); void Test_Snappy_FindMatchLengthRandom(); -string ReadTestDataFile(const string& base, size_t size_limit); +std::string ReadTestDataFile(const std::string& base, size_t size_limit); -string ReadTestDataFile(const string& base); +std::string ReadTestDataFile(const std::string& base); // A sprintf() variant that returns a std::string. // Not safe for general use due to truncation issues. -string StringPrintf(const char* format, ...); +std::string StrFormat(const char* format, ...); // A wall-time clock. This stub is not super-accurate, nor resistant to the // system time changing. @@ -241,8 +241,8 @@ typedef void (*BenchmarkFunction)(int, int); class Benchmark { public: - Benchmark(const string& name, BenchmarkFunction function) : - name_(name), function_(function) {} + Benchmark(const std::string& name, BenchmarkFunction function) + : name_(name), function_(function) {} Benchmark* DenseRange(int start, int stop) { start_ = start; @@ -253,7 +253,7 @@ class Benchmark { void Run(); private: - const string name_; + const std::string name_; const BenchmarkFunction function_; int start_, stop_; }; @@ -265,11 +265,13 @@ extern Benchmark* Benchmark_BM_UFlat; extern Benchmark* Benchmark_BM_UIOVec; extern Benchmark* Benchmark_BM_UValidate; extern Benchmark* Benchmark_BM_ZFlat; +extern Benchmark* Benchmark_BM_ZFlatAll; +extern Benchmark* Benchmark_BM_ZFlatIncreasingTableSize; void ResetBenchmarkTiming(); void StartBenchmarkTiming(); void StopBenchmarkTiming(); -void SetBenchmarkLabel(const string& str); +void SetBenchmarkLabel(const std::string& str); void SetBenchmarkBytesProcessed(int64 bytes); #ifdef HAVE_LIBZ @@ -416,6 +418,8 @@ static inline void RunSpecifiedBenchmarks() { snappy::Benchmark_BM_UIOVec->Run(); snappy::Benchmark_BM_UValidate->Run(); snappy::Benchmark_BM_ZFlat->Run(); + snappy::Benchmark_BM_ZFlatAll->Run(); + snappy::Benchmark_BM_ZFlatIncreasingTableSize->Run(); fprintf(stderr, "\n"); } diff --git a/chromium/third_party/snappy/src/snappy.cc b/chromium/third_party/snappy/src/snappy.cc index b36f69d6b1f..3ef0708bfaa 100644 --- a/chromium/third_party/snappy/src/snappy.cc +++ b/chromium/third_party/snappy/src/snappy.cc @@ -444,12 +444,14 @@ bool GetUncompressedLength(const char* start, size_t n, size_t* result) { namespace { uint32 CalculateTableSize(uint32 input_size) { - assert(kMaxHashTableSize >= 256); + static_assert( + kMaxHashTableSize >= kMinHashTableSize, + "kMaxHashTableSize should be greater or equal to kMinHashTableSize."); if (input_size > kMaxHashTableSize) { return kMaxHashTableSize; } - if (input_size < 256) { - return 256; + if (input_size < kMinHashTableSize) { + return kMinHashTableSize; } // This is equivalent to Log2Ceiling(input_size), assuming input_size > 1. // 2 << Log2Floor(x - 1) is equivalent to 1 << (1 + Log2Floor(x - 1)). @@ -1344,7 +1346,7 @@ bool RawUncompress(Source* compressed, char* uncompressed) { return InternalUncompress(compressed, &output); } -bool Uncompress(const char* compressed, size_t n, string* uncompressed) { +bool Uncompress(const char* compressed, size_t n, std::string* uncompressed) { size_t ulength; if (!GetUncompressedLength(compressed, n, &ulength)) { return false; @@ -1412,7 +1414,8 @@ void RawCompress(const char* input, *compressed_length = (writer.CurrentDestination() - compressed); } -size_t Compress(const char* input, size_t input_length, string* compressed) { +size_t Compress(const char* input, size_t input_length, + std::string* compressed) { // Pre-grow the buffer to the max length of the compressed output STLStringResizeUninitialized(compressed, MaxCompressedLength(input_length)); diff --git a/chromium/third_party/snappy/src/snappy.h b/chromium/third_party/snappy/src/snappy.h index f4d36ef3444..e9805bfb7de 100644 --- a/chromium/third_party/snappy/src/snappy.h +++ b/chromium/third_party/snappy/src/snappy.h @@ -69,11 +69,12 @@ namespace snappy { // Higher-level string based routines (should be sufficient for most users) // ------------------------------------------------------------------------ - // Sets "*output" to the compressed version of "input[0,input_length-1]". - // Original contents of *output are lost. + // Sets "*compressed" to the compressed version of "input[0,input_length-1]". + // Original contents of *compressed are lost. // - // REQUIRES: "input[]" is not an alias of "*output". - size_t Compress(const char* input, size_t input_length, string* output); + // REQUIRES: "input[]" is not an alias of "*compressed". + size_t Compress(const char* input, size_t input_length, + std::string* compressed); // Decompresses "compressed[0,compressed_length-1]" to "*uncompressed". // Original contents of "*uncompressed" are lost. @@ -82,7 +83,7 @@ namespace snappy { // // returns false if the message is corrupted and could not be decompressed bool Uncompress(const char* compressed, size_t compressed_length, - string* uncompressed); + std::string* uncompressed); // Decompresses "compressed" to "*uncompressed". // @@ -193,11 +194,14 @@ namespace snappy { // Note that there might be older data around that is compressed with larger // block sizes, so the decompression code should not rely on the // non-existence of long backreferences. - static const int kBlockLog = 16; - static const size_t kBlockSize = 1 << kBlockLog; + static constexpr int kBlockLog = 16; + static constexpr size_t kBlockSize = 1 << kBlockLog; - static const int kMaxHashTableBits = 14; - static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits; + static constexpr int kMinHashTableBits = 8; + static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits; + + static constexpr int kMaxHashTableBits = 14; + static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits; } // end namespace snappy #endif // THIRD_PARTY_SNAPPY_SNAPPY_H__ diff --git a/chromium/third_party/snappy/src/snappy_compress_fuzzer.cc b/chromium/third_party/snappy/src/snappy_compress_fuzzer.cc new file mode 100644 index 00000000000..1d0119e184d --- /dev/null +++ b/chromium/third_party/snappy/src/snappy_compress_fuzzer.cc @@ -0,0 +1,59 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// libFuzzer harness for fuzzing snappy compression code. + +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <string> + +#include "snappy.h" + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + std::string input(reinterpret_cast<const char*>(data), size); + + std::string compressed; + size_t compressed_size = + snappy::Compress(input.data(), input.size(), &compressed); + + (void)compressed_size; // Variable only used in debug builds. + assert(compressed_size == compressed.size()); + assert(compressed.size() <= snappy::MaxCompressedLength(input.size())); + assert(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); + + std::string uncompressed_after_compress; + bool uncompress_succeeded = snappy::Uncompress( + compressed.data(), compressed.size(), &uncompressed_after_compress); + + (void)uncompress_succeeded; // Variable only used in debug builds. + assert(uncompress_succeeded); + assert(input == uncompressed_after_compress); + return 0; +} diff --git a/chromium/third_party/snappy/src/snappy_uncompress_fuzzer.cc b/chromium/third_party/snappy/src/snappy_uncompress_fuzzer.cc new file mode 100644 index 00000000000..8071c00eea7 --- /dev/null +++ b/chromium/third_party/snappy/src/snappy_uncompress_fuzzer.cc @@ -0,0 +1,57 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// libFuzzer harness for fuzzing snappy's decompression code. + +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <string> + +#include "snappy.h" + +// Entry point for LibFuzzer. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + std::string input(reinterpret_cast<const char*>(data), size); + + // Avoid self-crafted decompression bombs. + size_t uncompressed_size; + constexpr size_t kMaxUncompressedSize = 1 << 20; + bool get_uncompressed_length_succeeded = snappy::GetUncompressedLength( + input.data(), input.size(), &uncompressed_size); + if (!get_uncompressed_length_succeeded || + (uncompressed_size > kMaxUncompressedSize)) { + return 0; + } + + std::string uncompressed; + // The return value of snappy::Uncompress() is ignored because decompression + // will fail on invalid inputs. + snappy::Uncompress(input.data(), input.size(), &uncompressed); + return 0; +} diff --git a/chromium/third_party/snappy/src/snappy_unittest.cc b/chromium/third_party/snappy/src/snappy_unittest.cc index 0221644ead4..37159c32d40 100644 --- a/chromium/third_party/snappy/src/snappy_unittest.cc +++ b/chromium/third_party/snappy/src/snappy_unittest.cc @@ -73,7 +73,7 @@ namespace snappy { // be able to read previously allocated memory while doing heap allocations. class DataEndingAtUnreadablePage { public: - explicit DataEndingAtUnreadablePage(const string& s) { + explicit DataEndingAtUnreadablePage(const std::string& s) { const size_t page_size = sysconf(_SC_PAGESIZE); const size_t size = s.size(); // Round up space for string to a multiple of page_size. @@ -112,7 +112,7 @@ class DataEndingAtUnreadablePage { #else // defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF) // Fallback for systems without mmap. -typedef string DataEndingAtUnreadablePage; +using DataEndingAtUnreadablePage = std::string; #endif @@ -154,7 +154,7 @@ static size_t MinimumRequiredOutputSpace(size_t input_size, // "compressed" must be preinitialized to at least MinCompressbufSize(comp) // number of bytes, and may contain junk bytes at the end after return. static bool Compress(const char* input, size_t input_size, CompressorType comp, - string* compressed, bool compressed_is_preallocated) { + std::string* compressed, bool compressed_is_preallocated) { if (!compressed_is_preallocated) { compressed->resize(MinimumRequiredOutputSpace(input_size, comp)); } @@ -215,8 +215,8 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp, return true; } -static bool Uncompress(const string& compressed, CompressorType comp, - int size, string* output) { +static bool Uncompress(const std::string& compressed, CompressorType comp, + int size, std::string* output) { switch (comp) { #ifdef ZLIB_VERSION case ZLIB: { @@ -279,8 +279,8 @@ static void Measure(const char* data, int num_blocks = (length + block_size - 1) / block_size; std::vector<const char*> input(num_blocks); std::vector<size_t> input_length(num_blocks); - std::vector<string> compressed(num_blocks); - std::vector<string> output(num_blocks); + std::vector<std::string> compressed(num_blocks); + std::vector<std::string> output(num_blocks); for (int b = 0; b < num_blocks; b++) { int input_start = b * block_size; int input_limit = std::min<int>((b+1)*block_size, length); @@ -344,11 +344,10 @@ static void Measure(const char* data, float comp_rate = (length / ctime[med]) * repeats / 1048576.0; float uncomp_rate = (length / utime[med]) * repeats / 1048576.0; - string x = names[comp]; + std::string x = names[comp]; x += ":"; - string urate = (uncomp_rate >= 0) - ? StringPrintf("%.1f", uncomp_rate) - : string("?"); + std::string urate = (uncomp_rate >= 0) ? StrFormat("%.1f", uncomp_rate) + : std::string("?"); printf("%-7s [b %dM] bytes %6d -> %6d %4.1f%% " "comp %5.1f MB/s uncomp %5s MB/s\n", x.c_str(), @@ -359,8 +358,8 @@ static void Measure(const char* data, urate.c_str()); } -static int VerifyString(const string& input) { - string compressed; +static int VerifyString(const std::string& input) { + std::string compressed; DataEndingAtUnreadablePage i(input); const size_t written = snappy::Compress(i.data(), i.size(), &compressed); CHECK_EQ(written, compressed.size()); @@ -368,15 +367,15 @@ static int VerifyString(const string& input) { snappy::MaxCompressedLength(input.size())); CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); - string uncompressed; + std::string uncompressed; DataEndingAtUnreadablePage c(compressed); CHECK(snappy::Uncompress(c.data(), c.size(), &uncompressed)); CHECK_EQ(uncompressed, input); return uncompressed.size(); } -static void VerifyStringSink(const string& input) { - string compressed; +static void VerifyStringSink(const std::string& input) { + std::string compressed; DataEndingAtUnreadablePage i(input); const size_t written = snappy::Compress(i.data(), i.size(), &compressed); CHECK_EQ(written, compressed.size()); @@ -384,7 +383,7 @@ static void VerifyStringSink(const string& input) { snappy::MaxCompressedLength(input.size())); CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); - string uncompressed; + std::string uncompressed; uncompressed.resize(input.size()); snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed)); DataEndingAtUnreadablePage c(compressed); @@ -393,8 +392,8 @@ static void VerifyStringSink(const string& input) { CHECK_EQ(uncompressed, input); } -static void VerifyIOVec(const string& input) { - string compressed; +static void VerifyIOVec(const std::string& input) { + std::string compressed; DataEndingAtUnreadablePage i(input); const size_t written = snappy::Compress(i.data(), i.size(), &compressed); CHECK_EQ(written, compressed.size()); @@ -440,13 +439,13 @@ static void VerifyIOVec(const string& input) { // Test that data compressed by a compressor that does not // obey block sizes is uncompressed properly. -static void VerifyNonBlockedCompression(const string& input) { +static void VerifyNonBlockedCompression(const std::string& input) { if (input.length() > snappy::kBlockSize) { // We cannot test larger blocks than the maximum block size, obviously. return; } - string prefix; + std::string prefix; Varint::Append32(&prefix, input.size()); // Setup compression table @@ -455,7 +454,7 @@ static void VerifyNonBlockedCompression(const string& input) { uint16* table = wmem.GetHashTable(input.size(), &table_size); // Compress entire input in one shot - string compressed; + std::string compressed; compressed += prefix; compressed.resize(prefix.size()+snappy::MaxCompressedLength(input.size())); char* dest = string_as_array(&compressed) + prefix.size(); @@ -463,13 +462,13 @@ static void VerifyNonBlockedCompression(const string& input) { dest, table, table_size); compressed.resize(end - compressed.data()); - // Uncompress into string - string uncomp_str; + // Uncompress into std::string + std::string uncomp_str; CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str)); CHECK_EQ(uncomp_str, input); // Uncompress using source/sink - string uncomp_str2; + std::string uncomp_str2; uncomp_str2.resize(input.size()); snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2)); snappy::ByteArraySource source(compressed.data(), compressed.size()); @@ -481,28 +480,28 @@ static void VerifyNonBlockedCompression(const string& input) { static const int kNumBlocks = 10; struct iovec vec[kNumBlocks]; const int block_size = 1 + input.size() / kNumBlocks; - string iovec_data(block_size * kNumBlocks, 'x'); + std::string iovec_data(block_size * kNumBlocks, 'x'); for (int i = 0; i < kNumBlocks; i++) { vec[i].iov_base = string_as_array(&iovec_data) + i * block_size; vec[i].iov_len = block_size; } CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(), vec, kNumBlocks)); - CHECK_EQ(string(iovec_data.data(), input.size()), input); + CHECK_EQ(std::string(iovec_data.data(), input.size()), input); } } // Expand the input so that it is at least K times as big as block size -static string Expand(const string& input) { +static std::string Expand(const std::string& input) { static const int K = 3; - string data = input; + std::string data = input; while (data.size() < K * snappy::kBlockSize) { data += input; } return data; } -static int Verify(const string& input) { +static int Verify(const std::string& input) { VLOG(1) << "Verifying input of size " << input.size(); // Compress using string based routines @@ -514,7 +513,7 @@ static int Verify(const string& input) { VerifyNonBlockedCompression(input); VerifyIOVec(input); if (!input.empty()) { - const string expanded = Expand(input); + const std::string expanded = Expand(input); VerifyNonBlockedCompression(expanded); VerifyIOVec(input); } @@ -522,20 +521,20 @@ static int Verify(const string& input) { return result; } -static bool IsValidCompressedBuffer(const string& c) { +static bool IsValidCompressedBuffer(const std::string& c) { return snappy::IsValidCompressedBuffer(c.data(), c.size()); } -static bool Uncompress(const string& c, string* u) { +static bool Uncompress(const std::string& c, std::string* u) { return snappy::Uncompress(c.data(), c.size(), u); } // This test checks to ensure that snappy doesn't coredump if it gets // corrupted data. TEST(CorruptedTest, VerifyCorrupted) { - string source = "making sure we don't crash with corrupted input"; + std::string source = "making sure we don't crash with corrupted input"; VLOG(1) << source; - string dest; - string uncmp; + std::string dest; + std::string uncmp; snappy::Compress(source.data(), source.size(), &dest); // Mess around with the data. It's hard to simulate all possible @@ -582,9 +581,9 @@ TEST(CorruptedTest, VerifyCorrupted) { // try reading stuff in from a bad file. for (int i = 1; i <= 3; ++i) { - string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str(), - 0); - string uncmp; + std::string data = + ReadTestDataFile(StrFormat("baddata%d.snappy", i).c_str(), 0); + std::string uncmp; // check that we don't return a crazy length size_t ulen; CHECK(!snappy::GetUncompressedLength(data.data(), data.size(), &ulen) @@ -602,7 +601,7 @@ TEST(CorruptedTest, VerifyCorrupted) { // These mirror the compression code in snappy.cc, but are copied // here so that we can bypass some limitations in the how snappy.cc // invokes these routines. -static void AppendLiteral(string* dst, const string& literal) { +static void AppendLiteral(std::string* dst, const std::string& literal) { if (literal.empty()) return; int n = literal.size() - 1; if (n < 60) { @@ -617,12 +616,12 @@ static void AppendLiteral(string* dst, const string& literal) { n >>= 8; } dst->push_back(0 | ((59+count) << 2)); - *dst += string(number, count); + *dst += std::string(number, count); } *dst += literal; } -static void AppendCopy(string* dst, int offset, int length) { +static void AppendCopy(std::string* dst, int offset, int length) { while (length > 0) { // Figure out how much to copy in one shot int to_copy; @@ -659,23 +658,23 @@ TEST(Snappy, SimpleTests) { Verify("ab"); Verify("abc"); - Verify("aaaaaaa" + string(16, 'b') + string("aaaaa") + "abc"); - Verify("aaaaaaa" + string(256, 'b') + string("aaaaa") + "abc"); - Verify("aaaaaaa" + string(2047, 'b') + string("aaaaa") + "abc"); - Verify("aaaaaaa" + string(65536, 'b') + string("aaaaa") + "abc"); - Verify("abcaaaaaaa" + string(65536, 'b') + string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(16, 'b') + std::string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(256, 'b') + std::string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(2047, 'b') + std::string("aaaaa") + "abc"); + Verify("aaaaaaa" + std::string(65536, 'b') + std::string("aaaaa") + "abc"); + Verify("abcaaaaaaa" + std::string(65536, 'b') + std::string("aaaaa") + "abc"); } // Verify max blowup (lots of four-byte copies) TEST(Snappy, MaxBlowup) { std::mt19937 rng; std::uniform_int_distribution<int> uniform_byte(0, 255); - string input; + std::string input; for (int i = 0; i < 80000; ++i) input.push_back(static_cast<char>(uniform_byte(rng))); for (int i = 0; i < 80000; i += 4) { - string four_bytes(input.end() - i - 4, input.end() - i); + std::string four_bytes(input.end() - i - 4, input.end() - i); input.append(four_bytes); } Verify(input); @@ -696,7 +695,7 @@ TEST(Snappy, RandomData) { VLOG(0) << "Random op " << i << " of " << num_ops; } - string x; + std::string x; size_t len = uniform_4k(rng); if (i < 100) { len = 65536 + uniform_64k(rng); @@ -733,19 +732,19 @@ TEST(Snappy, FourByteOffset) { // copy manually. // The two fragments that make up the input string. - string fragment1 = "012345689abcdefghijklmnopqrstuvwxyz"; - string fragment2 = "some other string"; + std::string fragment1 = "012345689abcdefghijklmnopqrstuvwxyz"; + std::string fragment2 = "some other string"; // How many times each fragment is emitted. const int n1 = 2; const int n2 = 100000 / fragment2.size(); const int length = n1 * fragment1.size() + n2 * fragment2.size(); - string compressed; + std::string compressed; Varint::Append32(&compressed, length); AppendLiteral(&compressed, fragment1); - string src = fragment1; + std::string src = fragment1; for (int i = 0; i < n2; i++) { AppendLiteral(&compressed, fragment2); src += fragment2; @@ -754,7 +753,7 @@ TEST(Snappy, FourByteOffset) { src += fragment1; CHECK_EQ(length, src.size()); - string uncompressed; + std::string uncompressed; CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncompressed)); @@ -776,7 +775,7 @@ TEST(Snappy, IOVecEdgeCases) { iov[i].iov_len = kLengths[i]; } - string compressed; + std::string compressed; Varint::Append32(&compressed, 22); // A literal whose output crosses three blocks. @@ -837,7 +836,7 @@ TEST(Snappy, IOVecLiteralOverflow) { iov[i].iov_len = kLengths[i]; } - string compressed; + std::string compressed; Varint::Append32(&compressed, 8); AppendLiteral(&compressed, "12345678"); @@ -859,7 +858,7 @@ TEST(Snappy, IOVecCopyOverflow) { iov[i].iov_len = kLengths[i]; } - string compressed; + std::string compressed; Varint::Append32(&compressed, 8); AppendLiteral(&compressed, "123"); @@ -873,7 +872,7 @@ TEST(Snappy, IOVecCopyOverflow) { } } -static bool CheckUncompressedLength(const string& compressed, +static bool CheckUncompressedLength(const std::string& compressed, size_t* ulength) { const bool result1 = snappy::GetUncompressedLength(compressed.data(), compressed.size(), @@ -887,7 +886,7 @@ static bool CheckUncompressedLength(const string& compressed, } TEST(SnappyCorruption, TruncatedVarint) { - string compressed, uncompressed; + std::string compressed, uncompressed; size_t ulength; compressed.push_back('\xf0'); CHECK(!CheckUncompressedLength(compressed, &ulength)); @@ -897,7 +896,7 @@ TEST(SnappyCorruption, TruncatedVarint) { } TEST(SnappyCorruption, UnterminatedVarint) { - string compressed, uncompressed; + std::string compressed, uncompressed; size_t ulength; compressed.push_back('\x80'); compressed.push_back('\x80'); @@ -912,7 +911,7 @@ TEST(SnappyCorruption, UnterminatedVarint) { } TEST(SnappyCorruption, OverflowingVarint) { - string compressed, uncompressed; + std::string compressed, uncompressed; size_t ulength; compressed.push_back('\xfb'); compressed.push_back('\xff'); @@ -929,14 +928,14 @@ TEST(Snappy, ReadPastEndOfBuffer) { // Check that we do not read past end of input // Make a compressed string that ends with a single-byte literal - string compressed; + std::string compressed; Varint::Append32(&compressed, 1); AppendLiteral(&compressed, "x"); - string uncompressed; + std::string uncompressed; DataEndingAtUnreadablePage c(compressed); CHECK(snappy::Uncompress(c.data(), c.size(), &uncompressed)); - CHECK_EQ(uncompressed, string("x")); + CHECK_EQ(uncompressed, std::string("x")); } // Check for an infinite loop caused by a copy with offset==0 @@ -1065,7 +1064,7 @@ TEST(Snappy, FindMatchLengthRandom) { std::bernoulli_distribution one_in_typical_length(1.0 / kTypicalLength); for (int i = 0; i < kNumTrials; i++) { - string s, t; + std::string s, t; char a = static_cast<char>(uniform_byte(rng)); char b = static_cast<char>(uniform_byte(rng)); while (!one_in_typical_length(rng)) { @@ -1180,33 +1179,33 @@ TEST(Snappy, VerifyCharTable) { } static void CompressFile(const char* fname) { - string fullinput; + std::string fullinput; CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); - string compressed; + std::string compressed; Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false); - CHECK_OK(file::SetContents(string(fname).append(".comp"), compressed, + CHECK_OK(file::SetContents(std::string(fname).append(".comp"), compressed, file::Defaults())); } static void UncompressFile(const char* fname) { - string fullinput; + std::string fullinput; CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); size_t uncompLength; CHECK(CheckUncompressedLength(fullinput, &uncompLength)); - string uncompressed; + std::string uncompressed; uncompressed.resize(uncompLength); CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed)); - CHECK_OK(file::SetContents(string(fname).append(".uncomp"), uncompressed, + CHECK_OK(file::SetContents(std::string(fname).append(".uncomp"), uncompressed, file::Defaults())); } static void MeasureFile(const char* fname) { - string fullinput; + std::string fullinput; CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults())); printf("%-40s :\n", fname); @@ -1259,10 +1258,10 @@ static void BM_UFlat(int iters, int arg) { // Pick file to process based on "arg" CHECK_GE(arg, 0); CHECK_LT(arg, ARRAYSIZE(files)); - string contents = ReadTestDataFile(files[arg].filename, - files[arg].size_limit); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); - string zcontents; + std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents); char* dst = new char[contents.size()]; @@ -1285,10 +1284,10 @@ static void BM_UValidate(int iters, int arg) { // Pick file to process based on "arg" CHECK_GE(arg, 0); CHECK_LT(arg, ARRAYSIZE(files)); - string contents = ReadTestDataFile(files[arg].filename, - files[arg].size_limit); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); - string zcontents; + std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents); SetBenchmarkBytesProcessed(static_cast<int64>(iters) * @@ -1308,10 +1307,10 @@ static void BM_UIOVec(int iters, int arg) { // Pick file to process based on "arg" CHECK_GE(arg, 0); CHECK_LT(arg, ARRAYSIZE(files)); - string contents = ReadTestDataFile(files[arg].filename, - files[arg].size_limit); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); - string zcontents; + std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents); // Uncompress into an iovec containing ten entries. @@ -1354,10 +1353,10 @@ static void BM_UFlatSink(int iters, int arg) { // Pick file to process based on "arg" CHECK_GE(arg, 0); CHECK_LT(arg, ARRAYSIZE(files)); - string contents = ReadTestDataFile(files[arg].filename, - files[arg].size_limit); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); - string zcontents; + std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents); char* dst = new char[contents.size()]; @@ -1372,7 +1371,7 @@ static void BM_UFlatSink(int iters, int arg) { } StopBenchmarkTiming(); - string s(dst, contents.size()); + std::string s(dst, contents.size()); CHECK_EQ(contents, s); delete[] dst; @@ -1386,8 +1385,8 @@ static void BM_ZFlat(int iters, int arg) { // Pick file to process based on "arg" CHECK_GE(arg, 0); CHECK_LT(arg, ARRAYSIZE(files)); - string contents = ReadTestDataFile(files[arg].filename, - files[arg].size_limit); + std::string contents = + ReadTestDataFile(files[arg].filename, files[arg].size_limit); char* dst = new char[snappy::MaxCompressedLength(contents.size())]; @@ -1402,14 +1401,88 @@ static void BM_ZFlat(int iters, int arg) { StopBenchmarkTiming(); const double compression_ratio = static_cast<double>(zsize) / std::max<size_t>(1, contents.size()); - SetBenchmarkLabel(StringPrintf("%s (%.2f %%)", - files[arg].label, 100.0 * compression_ratio)); - VLOG(0) << StringPrintf("compression for %s: %zd -> %zd bytes", - files[arg].label, contents.size(), zsize); + SetBenchmarkLabel(StrFormat("%s (%.2f %%)", files[arg].label, + 100.0 * compression_ratio)); + VLOG(0) << StrFormat("compression for %s: %zd -> %zd bytes", + files[arg].label, static_cast<int>(contents.size()), + static_cast<int>(zsize)); delete[] dst; } BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1); +static void BM_ZFlatAll(int iters, int arg) { + StopBenchmarkTiming(); + + CHECK_EQ(arg, 0); + const int num_files = ARRAYSIZE(files); + + std::vector<std::string> contents(num_files); + std::vector<char*> dst(num_files); + + int64 total_contents_size = 0; + for (int i = 0; i < num_files; ++i) { + contents[i] = ReadTestDataFile(files[i].filename, files[i].size_limit); + dst[i] = new char[snappy::MaxCompressedLength(contents[i].size())]; + total_contents_size += contents[i].size(); + } + + SetBenchmarkBytesProcessed(static_cast<int64>(iters) * total_contents_size); + StartBenchmarkTiming(); + + size_t zsize = 0; + while (iters-- > 0) { + for (int i = 0; i < num_files; ++i) { + snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], + &zsize); + } + } + StopBenchmarkTiming(); + + for (int i = 0; i < num_files; ++i) { + delete[] dst[i]; + } + SetBenchmarkLabel(StrFormat("%d files", num_files)); +} +BENCHMARK(BM_ZFlatAll)->DenseRange(0, 0); + +static void BM_ZFlatIncreasingTableSize(int iters, int arg) { + StopBenchmarkTiming(); + + CHECK_EQ(arg, 0); + CHECK_GT(ARRAYSIZE(files), 0); + const std::string base_content = + ReadTestDataFile(files[0].filename, files[0].size_limit); + + std::vector<std::string> contents; + std::vector<char*> dst; + int64 total_contents_size = 0; + for (int table_bits = kMinHashTableBits; table_bits <= kMaxHashTableBits; + ++table_bits) { + std::string content = base_content; + content.resize(1 << table_bits); + dst.push_back(new char[snappy::MaxCompressedLength(content.size())]); + total_contents_size += content.size(); + contents.push_back(std::move(content)); + } + + size_t zsize = 0; + SetBenchmarkBytesProcessed(static_cast<int64>(iters) * total_contents_size); + StartBenchmarkTiming(); + while (iters-- > 0) { + for (int i = 0; i < contents.size(); ++i) { + snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], + &zsize); + } + } + StopBenchmarkTiming(); + + for (int i = 0; i < dst.size(); ++i) { + delete[] dst[i]; + } + SetBenchmarkLabel(StrFormat("%zd tables", contents.size())); +} +BENCHMARK(BM_ZFlatIncreasingTableSize)->DenseRange(0, 0); + } // namespace snappy int main(int argc, char** argv) { |