summaryrefslogtreecommitdiff
path: root/platform/default/src
diff options
context:
space:
mode:
authorKonstantin Käfer <mail@kkaefer.com>2018-12-13 18:45:29 +0100
committerKonstantin Käfer <mail@kkaefer.com>2018-12-13 20:39:43 +0100
commit2049ff09c2b41a5ccff693a4a64e517d47a08e4a (patch)
tree25b8f9acdb5e1b6629293bcc6f014ea4bcb6d1a3 /platform/default/src
parentb6b1067caf6ba911efbb4a64a43425ce2d729a1a (diff)
downloadqtlocation-mapboxgl-upstream/build-changes.tar.gz
[build] rework platform/default directory and add -files.txt for vendored libsupstream/build-changes
Diffstat (limited to 'platform/default/src')
-rw-r--r--platform/default/src/mbgl/gl/headless_backend.cpp85
-rw-r--r--platform/default/src/mbgl/gl/headless_backend_osmesa.cpp47
-rw-r--r--platform/default/src/mbgl/gl/headless_frontend.cpp144
-rw-r--r--platform/default/src/mbgl/layermanager/layer_manager.cpp79
-rw-r--r--platform/default/src/mbgl/map/map_snapshotter.cpp224
-rw-r--r--platform/default/src/mbgl/storage/asset_file_source.cpp81
-rw-r--r--platform/default/src/mbgl/storage/default_file_source.cpp316
-rw-r--r--platform/default/src/mbgl/storage/file_source_request.cpp37
-rw-r--r--platform/default/src/mbgl/storage/http_file_source.cpp495
-rw-r--r--platform/default/src/mbgl/storage/local_file_source.cpp81
-rw-r--r--platform/default/src/mbgl/storage/offline.cpp158
-rw-r--r--platform/default/src/mbgl/storage/offline_database.cpp1129
-rw-r--r--platform/default/src/mbgl/storage/offline_download.cpp453
-rw-r--r--platform/default/src/mbgl/storage/online_file_source.cpp488
-rw-r--r--platform/default/src/mbgl/storage/sqlite3.cpp494
-rw-r--r--platform/default/src/mbgl/test/main.cpp20
-rw-r--r--platform/default/src/mbgl/text/bidi.cpp239
-rw-r--r--platform/default/src/mbgl/text/collator.cpp79
-rw-r--r--platform/default/src/mbgl/text/local_glyph_rasterizer.cpp22
-rw-r--r--platform/default/src/mbgl/text/unaccent.cpp43
-rw-r--r--platform/default/src/mbgl/util/async_task.cpp66
-rw-r--r--platform/default/src/mbgl/util/default_thread_pool.cpp57
-rw-r--r--platform/default/src/mbgl/util/image.cpp31
-rw-r--r--platform/default/src/mbgl/util/jpeg_reader.cpp151
-rw-r--r--platform/default/src/mbgl/util/logging_stderr.cpp12
-rw-r--r--platform/default/src/mbgl/util/png_reader.cpp142
-rw-r--r--platform/default/src/mbgl/util/png_writer.cpp77
-rw-r--r--platform/default/src/mbgl/util/run_loop.cpp219
-rw-r--r--platform/default/src/mbgl/util/shared_thread_pool.cpp14
-rw-r--r--platform/default/src/mbgl/util/string_stdlib.cpp74
-rw-r--r--platform/default/src/mbgl/util/thread.cpp37
-rw-r--r--platform/default/src/mbgl/util/thread_local.cpp66
-rw-r--r--platform/default/src/mbgl/util/timer.cpp73
-rw-r--r--platform/default/src/mbgl/util/utf.cpp17
34 files changed, 5750 insertions, 0 deletions
diff --git a/platform/default/src/mbgl/gl/headless_backend.cpp b/platform/default/src/mbgl/gl/headless_backend.cpp
new file mode 100644
index 0000000000..ba08aecab7
--- /dev/null
+++ b/platform/default/src/mbgl/gl/headless_backend.cpp
@@ -0,0 +1,85 @@
+#include <mbgl/gl/headless_backend.hpp>
+#include <mbgl/gl/context.hpp>
+#include <mbgl/renderer/backend_scope.hpp>
+
+#include <cassert>
+#include <stdexcept>
+#include <type_traits>
+
+namespace mbgl {
+
+class HeadlessBackend::View {
+public:
+ View(gl::Context& context, Size size_)
+ : color(context.createRenderbuffer<gl::RenderbufferType::RGBA>(size_)),
+ depthStencil(context.createRenderbuffer<gl::RenderbufferType::DepthStencil>(size_)),
+ framebuffer(context.createFramebuffer(color, depthStencil)) {
+ }
+
+ gl::Renderbuffer<gl::RenderbufferType::RGBA> color;
+ gl::Renderbuffer<gl::RenderbufferType::DepthStencil> depthStencil;
+ gl::Framebuffer framebuffer;
+};
+
+HeadlessBackend::HeadlessBackend(Size size_)
+ : size(size_) {
+}
+
+HeadlessBackend::~HeadlessBackend() {
+ BackendScope guard { *this };
+ view.reset();
+ context.reset();
+}
+
+gl::ProcAddress HeadlessBackend::getExtensionFunctionPointer(const char* name) {
+ assert(impl);
+ return impl->getExtensionFunctionPointer(name);
+}
+
+void HeadlessBackend::activate() {
+ active = true;
+
+ if (!impl) {
+ createImpl();
+ }
+
+ assert(impl);
+ impl->activateContext();
+}
+
+void HeadlessBackend::deactivate() {
+ assert(impl);
+ impl->deactivateContext();
+ active = false;
+}
+
+void HeadlessBackend::bind() {
+ gl::Context& context_ = getContext();
+
+ if (!view) {
+ view = std::make_unique<View>(context_, size);
+ }
+
+ context_.bindFramebuffer = view->framebuffer.framebuffer;
+ context_.scissorTest = false;
+ context_.viewport = { 0, 0, size };
+}
+
+Size HeadlessBackend::getFramebufferSize() const {
+ return size;
+}
+
+void HeadlessBackend::updateAssumedState() {
+ // no-op
+}
+
+void HeadlessBackend::setSize(Size size_) {
+ size = size_;
+ view.reset();
+}
+
+PremultipliedImage HeadlessBackend::readStillImage() {
+ return getContext().readFramebuffer<PremultipliedImage>(size);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/gl/headless_backend_osmesa.cpp b/platform/default/src/mbgl/gl/headless_backend_osmesa.cpp
new file mode 100644
index 0000000000..0da1caf9af
--- /dev/null
+++ b/platform/default/src/mbgl/gl/headless_backend_osmesa.cpp
@@ -0,0 +1,47 @@
+#include <mbgl/gl/headless_backend.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <GL/osmesa.h>
+
+#include <cassert>
+
+namespace mbgl {
+
+class OSMesaBackendImpl : public HeadlessBackend::Impl {
+public:
+ OSMesaBackendImpl() {
+#if OSMESA_MAJOR_VERSION * 100 + OSMESA_MINOR_VERSION >= 305
+ glContext = OSMesaCreateContextExt(OSMESA_RGBA, 16, 0, 0, nullptr);
+#else
+ glContext = OSMesaCreateContext(OSMESA_RGBA, nullptr);
+#endif
+ if (glContext == nullptr) {
+ throw std::runtime_error("Error creating GL context object.");
+ }
+ }
+
+ ~OSMesaBackendImpl() final {
+ OSMesaDestroyContext(glContext);
+ }
+
+ gl::ProcAddress getExtensionFunctionPointer(const char* name) final {
+ return OSMesaGetProcAddress(name);
+ }
+
+ void activateContext() final {
+ if (!OSMesaMakeCurrent(glContext, &fakeBuffer, GL_UNSIGNED_BYTE, 1, 1)) {
+ throw std::runtime_error("Switching OpenGL context failed.\n");
+ }
+ }
+
+private:
+ OSMesaContext glContext = nullptr;
+ GLubyte fakeBuffer = 0;
+};
+
+void HeadlessBackend::createImpl() {
+ assert(!impl);
+ impl = std::make_unique<OSMesaBackendImpl>();
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/gl/headless_frontend.cpp b/platform/default/src/mbgl/gl/headless_frontend.cpp
new file mode 100644
index 0000000000..37b0f91f32
--- /dev/null
+++ b/platform/default/src/mbgl/gl/headless_frontend.cpp
@@ -0,0 +1,144 @@
+#include <mbgl/gl/headless_frontend.hpp>
+#include <mbgl/renderer/renderer.hpp>
+#include <mbgl/renderer/renderer_state.hpp>
+#include <mbgl/renderer/update_parameters.hpp>
+#include <mbgl/map/map.hpp>
+#include <mbgl/map/transform_state.hpp>
+#include <mbgl/util/run_loop.hpp>
+
+namespace mbgl {
+
+HeadlessFrontend::HeadlessFrontend(float pixelRatio_, FileSource& fileSource, Scheduler& scheduler, const optional<std::string> programCacheDir, GLContextMode mode, const optional<std::string> localFontFamily)
+ : HeadlessFrontend({ 256, 256 }, pixelRatio_, fileSource, scheduler, programCacheDir, mode, localFontFamily) {
+}
+
+HeadlessFrontend::HeadlessFrontend(Size size_, float pixelRatio_, FileSource& fileSource, Scheduler& scheduler, const optional<std::string> programCacheDir, GLContextMode mode, const optional<std::string> localFontFamily)
+ : size(size_),
+ pixelRatio(pixelRatio_),
+ backend({ static_cast<uint32_t>(size.width * pixelRatio),
+ static_cast<uint32_t>(size.height * pixelRatio) }),
+ asyncInvalidate([this] {
+ if (renderer && updateParameters) {
+ mbgl::BackendScope guard { backend };
+ renderer->render(*updateParameters);
+ }
+ }),
+ renderer(std::make_unique<Renderer>(backend, pixelRatio, fileSource, scheduler, mode, programCacheDir, localFontFamily)) {
+}
+
+HeadlessFrontend::~HeadlessFrontend() = default;
+
+void HeadlessFrontend::reset() {
+ assert(renderer);
+ renderer.reset();
+}
+
+void HeadlessFrontend::update(std::shared_ptr<UpdateParameters> updateParameters_) {
+ updateParameters = updateParameters_;
+ asyncInvalidate.send();
+}
+
+void HeadlessFrontend::setObserver(RendererObserver& observer_) {
+ assert(renderer);
+ renderer->setObserver(&observer_);
+}
+
+Size HeadlessFrontend::getSize() const {
+ return size;
+}
+
+Renderer* HeadlessFrontend::getRenderer() {
+ assert(renderer);
+ return renderer.get();
+}
+
+RendererBackend* HeadlessFrontend::getBackend() {
+ return &backend;
+}
+
+CameraOptions HeadlessFrontend::getCameraOptions() {
+ if (updateParameters)
+ return RendererState::getCameraOptions(*updateParameters);
+
+ static CameraOptions nullCamera;
+ return nullCamera;
+}
+
+bool HeadlessFrontend::hasImage(const std::string& id) {
+ if (updateParameters) {
+ return RendererState::hasImage(*updateParameters, id);
+ }
+
+ return false;
+}
+
+bool HeadlessFrontend::hasLayer(const std::string& id) {
+ if (updateParameters) {
+ return RendererState::hasLayer(*updateParameters, id);
+ }
+
+ return false;
+}
+
+bool HeadlessFrontend::hasSource(const std::string& id) {
+ if (updateParameters) {
+ return RendererState::hasSource(*updateParameters, id);
+ }
+
+ return false;
+}
+ScreenCoordinate HeadlessFrontend::pixelForLatLng(const LatLng& coordinate) {
+ if (updateParameters) {
+ return RendererState::pixelForLatLng(*updateParameters, coordinate);
+ }
+
+ return ScreenCoordinate {};
+}
+
+LatLng HeadlessFrontend::latLngForPixel(const ScreenCoordinate& point) {
+ if (updateParameters) {
+ return RendererState::latLngForPixel(*updateParameters, point);
+ }
+
+ return LatLng {};
+}
+
+void HeadlessFrontend::setSize(Size size_) {
+ if (size != size_) {
+ size = size_;
+ backend.setSize({ static_cast<uint32_t>(size_.width * pixelRatio),
+ static_cast<uint32_t>(size_.height * pixelRatio) });
+ }
+}
+
+PremultipliedImage HeadlessFrontend::readStillImage() {
+ return backend.readStillImage();
+}
+
+PremultipliedImage HeadlessFrontend::render(Map& map) {
+ PremultipliedImage result;
+
+ map.renderStill([&](std::exception_ptr error) {
+ if (error) {
+ std::rethrow_exception(error);
+ } else {
+ result = backend.readStillImage();
+ }
+ });
+
+ while (!result.valid()) {
+ util::RunLoop::Get()->runOnce();
+ }
+
+ return result;
+}
+
+optional<TransformState> HeadlessFrontend::getTransformState() const {
+ if (updateParameters) {
+ return updateParameters->transformState;
+ } else {
+ return {};
+ }
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/layermanager/layer_manager.cpp b/platform/default/src/mbgl/layermanager/layer_manager.cpp
new file mode 100644
index 0000000000..05d0f4d1ae
--- /dev/null
+++ b/platform/default/src/mbgl/layermanager/layer_manager.cpp
@@ -0,0 +1,79 @@
+#include <mbgl/layermanager/layer_manager.hpp>
+
+#include <mbgl/layermanager/background_layer_factory.hpp>
+#include <mbgl/layermanager/circle_layer_factory.hpp>
+#include <mbgl/layermanager/custom_layer_factory.hpp>
+#include <mbgl/layermanager/fill_extrusion_layer_factory.hpp>
+#include <mbgl/layermanager/fill_layer_factory.hpp>
+#include <mbgl/layermanager/heatmap_layer_factory.hpp>
+#include <mbgl/layermanager/hillshade_layer_factory.hpp>
+#include <mbgl/layermanager/line_layer_factory.hpp>
+#include <mbgl/layermanager/raster_layer_factory.hpp>
+#include <mbgl/layermanager/symbol_layer_factory.hpp>
+
+#include <map>
+#include <memory>
+#include <vector>
+
+namespace mbgl {
+
+class LayerManagerDefault final : public LayerManager {
+public:
+ LayerManagerDefault();
+
+private:
+ void addLayerType(std::unique_ptr<LayerFactory>);
+ // LayerManager overrides.
+ LayerFactory* getFactory(const std::string& type) noexcept final;
+ LayerFactory* getFactory(const style::LayerTypeInfo*) noexcept final;
+
+ std::vector<std::unique_ptr<LayerFactory>> factories;
+ std::map<std::string, LayerFactory*> typeToFactory;
+};
+
+LayerManagerDefault::LayerManagerDefault() {
+ addLayerType(std::make_unique<FillLayerFactory>());
+ addLayerType(std::make_unique<LineLayerFactory>());
+ addLayerType(std::make_unique<CircleLayerFactory>());
+ addLayerType(std::make_unique<SymbolLayerFactory>());
+ addLayerType(std::make_unique<RasterLayerFactory>());
+ addLayerType(std::make_unique<BackgroundLayerFactory>());
+ addLayerType(std::make_unique<HillshadeLayerFactory>());
+ addLayerType(std::make_unique<FillExtrusionLayerFactory>());
+ addLayerType(std::make_unique<HeatmapLayerFactory>());
+ addLayerType(std::make_unique<CustomLayerFactory>());
+}
+
+void LayerManagerDefault::addLayerType(std::unique_ptr<LayerFactory> factory) {
+ std::string type{factory->getTypeInfo()->type};
+ if (!type.empty()) {
+ typeToFactory.emplace(std::make_pair(std::move(type), factory.get()));
+ }
+ factories.emplace_back(std::move(factory));
+}
+
+LayerFactory* LayerManagerDefault::getFactory(const mbgl::style::LayerTypeInfo* typeInfo) noexcept {
+ assert(typeInfo);
+ for (const auto& factory: factories) {
+ if (factory->getTypeInfo() == typeInfo) {
+ return factory.get();
+ }
+ }
+ assert(false);
+ return nullptr;
+}
+
+LayerFactory* LayerManagerDefault::getFactory(const std::string& type) noexcept {
+ auto search = typeToFactory.find(type);
+ return (search != typeToFactory.end()) ? search->second : nullptr;
+}
+
+// static
+LayerManager* LayerManager::get() noexcept {
+ static LayerManagerDefault instance;
+ return &instance;
+}
+
+const bool LayerManager::annotationsEnabled = true;
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/map/map_snapshotter.cpp b/platform/default/src/mbgl/map/map_snapshotter.cpp
new file mode 100644
index 0000000000..ae14b20721
--- /dev/null
+++ b/platform/default/src/mbgl/map/map_snapshotter.cpp
@@ -0,0 +1,224 @@
+#include <mbgl/map/map_snapshotter.hpp>
+
+#include <mbgl/actor/actor_ref.hpp>
+#include <mbgl/gl/headless_frontend.hpp>
+#include <mbgl/map/map.hpp>
+#include <mbgl/map/transform_state.hpp>
+#include <mbgl/storage/file_source.hpp>
+#include <mbgl/style/style.hpp>
+#include <mbgl/util/event.hpp>
+#include <mbgl/map/transform.hpp>
+
+namespace mbgl {
+
+class MapSnapshotter::Impl {
+public:
+ Impl(FileSource*,
+ std::shared_ptr<Scheduler>,
+ const std::pair<bool, std::string> style,
+ const Size&,
+ const float pixelRatio,
+ const optional<CameraOptions> cameraOptions,
+ const optional<LatLngBounds> region,
+ const optional<std::string> programCacheDir,
+ const optional<std::string> localFontFamily = {});
+
+ void setStyleURL(std::string styleURL);
+ std::string getStyleURL() const;
+
+ void setStyleJSON(std::string styleJSON);
+ std::string getStyleJSON() const;
+
+ void setSize(Size);
+ Size getSize() const;
+
+ void setCameraOptions(CameraOptions);
+ CameraOptions getCameraOptions() const;
+
+ void setRegion(LatLngBounds);
+ LatLngBounds getRegion() const;
+
+ void snapshot(ActorRef<MapSnapshotter::Callback>);
+
+private:
+ std::shared_ptr<Scheduler> scheduler;
+ HeadlessFrontend frontend;
+ Map map;
+};
+
+MapSnapshotter::Impl::Impl(FileSource* fileSource,
+ std::shared_ptr<Scheduler> scheduler_,
+ const std::pair<bool, std::string> style,
+ const Size& size,
+ const float pixelRatio,
+ const optional<CameraOptions> cameraOptions,
+ const optional<LatLngBounds> region,
+ const optional<std::string> programCacheDir,
+ const optional<std::string> localFontFamily)
+ : scheduler(std::move(scheduler_))
+ , frontend(size, pixelRatio, *fileSource, *scheduler, programCacheDir, GLContextMode::Unique, localFontFamily)
+ , map(frontend, MapObserver::nullObserver(), size, pixelRatio, *fileSource, *scheduler, MapMode::Static) {
+
+ if (style.first) {
+ map.getStyle().loadJSON(style.second);
+ } else{
+ map.getStyle().loadURL(style.second);
+ }
+
+ if (cameraOptions) {
+ map.jumpTo(*cameraOptions);
+ }
+
+ // Set region, if specified
+ if (region) {
+ this->setRegion(*region);
+ }
+}
+
+void MapSnapshotter::Impl::snapshot(ActorRef<MapSnapshotter::Callback> callback) {
+ map.renderStill([this, callback = std::move(callback)] (std::exception_ptr error) mutable {
+
+ // Create lambda that captures the current transform state
+ // and can be used to translate for geographic to screen
+ // coordinates
+ assert (frontend.getTransformState());
+ PointForFn pointForFn { [=, center=map.getLatLng(), transformState = *frontend.getTransformState()] (const LatLng& latLng) {
+ LatLng unwrappedLatLng = latLng.wrapped();
+ unwrappedLatLng.unwrapForShortestPath(center);
+ Transform transform { transformState };
+ return transform.latLngToScreenCoordinate(unwrappedLatLng);
+ }};
+
+ // Create lambda that captures the current transform state
+ // and can be used to translate for geographic to screen
+ // coordinates
+ assert (frontend.getTransformState());
+ LatLngForFn latLngForFn { [=, transformState = *frontend.getTransformState()] (const ScreenCoordinate& screenCoordinate) {
+ Transform transform { transformState };
+ return transform.screenCoordinateToLatLng(screenCoordinate);
+ }};
+
+ // Collect all source attributions
+ std::vector<std::string> attributions;
+ for (auto source : map.getStyle().getSources()) {
+ auto attribution = source->getAttribution();
+ if (attribution) {
+ attributions.push_back(*attribution);
+ }
+ }
+
+ // Invoke callback
+ callback.invoke(
+ &MapSnapshotter::Callback::operator(),
+ error,
+ error ? PremultipliedImage() : frontend.readStillImage(),
+ std::move(attributions),
+ std::move(pointForFn),
+ std::move(latLngForFn)
+ );
+ });
+}
+
+void MapSnapshotter::Impl::setStyleURL(std::string styleURL) {
+ map.getStyle().loadURL(styleURL);
+}
+
+std::string MapSnapshotter::Impl::getStyleURL() const {
+ return map.getStyle().getURL();
+}
+
+void MapSnapshotter::Impl::setStyleJSON(std::string styleJSON) {
+ map.getStyle().loadJSON(styleJSON);
+}
+
+std::string MapSnapshotter::Impl::getStyleJSON() const {
+ return map.getStyle().getJSON();
+}
+
+void MapSnapshotter::Impl::setSize(Size size) {
+ map.setSize(size);
+ frontend.setSize(size);
+}
+
+Size MapSnapshotter::Impl::getSize() const {
+ return map.getSize();
+}
+
+void MapSnapshotter::Impl::setCameraOptions(CameraOptions cameraOptions) {
+ map.jumpTo(cameraOptions);
+}
+
+CameraOptions MapSnapshotter::Impl::getCameraOptions() const {
+ EdgeInsets insets;
+ return map.getCameraOptions(insets);
+}
+
+void MapSnapshotter::Impl::setRegion(LatLngBounds region) {
+ mbgl::EdgeInsets insets = { 0, 0, 0, 0 };
+ std::vector<LatLng> latLngs = { region.southwest(), region.northeast() };
+ map.jumpTo(map.cameraForLatLngs(latLngs, insets));
+}
+
+LatLngBounds MapSnapshotter::Impl::getRegion() const {
+ return map.latLngBoundsForCamera(getCameraOptions());
+}
+
+MapSnapshotter::MapSnapshotter(FileSource* fileSource,
+ std::shared_ptr<Scheduler> scheduler,
+ const std::pair<bool, std::string> style,
+ const Size& size,
+ const float pixelRatio,
+ const optional<CameraOptions> cameraOptions,
+ const optional<LatLngBounds> region,
+ const optional<std::string> programCacheDir,
+ const optional<std::string> localFontFamily)
+ : impl(std::make_unique<util::Thread<MapSnapshotter::Impl>>("Map Snapshotter", fileSource, std::move(scheduler), style, size, pixelRatio, cameraOptions, region, programCacheDir, localFontFamily)) {
+}
+
+MapSnapshotter::~MapSnapshotter() = default;
+
+void MapSnapshotter::snapshot(ActorRef<MapSnapshotter::Callback> callback) {
+ impl->actor().invoke(&Impl::snapshot, std::move(callback));
+}
+
+void MapSnapshotter::setStyleURL(const std::string& styleURL) {
+ impl->actor().invoke(&Impl::setStyleURL, styleURL);
+}
+
+std::string MapSnapshotter::getStyleURL() const {
+ return impl->actor().ask(&Impl::getStyleURL).get();
+}
+
+void MapSnapshotter::setStyleJSON(const std::string& styleJSON) {
+ impl->actor().invoke(&Impl::setStyleJSON, styleJSON);
+}
+
+std::string MapSnapshotter::getStyleJSON() const {
+ return impl->actor().ask(&Impl::getStyleJSON).get();
+}
+
+void MapSnapshotter::setSize(const Size& size) {
+ impl->actor().invoke(&Impl::setSize, size);
+}
+
+Size MapSnapshotter::getSize() const {
+ return impl->actor().ask(&Impl::getSize).get();
+}
+
+void MapSnapshotter::setCameraOptions(const CameraOptions& options) {
+ impl->actor().invoke(&Impl::setCameraOptions, options);
+}
+
+CameraOptions MapSnapshotter::getCameraOptions() const {
+ return impl->actor().ask(&Impl::getCameraOptions).get();
+}
+
+void MapSnapshotter::setRegion(const LatLngBounds& bounds) {
+ impl->actor().invoke(&Impl::setRegion, std::move(bounds));
+}
+
+LatLngBounds MapSnapshotter::getRegion() const {
+ return impl->actor().ask(&Impl::getRegion).get();
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/asset_file_source.cpp b/platform/default/src/mbgl/storage/asset_file_source.cpp
new file mode 100644
index 0000000000..7988654ae5
--- /dev/null
+++ b/platform/default/src/mbgl/storage/asset_file_source.cpp
@@ -0,0 +1,81 @@
+#include <mbgl/storage/asset_file_source.hpp>
+#include <mbgl/storage/file_source_request.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/thread.hpp>
+#include <mbgl/util/url.hpp>
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+namespace {
+
+const std::string assetProtocol = "asset://";
+
+} // namespace
+
+namespace mbgl {
+
+class AssetFileSource::Impl {
+public:
+ Impl(ActorRef<Impl>, std::string root_)
+ : root(std::move(root_)) {
+ }
+
+ void request(const std::string& url, ActorRef<FileSourceRequest> req) {
+ Response response;
+
+ if (!acceptsURL(url)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
+ "Invalid asset URL");
+ req.invoke(&FileSourceRequest::setResponse, response);
+ return;
+ }
+
+ // Cut off the protocol and prefix with path.
+ const auto path = root + "/" + mbgl::util::percentDecode(url.substr(assetProtocol.size()));
+ struct stat buf;
+ int result = stat(path.c_str(), &buf);
+
+ if (result == 0 && (S_IFDIR & buf.st_mode)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else if (result == -1 && errno == ENOENT) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else {
+ try {
+ response.data = std::make_shared<std::string>(util::read_file(path));
+ } catch (...) {
+ response.error = std::make_unique<Response::Error>(
+ Response::Error::Reason::Other,
+ util::toString(std::current_exception()));
+ }
+ }
+
+ req.invoke(&FileSourceRequest::setResponse, response);
+ }
+
+private:
+ std::string root;
+};
+
+AssetFileSource::AssetFileSource(const std::string& root)
+ : impl(std::make_unique<util::Thread<Impl>>("AssetFileSource", root)) {
+}
+
+AssetFileSource::~AssetFileSource() = default;
+
+std::unique_ptr<AsyncRequest> AssetFileSource::request(const Resource& resource, Callback callback) {
+ auto req = std::make_unique<FileSourceRequest>(std::move(callback));
+
+ impl->actor().invoke(&Impl::request, resource.url, req->actor());
+
+ return std::move(req);
+}
+
+bool AssetFileSource::acceptsURL(const std::string& url) {
+ return 0 == url.rfind(assetProtocol, 0);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/default_file_source.cpp b/platform/default/src/mbgl/storage/default_file_source.cpp
new file mode 100644
index 0000000000..cad68e7de9
--- /dev/null
+++ b/platform/default/src/mbgl/storage/default_file_source.cpp
@@ -0,0 +1,316 @@
+#include <mbgl/storage/default_file_source.hpp>
+#include <mbgl/storage/asset_file_source.hpp>
+#include <mbgl/storage/file_source_request.hpp>
+#include <mbgl/storage/local_file_source.hpp>
+#include <mbgl/storage/online_file_source.hpp>
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/offline_download.hpp>
+#include <mbgl/storage/resource_transform.hpp>
+
+#include <mbgl/util/platform.hpp>
+#include <mbgl/util/url.hpp>
+#include <mbgl/util/thread.hpp>
+#include <mbgl/util/work_request.hpp>
+#include <mbgl/util/stopwatch.hpp>
+
+#include <cassert>
+
+namespace mbgl {
+
+class DefaultFileSource::Impl {
+public:
+ Impl(std::shared_ptr<FileSource> assetFileSource_, std::string cachePath, uint64_t maximumCacheSize)
+ : assetFileSource(assetFileSource_)
+ , localFileSource(std::make_unique<LocalFileSource>())
+ , offlineDatabase(std::make_unique<OfflineDatabase>(cachePath, maximumCacheSize)) {
+ }
+
+ void setAPIBaseURL(const std::string& url) {
+ onlineFileSource.setAPIBaseURL(url);
+ }
+
+ std::string getAPIBaseURL() const{
+ return onlineFileSource.getAPIBaseURL();
+ }
+
+ void setAccessToken(const std::string& accessToken) {
+ onlineFileSource.setAccessToken(accessToken);
+ }
+
+ std::string getAccessToken() const {
+ return onlineFileSource.getAccessToken();
+ }
+
+ void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ onlineFileSource.setResourceTransform(std::move(transform));
+ }
+
+ void listRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->listRegions());
+ }
+
+ void createRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->createRegion(definition, metadata));
+ }
+
+ void mergeOfflineRegions(const std::string& sideDatabasePath,
+ std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->mergeDatabase(sideDatabasePath));
+ }
+
+ void updateMetadata(const int64_t regionID,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->updateMetadata(regionID, metadata));
+ }
+
+ void getRegionStatus(int64_t regionID, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) {
+ if (auto download = getDownload(regionID)) {
+ callback(download.value()->getStatus());
+ } else {
+ callback(unexpected<std::exception_ptr>(download.error()));
+ }
+ }
+
+ void deleteRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) {
+ downloads.erase(region.getID());
+ callback(offlineDatabase->deleteRegion(std::move(region)));
+ }
+
+ void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) {
+ if (auto download = getDownload(regionID)) {
+ download.value()->setObserver(std::move(observer));
+ }
+ }
+
+ void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) {
+ if (auto download = getDownload(regionID)) {
+ download.value()->setState(state);
+ }
+ }
+
+ void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) {
+ auto callback = [ref] (const Response& res) mutable {
+ ref.invoke(&FileSourceRequest::setResponse, res);
+ };
+
+ if (AssetFileSource::acceptsURL(resource.url)) {
+ //Asset request
+ tasks[req] = assetFileSource->request(resource, callback);
+ } else if (LocalFileSource::acceptsURL(resource.url)) {
+ //Local file request
+ tasks[req] = localFileSource->request(resource, callback);
+ } else {
+ // Try the offline database
+ if (resource.hasLoadingMethod(Resource::LoadingMethod::Cache)) {
+ auto offlineResponse = offlineDatabase->get(resource);
+
+ if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) {
+ if (!offlineResponse) {
+ // Ensure there's always a response that we can send, so the caller knows that
+ // there's no optional data available in the cache, when it's the only place
+ // we're supposed to load from.
+ offlineResponse.emplace();
+ offlineResponse->noContent = true;
+ offlineResponse->error = std::make_unique<Response::Error>(
+ Response::Error::Reason::NotFound, "Not found in offline database");
+ } else if (!offlineResponse->isUsable()) {
+ // Don't return resources the server requested not to show when they're stale.
+ // Even if we can't directly use the response, we may still use it to send a
+ // conditional HTTP request, which is why we're saving it above.
+ offlineResponse->error = std::make_unique<Response::Error>(
+ Response::Error::Reason::NotFound, "Cached resource is unusable");
+ }
+ callback(*offlineResponse);
+ } else if (offlineResponse) {
+ // Copy over the fields so that we can use them when making a refresh request.
+ resource.priorModified = offlineResponse->modified;
+ resource.priorExpires = offlineResponse->expires;
+ resource.priorEtag = offlineResponse->etag;
+ resource.priorData = offlineResponse->data;
+
+ if (offlineResponse->isUsable()) {
+ callback(*offlineResponse);
+ }
+ }
+ }
+
+ // Get from the online file source
+ if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) {
+ MBGL_TIMING_START(watch);
+ tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) mutable {
+ this->offlineDatabase->put(resource, onlineResponse);
+ if (resource.kind == Resource::Kind::Tile) {
+ // onlineResponse.data will be null if data not modified
+ MBGL_TIMING_FINISH(watch,
+ " Action: " << "Requesting," <<
+ " URL: " << resource.url.c_str() <<
+ " Size: " << (onlineResponse.data != nullptr ? onlineResponse.data->size() : 0) << "B," <<
+ " Time")
+ }
+ callback(onlineResponse);
+ });
+ }
+ }
+ }
+
+ void cancel(AsyncRequest* req) {
+ tasks.erase(req);
+ }
+
+ void setOfflineMapboxTileCountLimit(uint64_t limit) {
+ offlineDatabase->setOfflineMapboxTileCountLimit(limit);
+ }
+
+ void setOnlineStatus(const bool status) {
+ onlineFileSource.setOnlineStatus(status);
+ }
+
+ void put(const Resource& resource, const Response& response) {
+ offlineDatabase->put(resource, response);
+ }
+
+private:
+ expected<OfflineDownload*, std::exception_ptr> getDownload(int64_t regionID) {
+ auto it = downloads.find(regionID);
+ if (it != downloads.end()) {
+ return it->second.get();
+ }
+ auto definition = offlineDatabase->getRegionDefinition(regionID);
+ if (!definition) {
+ return unexpected<std::exception_ptr>(definition.error());
+ }
+ auto download = std::make_unique<OfflineDownload>(regionID, std::move(definition.value()),
+ *offlineDatabase, onlineFileSource);
+ return downloads.emplace(regionID, std::move(download)).first->second.get();
+ }
+
+ // shared so that destruction is done on the creating thread
+ const std::shared_ptr<FileSource> assetFileSource;
+ const std::unique_ptr<FileSource> localFileSource;
+ std::unique_ptr<OfflineDatabase> offlineDatabase;
+ OnlineFileSource onlineFileSource;
+ std::unordered_map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks;
+ std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads;
+};
+
+DefaultFileSource::DefaultFileSource(const std::string& cachePath,
+ const std::string& assetRoot,
+ uint64_t maximumCacheSize)
+ : DefaultFileSource(cachePath, std::make_unique<AssetFileSource>(assetRoot), maximumCacheSize) {
+}
+
+DefaultFileSource::DefaultFileSource(const std::string& cachePath,
+ std::unique_ptr<FileSource>&& assetFileSource_,
+ uint64_t maximumCacheSize)
+ : assetFileSource(std::move(assetFileSource_))
+ , impl(std::make_unique<util::Thread<Impl>>("DefaultFileSource", assetFileSource, cachePath, maximumCacheSize)) {
+}
+
+DefaultFileSource::~DefaultFileSource() = default;
+
+void DefaultFileSource::setAPIBaseURL(const std::string& baseURL) {
+ impl->actor().invoke(&Impl::setAPIBaseURL, baseURL);
+
+ {
+ std::lock_guard<std::mutex> lock(cachedBaseURLMutex);
+ cachedBaseURL = baseURL;
+ }
+}
+
+std::string DefaultFileSource::getAPIBaseURL() {
+ std::lock_guard<std::mutex> lock(cachedBaseURLMutex);
+ return cachedBaseURL;
+}
+
+void DefaultFileSource::setAccessToken(const std::string& accessToken) {
+ impl->actor().invoke(&Impl::setAccessToken, accessToken);
+
+ {
+ std::lock_guard<std::mutex> lock(cachedAccessTokenMutex);
+ cachedAccessToken = accessToken;
+ }
+}
+
+std::string DefaultFileSource::getAccessToken() {
+ std::lock_guard<std::mutex> lock(cachedAccessTokenMutex);
+ return cachedAccessToken;
+}
+
+void DefaultFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ impl->actor().invoke(&Impl::setResourceTransform, std::move(transform));
+}
+
+std::unique_ptr<AsyncRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
+ auto req = std::make_unique<FileSourceRequest>(std::move(callback));
+
+ req->onCancel([fs = impl->actor(), req = req.get()] () mutable { fs.invoke(&Impl::cancel, req); });
+
+ impl->actor().invoke(&Impl::request, req.get(), resource, req->actor());
+
+ return std::move(req);
+}
+
+void DefaultFileSource::listOfflineRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::listRegions, callback);
+}
+
+void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::createRegion, definition, metadata, callback);
+}
+
+void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath,
+ std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback);
+}
+
+void DefaultFileSource::updateOfflineMetadata(const int64_t regionID,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegionMetadata,
+ std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback);
+}
+
+void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) {
+ impl->actor().invoke(&Impl::deleteRegion, std::move(region), callback);
+}
+
+void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr<OfflineRegionObserver> observer) {
+ impl->actor().invoke(&Impl::setRegionObserver, region.getID(), std::move(observer));
+}
+
+void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) {
+ impl->actor().invoke(&Impl::setRegionDownloadState, region.getID(), state);
+}
+
+void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) const {
+ impl->actor().invoke(&Impl::getRegionStatus, region.getID(), callback);
+}
+
+void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const {
+ impl->actor().invoke(&Impl::setOfflineMapboxTileCountLimit, limit);
+}
+
+void DefaultFileSource::pause() {
+ impl->pause();
+}
+
+void DefaultFileSource::resume() {
+ impl->resume();
+}
+
+void DefaultFileSource::put(const Resource& resource, const Response& response) {
+ impl->actor().invoke(&Impl::put, resource, response);
+}
+
+// For testing only:
+
+void DefaultFileSource::setOnlineStatus(const bool status) {
+ impl->actor().invoke(&Impl::setOnlineStatus, status);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/file_source_request.cpp b/platform/default/src/mbgl/storage/file_source_request.cpp
new file mode 100644
index 0000000000..09ea8cc32a
--- /dev/null
+++ b/platform/default/src/mbgl/storage/file_source_request.cpp
@@ -0,0 +1,37 @@
+#include <mbgl/storage/file_source_request.hpp>
+
+#include <mbgl/actor/mailbox.hpp>
+#include <mbgl/actor/scheduler.hpp>
+
+namespace mbgl {
+
+FileSourceRequest::FileSourceRequest(FileSource::Callback&& callback)
+ : responseCallback(callback)
+ , mailbox(std::make_shared<Mailbox>(*Scheduler::GetCurrent())) {
+}
+
+FileSourceRequest::~FileSourceRequest() {
+ if (cancelCallback) {
+ cancelCallback();
+ }
+
+ mailbox->close();
+}
+
+void FileSourceRequest::onCancel(std::function<void()>&& callback) {
+ cancelCallback = std::move(callback);
+}
+
+void FileSourceRequest::setResponse(const Response& response) {
+ // Copy, because calling the callback will sometimes self
+ // destroy this object. We cannot move because this method
+ // can be called more than one.
+ auto callback = responseCallback;
+ callback(response);
+}
+
+ActorRef<FileSourceRequest> FileSourceRequest::actor() {
+ return ActorRef<FileSourceRequest>(*this, mailbox);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/http_file_source.cpp b/platform/default/src/mbgl/storage/http_file_source.cpp
new file mode 100644
index 0000000000..213b53de98
--- /dev/null
+++ b/platform/default/src/mbgl/storage/http_file_source.cpp
@@ -0,0 +1,495 @@
+#include <mbgl/storage/http_file_source.hpp>
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/timer.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/http_header.hpp>
+
+#include <curl/curl.h>
+
+// Dynamically load all cURL functions. Debian-derived systems upgraded the OpenSSL version linked
+// to in https://salsa.debian.org/debian/curl/commit/95c94957bb7e89e36e78b995fed468c42f64d18d
+// They state:
+// Rename libcurl3 to libcurl4, because libcurl exposes an SSL_CTX via
+// CURLOPT_SSL_CTX_FUNCTION, and this object changes incompatibly between
+// openssl 1.0 and openssl 1.1.
+// Since we are not accessing the underlying OpenSSL context, we don't care whether we're linking
+// against libcurl3 or libcurl4; both use the ABI version 4 which hasn't changed since 2006
+// (see https://curl.haxx.se/libcurl/abi.html). In fact, cURL's ABI compatibility is very good as
+// shown on https://abi-laboratory.pro/tracker/timeline/curl/
+// Therefore, we're dynamically loading the cURL symbols we need to avoid linking against versioned
+// symbols.
+#include <dlfcn.h>
+
+namespace curl {
+
+#define CURL_FUNCTIONS \
+ X(global_init) \
+ X(getdate) \
+ X(easy_strerror) \
+ X(easy_init) \
+ X(easy_setopt) \
+ X(easy_cleanup) \
+ X(easy_getinfo) \
+ X(easy_reset) \
+ X(multi_init) \
+ X(multi_add_handle) \
+ X(multi_remove_handle) \
+ X(multi_cleanup) \
+ X(multi_info_read) \
+ X(multi_strerror) \
+ X(multi_socket_action) \
+ X(multi_setopt) \
+ X(share_init) \
+ X(share_cleanup) \
+ X(slist_append) \
+ X(slist_free_all)
+
+#define X(name) static decltype(&curl_ ## name) name = nullptr;
+CURL_FUNCTIONS
+#undef X
+
+static void* handle = nullptr;
+
+static void* load(const char* name) {
+ void* symbol = dlsym(handle, name);
+ if (const char* error = dlerror()) {
+ fprintf(stderr, "Cannot load symbol '%s': %s\n", name, error);
+ dlclose(handle);
+ handle = nullptr;
+ abort();
+ }
+ return symbol;
+}
+
+__attribute__((constructor))
+static void load() {
+ assert(!handle);
+ handle = dlopen("libcurl.so.4", RTLD_LAZY | RTLD_LOCAL);
+ if (!handle) {
+ fprintf(stderr, "Could not open shared library '%s'\n", "libcurl.so.4");
+ abort();
+ }
+
+ #define X(name) name = (decltype(&curl_ ## name))load("curl_" #name);
+ CURL_FUNCTIONS
+ #undef X
+}
+
+__attribute__((constructor))
+static void unload() {
+ if (handle) {
+ dlclose(handle);
+ }
+}
+
+} // namespace curl
+
+
+#include <queue>
+#include <map>
+#include <cassert>
+#include <cstring>
+#include <cstdio>
+
+static void handleError(CURLMcode code) {
+ if (code != CURLM_OK) {
+ throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(code));
+ }
+}
+
+static void handleError(CURLcode code) {
+ if (code != CURLE_OK) {
+ throw std::runtime_error(std::string("CURL easy error: ") + curl::easy_strerror(code));
+ }
+}
+
+namespace mbgl {
+
+class HTTPFileSource::Impl {
+public:
+ Impl();
+ ~Impl();
+
+ static int handleSocket(CURL *handle, curl_socket_t s, int action, void *userp, void *socketp);
+ static int startTimeout(CURLM *multi, long timeout_ms, void *userp);
+ static void onTimeout(HTTPFileSource::Impl *context);
+
+ void perform(curl_socket_t s, util::RunLoop::Event event);
+ CURL *getHandle();
+ void returnHandle(CURL *handle);
+ void checkMultiInfo();
+
+ // Used as the CURL timer function to periodically check for socket updates.
+ util::Timer timeout;
+
+ // CURL multi handle that we use to request multiple URLs at the same time, without having to
+ // block and spawn threads.
+ CURLM *multi = nullptr;
+
+ // CURL share handles are used for sharing session state (e.g.)
+ CURLSH *share = nullptr;
+
+ // A queue that we use for storing resuable CURL easy handles to avoid creating and destroying
+ // them all the time.
+ std::queue<CURL *> handles;
+};
+
+class HTTPRequest : public AsyncRequest {
+public:
+ HTTPRequest(HTTPFileSource::Impl*, Resource, FileSource::Callback);
+ ~HTTPRequest() override;
+
+ void handleResult(CURLcode code);
+
+private:
+ static size_t headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp);
+ static size_t writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp);
+
+ HTTPFileSource::Impl* context = nullptr;
+ Resource resource;
+ FileSource::Callback callback;
+
+ // Will store the current response.
+ std::shared_ptr<std::string> data;
+ std::unique_ptr<Response> response;
+
+ optional<std::string> retryAfter;
+ optional<std::string> xRateLimitReset;
+
+ CURL *handle = nullptr;
+ curl_slist *headers = nullptr;
+
+ char error[CURL_ERROR_SIZE] = { 0 };
+};
+
+HTTPFileSource::Impl::Impl() {
+ if (curl::global_init(CURL_GLOBAL_ALL)) {
+ throw std::runtime_error("Could not init cURL");
+ }
+
+ share = curl::share_init();
+
+ multi = curl::multi_init();
+ handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETFUNCTION, handleSocket));
+ handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETDATA, this));
+ handleError(curl::multi_setopt(multi, CURLMOPT_TIMERFUNCTION, startTimeout));
+ handleError(curl::multi_setopt(multi, CURLMOPT_TIMERDATA, this));
+}
+
+HTTPFileSource::Impl::~Impl() {
+ while (!handles.empty()) {
+ curl::easy_cleanup(handles.front());
+ handles.pop();
+ }
+
+ curl::multi_cleanup(multi);
+ multi = nullptr;
+
+ curl::share_cleanup(share);
+ share = nullptr;
+
+ timeout.stop();
+}
+
+CURL *HTTPFileSource::Impl::getHandle() {
+ if (!handles.empty()) {
+ auto handle = handles.front();
+ handles.pop();
+ return handle;
+ } else {
+ return curl::easy_init();
+ }
+}
+
+void HTTPFileSource::Impl::returnHandle(CURL *handle) {
+ curl::easy_reset(handle);
+ handles.push(handle);
+}
+
+void HTTPFileSource::Impl::checkMultiInfo() {
+ CURLMsg *message = nullptr;
+ int pending = 0;
+
+ while ((message = curl::multi_info_read(multi, &pending))) {
+ switch (message->msg) {
+ case CURLMSG_DONE: {
+ HTTPRequest *baton = nullptr;
+ curl::easy_getinfo(message->easy_handle, CURLINFO_PRIVATE, (char *)&baton);
+ assert(baton);
+ baton->handleResult(message->data.result);
+ } break;
+
+ default:
+ // This should never happen, because there are no other message types.
+ throw std::runtime_error("CURLMsg returned unknown message type");
+ }
+ }
+}
+
+void HTTPFileSource::Impl::perform(curl_socket_t s, util::RunLoop::Event events) {
+ int flags = 0;
+
+ if (events == util::RunLoop::Event::Read) {
+ flags |= CURL_CSELECT_IN;
+ }
+ if (events == util::RunLoop::Event::Write) {
+ flags |= CURL_CSELECT_OUT;
+ }
+
+
+ int running_handles = 0;
+ curl::multi_socket_action(multi, s, flags, &running_handles);
+ checkMultiInfo();
+}
+
+int HTTPFileSource::Impl::handleSocket(CURL * /* handle */, curl_socket_t s, int action, void *userp,
+ void * /* socketp */) {
+ assert(userp);
+ auto context = reinterpret_cast<Impl *>(userp);
+
+ switch (action) {
+ case CURL_POLL_IN: {
+ using namespace std::placeholders;
+ util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Read,
+ std::bind(&Impl::perform, context, _1, _2));
+ break;
+ }
+ case CURL_POLL_OUT: {
+ using namespace std::placeholders;
+ util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Write,
+ std::bind(&Impl::perform, context, _1, _2));
+ break;
+ }
+ case CURL_POLL_REMOVE:
+ util::RunLoop::Get()->removeWatch(s);
+ break;
+ default:
+ throw std::runtime_error("Unhandled CURL socket action");
+ }
+
+ return 0;
+}
+
+void HTTPFileSource::Impl::onTimeout(Impl *context) {
+ int running_handles;
+ CURLMcode error = curl::multi_socket_action(context->multi, CURL_SOCKET_TIMEOUT, 0, &running_handles);
+ if (error != CURLM_OK) {
+ throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(error));
+ }
+ context->checkMultiInfo();
+}
+
+int HTTPFileSource::Impl::startTimeout(CURLM * /* multi */, long timeout_ms, void *userp) {
+ assert(userp);
+ auto context = reinterpret_cast<Impl *>(userp);
+
+ if (timeout_ms < 0) {
+ // A timeout of 0 ms means that the timer will invoked in the next loop iteration.
+ timeout_ms = 0;
+ }
+
+ context->timeout.stop();
+ context->timeout.start(mbgl::Milliseconds(timeout_ms), Duration::zero(),
+ std::bind(&Impl::onTimeout, context));
+
+ return 0;
+}
+
+HTTPRequest::HTTPRequest(HTTPFileSource::Impl* context_, Resource resource_, FileSource::Callback callback_)
+ : context(context_),
+ resource(std::move(resource_)),
+ callback(std::move(callback_)),
+ handle(context->getHandle()) {
+
+ // If there's already a response, set the correct etags/modified headers to make sure we are
+ // getting a 304 response if possible. This avoids redownloading unchanged data.
+ if (resource.priorEtag) {
+ const std::string header = std::string("If-None-Match: ") + *resource.priorEtag;
+ headers = curl::slist_append(headers, header.c_str());
+ } else if (resource.priorModified) {
+ const std::string time =
+ std::string("If-Modified-Since: ") + util::rfc1123(*resource.priorModified);
+ headers = curl::slist_append(headers, time.c_str());
+ }
+
+ if (headers) {
+ curl::easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
+ }
+
+ handleError(curl::easy_setopt(handle, CURLOPT_PRIVATE, this));
+ handleError(curl::easy_setopt(handle, CURLOPT_ERRORBUFFER, error));
+ handleError(curl::easy_setopt(handle, CURLOPT_CAINFO, "ca-bundle.crt"));
+ handleError(curl::easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1));
+ handleError(curl::easy_setopt(handle, CURLOPT_URL, resource.url.c_str()));
+ handleError(curl::easy_setopt(handle, CURLOPT_WRITEFUNCTION, writeCallback));
+ handleError(curl::easy_setopt(handle, CURLOPT_WRITEDATA, this));
+ handleError(curl::easy_setopt(handle, CURLOPT_HEADERFUNCTION, headerCallback));
+ handleError(curl::easy_setopt(handle, CURLOPT_HEADERDATA, this));
+#if LIBCURL_VERSION_NUM >= ((7) << 16 | (21) << 8 | 6) // Renamed in 7.21.6
+ handleError(curl::easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate"));
+#else
+ handleError(curl::easy_setopt(handle, CURLOPT_ENCODING, "gzip, deflate"));
+#endif
+ handleError(curl::easy_setopt(handle, CURLOPT_USERAGENT, "MapboxGL/1.0"));
+ handleError(curl::easy_setopt(handle, CURLOPT_SHARE, context->share));
+
+ // Start requesting the information.
+ handleError(curl::multi_add_handle(context->multi, handle));
+}
+
+HTTPRequest::~HTTPRequest() {
+ handleError(curl::multi_remove_handle(context->multi, handle));
+ context->returnHandle(handle);
+ handle = nullptr;
+
+ if (headers) {
+ curl::slist_free_all(headers);
+ headers = nullptr;
+ }
+}
+
+// This function is called when we have new data for a request. We just append it to the string
+// containing the previous data.
+size_t HTTPRequest::writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp) {
+ assert(userp);
+ auto impl = reinterpret_cast<HTTPRequest *>(userp);
+
+ if (!impl->data) {
+ impl->data = std::make_shared<std::string>();
+ }
+
+ impl->data->append((char *)contents, size * nmemb);
+ return size * nmemb;
+}
+
+// Compares the beginning of the (non-zero-terminated!) data buffer with the (zero-terminated!)
+// header string. If the data buffer contains the header string at the beginning, it returns
+// the length of the header string == begin of the value, otherwise it returns npos.
+// The comparison of the header is ASCII-case-insensitive.
+size_t headerMatches(const char *const header, const char *const buffer, const size_t length) {
+ const size_t headerLength = strlen(header);
+ if (length < headerLength) {
+ return std::string::npos;
+ }
+ size_t i = 0;
+ while (i < length && i < headerLength && std::tolower(buffer[i]) == std::tolower(header[i])) {
+ i++;
+ }
+ return i == headerLength ? i : std::string::npos;
+}
+
+size_t HTTPRequest::headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp) {
+ assert(userp);
+ auto baton = reinterpret_cast<HTTPRequest *>(userp);
+
+ if (!baton->response) {
+ baton->response = std::make_unique<Response>();
+ }
+
+ const size_t length = size * nmemb;
+ size_t begin = std::string::npos;
+ if ((begin = headerMatches("last-modified: ", buffer, length)) != std::string::npos) {
+ // Always overwrite the modification date; We might already have a value here from the
+ // Date header, but this one is more accurate.
+ const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
+ baton->response->modified = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) };
+ } else if ((begin = headerMatches("etag: ", buffer, length)) != std::string::npos) {
+ baton->response->etag = std::string(buffer + begin, length - begin - 2); // remove \r\n
+ } else if ((begin = headerMatches("cache-control: ", buffer, length)) != std::string::npos) {
+ const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
+ const auto cc = http::CacheControl::parse(value.c_str());
+ baton->response->expires = cc.toTimePoint();
+ baton->response->mustRevalidate = cc.mustRevalidate;
+ } else if ((begin = headerMatches("expires: ", buffer, length)) != std::string::npos) {
+ const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
+ baton->response->expires = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) };
+ } else if ((begin = headerMatches("retry-after: ", buffer, length)) != std::string::npos) {
+ baton->retryAfter = std::string(buffer + begin, length - begin - 2); // remove \r\n
+ } else if ((begin = headerMatches("x-rate-limit-reset: ", buffer, length)) != std::string::npos) {
+ baton->xRateLimitReset = std::string(buffer + begin, length - begin - 2); // remove \r\n
+ }
+
+ return length;
+}
+
+void HTTPRequest::handleResult(CURLcode code) {
+ // Make sure a response object exists in case we haven't got any headers or content.
+ if (!response) {
+ response = std::make_unique<Response>();
+ }
+
+ using Error = Response::Error;
+
+ // Add human-readable error code
+ if (code != CURLE_OK) {
+ switch (code) {
+ case CURLE_COULDNT_RESOLVE_PROXY:
+ case CURLE_COULDNT_RESOLVE_HOST:
+ case CURLE_COULDNT_CONNECT:
+ case CURLE_OPERATION_TIMEDOUT:
+
+ response->error = std::make_unique<Error>(
+ Error::Reason::Connection, std::string{ curl::easy_strerror(code) } + ": " + error);
+ break;
+
+ default:
+ response->error = std::make_unique<Error>(
+ Error::Reason::Other, std::string{ curl::easy_strerror(code) } + ": " + error);
+ break;
+ }
+ } else {
+ long responseCode = 0;
+ curl::easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode);
+
+ if (responseCode == 200) {
+ if (data) {
+ response->data = std::move(data);
+ } else {
+ response->data = std::make_shared<std::string>();
+ }
+ } else if (responseCode == 204 || (responseCode == 404 && resource.kind == Resource::Kind::Tile)) {
+ response->noContent = true;
+ } else if (responseCode == 304) {
+ response->notModified = true;
+ } else if (responseCode == 404) {
+ response->error =
+ std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404");
+ } else if (responseCode == 429) {
+ response->error =
+ std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429",
+ http::parseRetryHeaders(retryAfter, xRateLimitReset));
+ } else if (responseCode >= 500 && responseCode < 600) {
+ response->error =
+ std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } +
+ util::toString(responseCode));
+ } else {
+ response->error =
+ std::make_unique<Error>(Error::Reason::Other, std::string{ "HTTP status code " } +
+ util::toString(responseCode));
+ }
+ }
+
+ // Calling `callback` may result in deleting `this`. Copy data to temporaries first.
+ auto callback_ = callback;
+ auto response_ = *response;
+ callback_(response_);
+}
+
+HTTPFileSource::HTTPFileSource()
+ : impl(std::make_unique<Impl>()) {
+}
+
+HTTPFileSource::~HTTPFileSource() = default;
+
+std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource, Callback callback) {
+ return std::make_unique<HTTPRequest>(impl.get(), resource, callback);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp
new file mode 100644
index 0000000000..1b7b7b9278
--- /dev/null
+++ b/platform/default/src/mbgl/storage/local_file_source.cpp
@@ -0,0 +1,81 @@
+#include <mbgl/storage/local_file_source.hpp>
+#include <mbgl/storage/file_source_request.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/thread.hpp>
+#include <mbgl/util/url.hpp>
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#if defined(_WINDOWS) && !defined(S_ISDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
+namespace {
+
+const std::string fileProtocol = "file://";
+
+} // namespace
+
+namespace mbgl {
+
+class LocalFileSource::Impl {
+public:
+ Impl(ActorRef<Impl>) {}
+
+ void request(const std::string& url, ActorRef<FileSourceRequest> req) {
+ Response response;
+
+ if (!acceptsURL(url)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
+ "Invalid file URL");
+ req.invoke(&FileSourceRequest::setResponse, response);
+ return;
+ }
+
+ // Cut off the protocol and prefix with path.
+ const auto path = mbgl::util::percentDecode(url.substr(fileProtocol.size()));
+ struct stat buf;
+ int result = stat(path.c_str(), &buf);
+
+ if (result == 0 && S_ISDIR(buf.st_mode)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else if (result == -1 && errno == ENOENT) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else {
+ try {
+ response.data = std::make_shared<std::string>(util::read_file(path));
+ } catch (...) {
+ response.error = std::make_unique<Response::Error>(
+ Response::Error::Reason::Other,
+ util::toString(std::current_exception()));
+ }
+ }
+
+ req.invoke(&FileSourceRequest::setResponse, response);
+ }
+
+};
+
+LocalFileSource::LocalFileSource()
+ : impl(std::make_unique<util::Thread<Impl>>("LocalFileSource")) {
+}
+
+LocalFileSource::~LocalFileSource() = default;
+
+std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, Callback callback) {
+ auto req = std::make_unique<FileSourceRequest>(std::move(callback));
+
+ impl->actor().invoke(&Impl::request, resource.url, req->actor());
+
+ return std::move(req);
+}
+
+bool LocalFileSource::acceptsURL(const std::string& url) {
+ return 0 == url.rfind(fileProtocol, 0);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/offline.cpp b/platform/default/src/mbgl/storage/offline.cpp
new file mode 100644
index 0000000000..e1ec0acb31
--- /dev/null
+++ b/platform/default/src/mbgl/storage/offline.cpp
@@ -0,0 +1,158 @@
+#include <mbgl/storage/offline.hpp>
+#include <mbgl/util/tileset.hpp>
+#include <mbgl/util/projection.hpp>
+
+#include <mapbox/geojson.hpp>
+#include <mapbox/geojson/rapidjson.hpp>
+
+#include <rapidjson/document.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+
+#include <cmath>
+
+namespace mbgl {
+
+// OfflineTilePyramidRegionDefinition
+
+OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition(
+ std::string styleURL_, LatLngBounds bounds_, double minZoom_, double maxZoom_, float pixelRatio_)
+ : styleURL(std::move(styleURL_)),
+ bounds(std::move(bounds_)),
+ minZoom(minZoom_),
+ maxZoom(maxZoom_),
+ pixelRatio(pixelRatio_) {
+ if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 ||
+ !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) {
+ throw std::invalid_argument("Invalid offline region definition");
+ }
+}
+
+
+// OfflineGeometryRegionDefinition
+
+OfflineGeometryRegionDefinition::OfflineGeometryRegionDefinition(std::string styleURL_, Geometry<double> geometry_, double minZoom_, double maxZoom_, float pixelRatio_)
+ : styleURL(styleURL_)
+ , geometry(std::move(geometry_))
+ , minZoom(minZoom_)
+ , maxZoom(maxZoom_)
+ , pixelRatio(pixelRatio_) {
+ if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 ||
+ !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) {
+ throw std::invalid_argument("Invalid offline region definition");
+ }
+}
+
+OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) {
+ rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc;
+ doc.Parse<0>(region.c_str());
+
+ // validation
+
+ auto hasValidBounds = [&] {
+ return doc.HasMember("bounds") && doc["bounds"].IsArray() && doc["bounds"].Size() == 4
+ && doc["bounds"][0].IsDouble() && doc["bounds"][1].IsDouble()
+ && doc["bounds"][2].IsDouble() && doc["bounds"][3].IsDouble();
+ };
+
+ auto hasValidGeometry = [&] {
+ return doc.HasMember("geometry") && doc["geometry"].IsObject();
+ };
+
+ if (doc.HasParseError()
+ || !doc.HasMember("style_url") || !doc["style_url"].IsString()
+ || !(hasValidBounds() || hasValidGeometry())
+ || !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble()
+ || (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble())
+ || !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) {
+ throw std::runtime_error("Malformed offline region definition");
+ }
+
+ // Common properties
+
+ std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() };
+ double minZoom = doc["min_zoom"].GetDouble();
+ double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY;
+ float pixelRatio = doc["pixel_ratio"].GetDouble();
+
+ if (doc.HasMember("bounds")) {
+ return OfflineTilePyramidRegionDefinition{
+ styleURL,
+ LatLngBounds::hull(
+ LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()),
+ LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())),
+ minZoom, maxZoom, pixelRatio };
+ } else {
+ return OfflineGeometryRegionDefinition{
+ styleURL,
+ mapbox::geojson::convert<Geometry<double>>(doc["geometry"].GetObject()),
+ minZoom, maxZoom, pixelRatio };
+ };
+
+}
+
+std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) {
+ rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc;
+ doc.SetObject();
+
+ // Encode common properties
+ region.match([&](auto& _region) {
+ doc.AddMember("style_url", rapidjson::StringRef(_region.styleURL.data(), _region.styleURL.length()), doc.GetAllocator());
+ doc.AddMember("min_zoom", _region.minZoom, doc.GetAllocator());
+ if (std::isfinite(_region.maxZoom)) {
+ doc.AddMember("max_zoom", _region.maxZoom, doc.GetAllocator());
+ }
+
+ doc.AddMember("pixel_ratio", _region.pixelRatio, doc.GetAllocator());
+ });
+
+ // Encode specific properties
+ region.match(
+ [&] (const OfflineTilePyramidRegionDefinition& _region) {
+ rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType);
+ bounds.PushBack(_region.bounds.south(), doc.GetAllocator());
+ bounds.PushBack(_region.bounds.west(), doc.GetAllocator());
+ bounds.PushBack(_region.bounds.north(), doc.GetAllocator());
+ bounds.PushBack(_region.bounds.east(), doc.GetAllocator());
+ doc.AddMember("bounds", bounds, doc.GetAllocator());
+
+ },
+ [&] (const OfflineGeometryRegionDefinition& _region) {
+ doc.AddMember("geometry", mapbox::geojson::convert(_region.geometry, doc.GetAllocator()), doc.GetAllocator());
+
+ }
+ );
+
+ rapidjson::StringBuffer buffer;
+ rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+ doc.Accept(writer);
+
+ return buffer.GetString();
+}
+
+
+// OfflineRegion
+
+OfflineRegion::OfflineRegion(int64_t id_,
+ OfflineRegionDefinition definition_,
+ OfflineRegionMetadata metadata_)
+ : id(id_),
+ definition(std::move(definition_)),
+ metadata(std::move(metadata_)) {
+}
+
+OfflineRegion::OfflineRegion(OfflineRegion&&) = default;
+OfflineRegion::~OfflineRegion() = default;
+
+const OfflineRegionDefinition& OfflineRegion::getDefinition() const {
+ return definition;
+}
+
+const OfflineRegionMetadata& OfflineRegion::getMetadata() const {
+ return metadata;
+}
+
+int64_t OfflineRegion::getID() const {
+ return id;
+}
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/offline_database.cpp b/platform/default/src/mbgl/storage/offline_database.cpp
new file mode 100644
index 0000000000..7732076991
--- /dev/null
+++ b/platform/default/src/mbgl/storage/offline_database.cpp
@@ -0,0 +1,1129 @@
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/storage/sqlite3.hpp>
+#include <mbgl/util/compression.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <mbgl/storage/offline_schema.hpp>
+#include <mbgl/storage/merge_sideloaded.hpp>
+
+
+namespace mbgl {
+
+OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_)
+ : path(std::move(path_)),
+ maximumCacheSize(maximumCacheSize_) {
+ try {
+ initialize();
+ } catch (const util::IOException& ex) {
+ handleError(ex, "open database");
+ } catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "open database");
+ }
+ // Assume that we can't open the database right now and work with an empty database object.
+}
+
+OfflineDatabase::~OfflineDatabase() {
+ // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
+ // can't throw anything.
+ try {
+ statements.clear();
+ db.reset();
+ } catch (const util::IOException& ex) {
+ handleError(ex, "close database");
+ } catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "close database");
+ }
+}
+
+void OfflineDatabase::initialize() {
+ assert(!db);
+ assert(statements.empty());
+
+ db = std::make_unique<mapbox::sqlite::Database>(
+ mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate));
+ db->setBusyTimeout(Milliseconds::max());
+ db->exec("PRAGMA foreign_keys = ON");
+
+ const auto userVersion = getPragma<int64_t>("PRAGMA user_version");
+ switch (userVersion) {
+ case 0:
+ case 1:
+ // Newly created database, or old cache-only database; remove old table if it exists.
+ removeOldCacheTable();
+ createSchema();
+ return;
+ case 2:
+ migrateToVersion3();
+ // fall through
+ case 3:
+ // Removed migration, see below.
+ // fall through
+ case 4:
+ migrateToVersion5();
+ // fall through
+ case 5:
+ migrateToVersion6();
+ // fall through
+ case 6:
+ // Happy path; we're done
+ return;
+ default:
+ // Downgrade: delete the database and try to reinitialize.
+ removeExisting();
+ initialize();
+ }
+}
+
+void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex, const char* action) {
+ if (ex.code == mapbox::sqlite::ResultCode::NotADB ||
+ ex.code == mapbox::sqlite::ResultCode::Corrupt ||
+ (ex.code == mapbox::sqlite::ResultCode::ReadOnly &&
+ ex.extendedCode == mapbox::sqlite::ExtendedResultCode::ReadOnlyDBMoved)) {
+ // The database was corruped, moved away, or deleted. We're going to start fresh with a
+ // clean slate for the next operation.
+ Log::Error(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what());
+ try {
+ removeExisting();
+ } catch (const util::IOException& ioEx) {
+ handleError(ioEx, action);
+ }
+ } else {
+ // We treat the error as temporary, and pretend we have an inaccessible DB.
+ Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what());
+ }
+}
+
+void OfflineDatabase::handleError(const util::IOException& ex, const char* action) {
+ // We failed to delete the database file.
+ Log::Error(Event::Database, ex.code, "Can't %s: %s", action, ex.what());
+}
+
+void OfflineDatabase::removeExisting() {
+ Log::Warning(Event::Database, "Removing existing incompatible offline database");
+
+ statements.clear();
+ db.reset();
+
+ util::deleteFile(path);
+}
+
+void OfflineDatabase::removeOldCacheTable() {
+ assert(db);
+ db->exec("DROP TABLE IF EXISTS http_cache");
+ db->exec("VACUUM");
+}
+
+void OfflineDatabase::createSchema() {
+ assert(db);
+ db->exec("PRAGMA auto_vacuum = INCREMENTAL");
+ db->exec("PRAGMA journal_mode = DELETE");
+ db->exec("PRAGMA synchronous = FULL");
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec(offlineDatabaseSchema);
+ db->exec("PRAGMA user_version = 6");
+ transaction.commit();
+}
+
+void OfflineDatabase::migrateToVersion3() {
+ assert(db);
+ db->exec("PRAGMA auto_vacuum = INCREMENTAL");
+ db->exec("VACUUM");
+ db->exec("PRAGMA user_version = 3");
+}
+
+// Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre-
+// release development and the migration was removed entirely to avoid potential
+// conflicts from quickly (and needlessly) switching journal and sync modes.
+//
+// See: https://github.com/mapbox/mapbox-gl-native/pull/6320
+
+void OfflineDatabase::migrateToVersion5() {
+ assert(db);
+ db->exec("PRAGMA journal_mode = DELETE");
+ db->exec("PRAGMA synchronous = FULL");
+ db->exec("PRAGMA user_version = 5");
+}
+
+void OfflineDatabase::migrateToVersion6() {
+ assert(db);
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
+ db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
+ db->exec("PRAGMA user_version = 6");
+ transaction.commit();
+}
+
+mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) {
+ if (!db) {
+ initialize();
+ }
+ auto it = statements.find(sql);
+ if (it == statements.end()) {
+ it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first;
+ }
+ return *it->second;
+}
+
+optional<Response> OfflineDatabase::get(const Resource& resource) try {
+ auto result = getInternal(resource);
+ return result ? optional<Response>{ result->first } : nullopt;
+} catch (const util::IOException& ex) {
+ handleError(ex, "read resource");
+ return nullopt;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "read resource");
+ return nullopt;
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ return getTile(*resource.tileData);
+ } else {
+ return getResource(resource);
+ }
+}
+
+optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ return hasTile(*resource.tileData);
+ } else {
+ return hasResource(resource);
+ }
+}
+
+std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try {
+ if (!db) {
+ initialize();
+ }
+ mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
+ auto result = putInternal(resource, response, true);
+ transaction.commit();
+ return result;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "write resource");
+ return { false, 0 };
+}
+
+std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) {
+ if (response.error) {
+ return { false, 0 };
+ }
+
+ std::string compressedData;
+ bool compressed = false;
+ uint64_t size = 0;
+
+ if (response.data) {
+ compressedData = util::compress(*response.data);
+ compressed = compressedData.size() < response.data->size();
+ size = compressed ? compressedData.size() : response.data->size();
+ }
+
+ if (evict_ && !evict(size)) {
+ Log::Info(Event::Database, "Unable to make space for entry");
+ return { false, 0 };
+ }
+
+ bool inserted;
+
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ inserted = putTile(*resource.tileData, response,
+ compressed ? compressedData : response.data ? *response.data : "",
+ compressed);
+ } else {
+ inserted = putResource(resource, response,
+ compressed ? compressedData : response.data ? *response.data : "",
+ compressed);
+ }
+
+ return { inserted, size };
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) {
+ // Update accessed timestamp used for LRU eviction.
+ try {
+ mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") };
+ accessedQuery.bind(1, util::now());
+ accessedQuery.bind(2, resource.url);
+ accessedQuery.run();
+ } catch (const mapbox::sqlite::Exception& ex) {
+ if (ex.code == mapbox::sqlite::ResultCode::NotADB ||
+ ex.code == mapbox::sqlite::ResultCode::Corrupt) {
+ throw;
+ }
+
+ // If we don't have any indication that the database is corrupt, continue as usual.
+ Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what());
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ // 0 1 2 3 4 5
+ "SELECT etag, expires, must_revalidate, modified, data, compressed "
+ "FROM resources "
+ "WHERE url = ?") };
+ // clang-format on
+
+ query.bind(1, resource.url);
+
+ if (!query.run()) {
+ return nullopt;
+ }
+
+ Response response;
+ uint64_t size = 0;
+
+ response.etag = query.get<optional<std::string>>(0);
+ response.expires = query.get<optional<Timestamp>>(1);
+ response.mustRevalidate = query.get<bool>(2);
+ response.modified = query.get<optional<Timestamp>>(3);
+
+ auto data = query.get<optional<std::string>>(4);
+ if (!data) {
+ response.noContent = true;
+ } else if (query.get<bool>(5)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ size = data->length();
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ size = data->length();
+ }
+
+ return std::make_pair(response, size);
+}
+
+optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
+ mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") };
+ query.bind(1, resource.url);
+ if (!query.run()) {
+ return nullopt;
+ }
+
+ return query.get<optional<int64_t>>(0);
+}
+
+bool OfflineDatabase::putResource(const Resource& resource,
+ const Response& response,
+ const std::string& data,
+ bool compressed) {
+ if (response.notModified) {
+ // clang-format off
+ mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ "UPDATE resources "
+ "SET accessed = ?1, "
+ " expires = ?2, "
+ " must_revalidate = ?3 "
+ "WHERE url = ?4 ") };
+ // clang-format on
+
+ notModifiedQuery.bind(1, util::now());
+ notModifiedQuery.bind(2, response.expires);
+ notModifiedQuery.bind(3, response.mustRevalidate);
+ notModifiedQuery.bind(4, resource.url);
+ notModifiedQuery.run();
+ return false;
+ }
+
+ // We can't use REPLACE because it would change the id value.
+ // clang-format off
+ mapbox::sqlite::Query updateQuery{ getStatement(
+ "UPDATE resources "
+ "SET kind = ?1, "
+ " etag = ?2, "
+ " expires = ?3, "
+ " must_revalidate = ?4, "
+ " modified = ?5, "
+ " accessed = ?6, "
+ " data = ?7, "
+ " compressed = ?8 "
+ "WHERE url = ?9 ") };
+ // clang-format on
+
+ updateQuery.bind(1, int(resource.kind));
+ updateQuery.bind(2, response.etag);
+ updateQuery.bind(3, response.expires);
+ updateQuery.bind(4, response.mustRevalidate);
+ updateQuery.bind(5, response.modified);
+ updateQuery.bind(6, util::now());
+ updateQuery.bind(9, resource.url);
+
+ if (response.noContent) {
+ updateQuery.bind(7, nullptr);
+ updateQuery.bind(8, false);
+ } else {
+ updateQuery.bindBlob(7, data.data(), data.size(), false);
+ updateQuery.bind(8, compressed);
+ }
+
+ updateQuery.run();
+ if (updateQuery.changes() != 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
+ // clang-format on
+
+ insertQuery.bind(1, resource.url);
+ insertQuery.bind(2, int(resource.kind));
+ insertQuery.bind(3, response.etag);
+ insertQuery.bind(4, response.expires);
+ insertQuery.bind(5, response.mustRevalidate);
+ insertQuery.bind(6, response.modified);
+ insertQuery.bind(7, util::now());
+
+ if (response.noContent) {
+ insertQuery.bind(8, nullptr);
+ insertQuery.bind(9, false);
+ } else {
+ insertQuery.bindBlob(8, data.data(), data.size(), false);
+ insertQuery.bind(9, compressed);
+ }
+
+ insertQuery.run();
+
+ return true;
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
+ // Update accessed timestamp used for LRU eviction.
+ try {
+ // clang-format off
+ mapbox::sqlite::Query accessedQuery{ getStatement(
+ "UPDATE tiles "
+ "SET accessed = ?1 "
+ "WHERE url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 ") };
+ // clang-format on
+
+ accessedQuery.bind(1, util::now());
+ accessedQuery.bind(2, tile.urlTemplate);
+ accessedQuery.bind(3, tile.pixelRatio);
+ accessedQuery.bind(4, tile.x);
+ accessedQuery.bind(5, tile.y);
+ accessedQuery.bind(6, tile.z);
+ accessedQuery.run();
+ } catch (const mapbox::sqlite::Exception& ex) {
+ if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) {
+ throw;
+ }
+
+ // If we don't have any indication that the database is corrupt, continue as usual.
+ Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what());
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ // 0 1 2, 3, 4, 5
+ "SELECT etag, expires, must_revalidate, modified, data, compressed "
+ "FROM tiles "
+ "WHERE url_template = ?1 "
+ " AND pixel_ratio = ?2 "
+ " AND x = ?3 "
+ " AND y = ?4 "
+ " AND z = ?5 ") };
+ // clang-format on
+
+ query.bind(1, tile.urlTemplate);
+ query.bind(2, tile.pixelRatio);
+ query.bind(3, tile.x);
+ query.bind(4, tile.y);
+ query.bind(5, tile.z);
+
+ if (!query.run()) {
+ return nullopt;
+ }
+
+ Response response;
+ uint64_t size = 0;
+
+ response.etag = query.get<optional<std::string>>(0);
+ response.expires = query.get<optional<Timestamp>>(1);
+ response.mustRevalidate = query.get<bool>(2);
+ response.modified = query.get<optional<Timestamp>>(3);
+
+ optional<std::string> data = query.get<optional<std::string>>(4);
+ if (!data) {
+ response.noContent = true;
+ } else if (query.get<bool>(5)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ size = data->length();
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ size = data->length();
+ }
+
+ return std::make_pair(response, size);
+}
+
+optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) {
+ // clang-format off
+ mapbox::sqlite::Query size{ getStatement(
+ "SELECT length(data) "
+ "FROM tiles "
+ "WHERE url_template = ?1 "
+ " AND pixel_ratio = ?2 "
+ " AND x = ?3 "
+ " AND y = ?4 "
+ " AND z = ?5 ") };
+ // clang-format on
+
+ size.bind(1, tile.urlTemplate);
+ size.bind(2, tile.pixelRatio);
+ size.bind(3, tile.x);
+ size.bind(4, tile.y);
+ size.bind(5, tile.z);
+
+ if (!size.run()) {
+ return nullopt;
+ }
+
+ return size.get<optional<int64_t>>(0);
+}
+
+bool OfflineDatabase::putTile(const Resource::TileData& tile,
+ const Response& response,
+ const std::string& data,
+ bool compressed) {
+ if (response.notModified) {
+ // clang-format off
+ mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ "UPDATE tiles "
+ "SET accessed = ?1, "
+ " expires = ?2, "
+ " must_revalidate = ?3 "
+ "WHERE url_template = ?4 "
+ " AND pixel_ratio = ?5 "
+ " AND x = ?6 "
+ " AND y = ?7 "
+ " AND z = ?8 ") };
+ // clang-format on
+
+ notModifiedQuery.bind(1, util::now());
+ notModifiedQuery.bind(2, response.expires);
+ notModifiedQuery.bind(3, response.mustRevalidate);
+ notModifiedQuery.bind(4, tile.urlTemplate);
+ notModifiedQuery.bind(5, tile.pixelRatio);
+ notModifiedQuery.bind(6, tile.x);
+ notModifiedQuery.bind(7, tile.y);
+ notModifiedQuery.bind(8, tile.z);
+ notModifiedQuery.run();
+ return false;
+ }
+
+ // We can't use REPLACE because it would change the id value.
+
+ // clang-format off
+ mapbox::sqlite::Query updateQuery{ getStatement(
+ "UPDATE tiles "
+ "SET modified = ?1, "
+ " etag = ?2, "
+ " expires = ?3, "
+ " must_revalidate = ?4, "
+ " accessed = ?5, "
+ " data = ?6, "
+ " compressed = ?7 "
+ "WHERE url_template = ?8 "
+ " AND pixel_ratio = ?9 "
+ " AND x = ?10 "
+ " AND y = ?11 "
+ " AND z = ?12 ") };
+ // clang-format on
+
+ updateQuery.bind(1, response.modified);
+ updateQuery.bind(2, response.etag);
+ updateQuery.bind(3, response.expires);
+ updateQuery.bind(4, response.mustRevalidate);
+ updateQuery.bind(5, util::now());
+ updateQuery.bind(8, tile.urlTemplate);
+ updateQuery.bind(9, tile.pixelRatio);
+ updateQuery.bind(10, tile.x);
+ updateQuery.bind(11, tile.y);
+ updateQuery.bind(12, tile.z);
+
+ if (response.noContent) {
+ updateQuery.bind(6, nullptr);
+ updateQuery.bind(7, false);
+ } else {
+ updateQuery.bindBlob(6, data.data(), data.size(), false);
+ updateQuery.bind(7, compressed);
+ }
+
+ updateQuery.run();
+ if (updateQuery.changes() != 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
+ // clang-format on
+
+ insertQuery.bind(1, tile.urlTemplate);
+ insertQuery.bind(2, tile.pixelRatio);
+ insertQuery.bind(3, tile.x);
+ insertQuery.bind(4, tile.y);
+ insertQuery.bind(5, tile.z);
+ insertQuery.bind(6, response.modified);
+ insertQuery.bind(7, response.mustRevalidate);
+ insertQuery.bind(8, response.etag);
+ insertQuery.bind(9, response.expires);
+ insertQuery.bind(10, util::now());
+
+ if (response.noContent) {
+ insertQuery.bind(11, nullptr);
+ insertQuery.bind(12, false);
+ } else {
+ insertQuery.bindBlob(11, data.data(), data.size(), false);
+ insertQuery.bind(12, compressed);
+ }
+
+ insertQuery.run();
+
+ return true;
+}
+
+expected<OfflineRegions, std::exception_ptr> OfflineDatabase::listRegions() try {
+ mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") };
+ OfflineRegions result;
+ while (query.run()) {
+ const auto id = query.get<int64_t>(0);
+ const auto definition = query.get<std::string>(1);
+ const auto description = query.get<std::vector<uint8_t>>(2);
+ try {
+ // Construct, then move because this constructor is private.
+ OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description);
+ result.emplace_back(std::move(region));
+ } catch (const std::exception& ex) {
+ // Catch errors from malformed offline region definitions
+ // and skip them.
+ Log::Error(Event::General, "%s", ex.what());
+ }
+ }
+ // Explicit move to avoid triggering the copy constructor.
+ return { std::move(result) };
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "list regions");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+expected<OfflineRegion, std::exception_ptr>
+OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata) try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "INSERT INTO regions (definition, description) "
+ "VALUES (?1, ?2) ") };
+ // clang-format on
+
+ query.bind(1, encodeOfflineRegionDefinition(definition));
+ query.bindBlob(2, metadata);
+ query.run();
+ return OfflineRegion(query.lastInsertRowId(), definition, metadata);
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "create region");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+expected<OfflineRegions, std::exception_ptr>
+OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) {
+ try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") };
+ // clang-format on
+
+ query.bind(1, sideDatabasePath);
+ query.run();
+ } catch (const mapbox::sqlite::Exception& ex) {
+ Log::Error(Event::Database, static_cast<int>(ex.code), "Can't attach database (%s) for merge: %s", sideDatabasePath.c_str(), ex.what());
+ return unexpected<std::exception_ptr>(std::current_exception());
+ }
+ try {
+ // Support sideloaded databases at user_version = 6. Future schema version
+ // changes will need to implement migration paths for sideloaded databases at
+ // version 6.
+ auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version"));
+ const auto mainUserVersion = getPragma<int64_t>("PRAGMA user_version");
+ if (sideUserVersion < 6 || sideUserVersion != mainUserVersion) {
+ throw std::runtime_error("Merge database has incorrect user_version");
+ }
+
+ auto currentTileCount = getOfflineMapboxTileCount();
+ // clang-format off
+ mapbox::sqlite::Query queryTiles{ getStatement(
+ "SELECT COUNT(DISTINCT st.id) "
+ "FROM side.tiles st "
+ //only consider region tiles, and not ambient tiles.
+ "JOIN side.region_tiles srt ON srt.tile_id = st.id "
+ "LEFT JOIN tiles t ON st.url_template = t.url_template AND "
+ "st.pixel_ratio = t.pixel_ratio AND "
+ "st.z = t.z AND "
+ "st.x = t.x AND "
+ "st.y = t.y "
+ "WHERE t.id IS NULL "
+ "AND st.url_template LIKE 'mapbox://%' ") };
+ // clang-format on
+ queryTiles.run();
+ auto countOfTilesToMerge = queryTiles.get<int64_t>(0);
+ if ((countOfTilesToMerge + currentTileCount) > offlineMapboxTileCountLimit) {
+ throw MapboxTileLimitExceededException();
+ }
+ queryTiles.reset();
+
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec(mergeSideloadedDatabaseSQL);
+ transaction.commit();
+
+ // clang-format off
+ mapbox::sqlite::Query queryRegions{ getStatement(
+ "SELECT DISTINCT r.id, r.definition, r.description "
+ "FROM side.regions sr "
+ "JOIN regions r ON sr.definition = r.definition AND sr.description IS r.description") };
+ // clang-format on
+
+ OfflineRegions result;
+ while (queryRegions.run()) {
+ // Construct, then move because this constructor is private.
+ OfflineRegion region(queryRegions.get<int64_t>(0),
+ decodeOfflineRegionDefinition(queryRegions.get<std::string>(1)),
+ queryRegions.get<std::vector<uint8_t>>(2));
+ result.emplace_back(std::move(region));
+ }
+ db->exec("DETACH DATABASE side");
+ // Explicit move to avoid triggering the copy constructor.
+ return { std::move(result) };
+ } catch (const std::runtime_error& ex) {
+ db->exec("DETACH DATABASE side");
+ Log::Error(Event::Database, "%s", ex.what());
+
+ return unexpected<std::exception_ptr>(std::current_exception());
+ }
+ return {};
+}
+
+expected<OfflineRegionMetadata, std::exception_ptr>
+OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "UPDATE regions SET description = ?1 "
+ "WHERE id = ?2") };
+ // clang-format on
+ query.bindBlob(1, metadata);
+ query.bind(2, regionID);
+ query.run();
+
+ return metadata;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "update region metadata");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try {
+ {
+ mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") };
+ query.bind(1, region.getID());
+ query.run();
+ }
+
+ evict(0);
+ assert(db);
+ db->exec("PRAGMA incremental_vacuum");
+
+ // Ensure that the cached offlineTileCount value is recalculated.
+ offlineMapboxTileCount = {};
+ return nullptr;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "delete region");
+ return std::current_exception();
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try {
+ auto response = getInternal(resource);
+
+ if (response) {
+ markUsed(regionID, resource);
+ }
+
+ return response;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "read region resource");
+ return nullopt;
+}
+
+optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try {
+ auto response = hasInternal(resource);
+
+ if (response) {
+ markUsed(regionID, resource);
+ }
+
+ return response;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "query region resource");
+ return nullopt;
+}
+
+uint64_t OfflineDatabase::putRegionResource(int64_t regionID,
+ const Resource& resource,
+ const Response& response) try {
+ if (!db) {
+ initialize();
+ }
+ mapbox::sqlite::Transaction transaction(*db);
+ auto size = putRegionResourceInternal(regionID, resource, response);
+ transaction.commit();
+ return size;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "write region resource");
+ return 0;
+}
+
+void OfflineDatabase::putRegionResources(int64_t regionID,
+ const std::list<std::tuple<Resource, Response>>& resources,
+ OfflineRegionStatus& status) try {
+ if (!db) {
+ initialize();
+ }
+ mapbox::sqlite::Transaction transaction(*db);
+
+ // Accumulate all statistics locally first before adding them to the OfflineRegionStatus object
+ // to ensure correctness when the transaction fails.
+ uint64_t completedResourceCount = 0;
+ uint64_t completedResourceSize = 0;
+ uint64_t completedTileCount = 0;
+ uint64_t completedTileSize = 0;
+
+ for (const auto& elem : resources) {
+ const auto& resource = std::get<0>(elem);
+ const auto& response = std::get<1>(elem);
+
+ try {
+ uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response);
+ completedResourceCount++;
+ completedResourceSize += resourceSize;
+ if (resource.kind == Resource::Kind::Tile) {
+ completedTileCount += 1;
+ completedTileSize += resourceSize;
+ }
+ } catch (const MapboxTileLimitExceededException&) {
+ // Commit the rest of the batch and rethrow
+ transaction.commit();
+ throw;
+ }
+ }
+
+ // Commit the completed batch
+ transaction.commit();
+
+ status.completedResourceCount += completedResourceCount;
+ status.completedResourceSize += completedResourceSize;
+ status.completedTileCount += completedTileCount;
+ status.completedTileSize += completedTileSize;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "write region resources");
+}
+
+uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) {
+ if (exceedsOfflineMapboxTileCountLimit(resource)) {
+ throw MapboxTileLimitExceededException();
+ }
+
+ uint64_t size = putInternal(resource, response, false).second;
+ bool previouslyUnused = markUsed(regionID, resource);
+
+ if (offlineMapboxTileCount
+ && resource.kind == Resource::Kind::Tile
+ && util::mapbox::isMapboxURL(resource.url)
+ && previouslyUnused) {
+ *offlineMapboxTileCount += 1;
+ }
+
+ return size;
+}
+
+bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
+ "SELECT ?1, tiles.id "
+ "FROM tiles "
+ "WHERE url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 ") };
+ // clang-format on
+
+ const Resource::TileData& tile = *resource.tileData;
+ insertQuery.bind(1, regionID);
+ insertQuery.bind(2, tile.urlTemplate);
+ insertQuery.bind(3, tile.pixelRatio);
+ insertQuery.bind(4, tile.x);
+ insertQuery.bind(5, tile.y);
+ insertQuery.bind(6, tile.z);
+ insertQuery.run();
+
+ if (insertQuery.changes() == 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query selectQuery{ getStatement(
+ "SELECT region_id "
+ "FROM region_tiles, tiles "
+ "WHERE region_id != ?1 "
+ " AND url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 "
+ "LIMIT 1 ") };
+ // clang-format on
+
+ selectQuery.bind(1, regionID);
+ selectQuery.bind(2, tile.urlTemplate);
+ selectQuery.bind(3, tile.pixelRatio);
+ selectQuery.bind(4, tile.x);
+ selectQuery.bind(5, tile.y);
+ selectQuery.bind(6, tile.z);
+ return !selectQuery.run();
+ } else {
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
+ "SELECT ?1, resources.id "
+ "FROM resources "
+ "WHERE resources.url = ?2 ") };
+ // clang-format on
+
+ insertQuery.bind(1, regionID);
+ insertQuery.bind(2, resource.url);
+ insertQuery.run();
+
+ if (insertQuery.changes() == 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query selectQuery{ getStatement(
+ "SELECT region_id "
+ "FROM region_resources, resources "
+ "WHERE region_id != ?1 "
+ " AND resources.url = ?2 "
+ "LIMIT 1 ") };
+ // clang-format on
+
+ selectQuery.bind(1, regionID);
+ selectQuery.bind(2, resource.url);
+ return !selectQuery.run();
+ }
+}
+
+expected<OfflineRegionDefinition, std::exception_ptr> OfflineDatabase::getRegionDefinition(int64_t regionID) try {
+ mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") };
+ query.bind(1, regionID);
+ query.run();
+
+ return decodeOfflineRegionDefinition(query.get<std::string>(0));
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "load region");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+expected<OfflineRegionStatus, std::exception_ptr> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try {
+ OfflineRegionStatus result;
+
+ std::tie(result.completedResourceCount, result.completedResourceSize)
+ = getCompletedResourceCountAndSize(regionID);
+ std::tie(result.completedTileCount, result.completedTileSize)
+ = getCompletedTileCountAndSize(regionID);
+
+ result.completedResourceCount += result.completedTileCount;
+ result.completedResourceSize += result.completedTileSize;
+
+ return result;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "get region status");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(*), SUM(LENGTH(data)) "
+ "FROM region_resources, resources "
+ "WHERE region_id = ?1 "
+ "AND resource_id = resources.id ") };
+ // clang-format on
+ query.bind(1, regionID);
+ query.run();
+ return { query.get<int64_t>(0), query.get<int64_t>(1) };
+}
+
+std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(*), SUM(LENGTH(data)) "
+ "FROM region_tiles, tiles "
+ "WHERE region_id = ?1 "
+ "AND tile_id = tiles.id ") };
+ // clang-format on
+ query.bind(1, regionID);
+ query.run();
+ return { query.get<int64_t>(0), query.get<int64_t>(1) };
+}
+
+template <class T>
+T OfflineDatabase::getPragma(const char* sql) {
+ mapbox::sqlite::Query query{ getStatement(sql) };
+ query.run();
+ return query.get<T>(0);
+}
+
+// Remove least-recently used resources and tiles until the used database size,
+// as calculated by multiplying the number of in-use pages by the page size, is
+// less than the maximum cache size. Returns false if this condition cannot be
+// satisfied.
+//
+// SQLite database never shrinks in size unless we call VACCUM. We here
+// are monitoring the soft limit (i.e. number of free pages in the file)
+// and as it approaches to the hard limit (i.e. the actual file size) we
+// delete an arbitrary number of old cache entries. The free pages approach saves
+// us from calling VACCUM or keeping a running total, which can be costly.
+bool OfflineDatabase::evict(uint64_t neededFreeSize) {
+ uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
+ uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
+
+ auto usedSize = [&] {
+ return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
+ };
+
+ // The addition of pageSize is a fudge factor to account for non `data` column
+ // size, and because pages can get fragmented on the database.
+ while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
+ // clang-format off
+ mapbox::sqlite::Query accessedQuery{ getStatement(
+ "SELECT max(accessed) "
+ "FROM ( "
+ " SELECT accessed "
+ " FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resource_id = resources.id "
+ " WHERE resource_id IS NULL "
+ " UNION ALL "
+ " SELECT accessed "
+ " FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tile_id = tiles.id "
+ " WHERE tile_id IS NULL "
+ " ORDER BY accessed ASC LIMIT ?1 "
+ ") "
+ ) };
+ accessedQuery.bind(1, 50);
+ // clang-format on
+ if (!accessedQuery.run()) {
+ return false;
+ }
+ Timestamp accessed = accessedQuery.get<Timestamp>(0);
+
+ // clang-format off
+ mapbox::sqlite::Query resourceQuery{ getStatement(
+ "DELETE FROM resources "
+ "WHERE id IN ( "
+ " SELECT id FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resource_id = resources.id "
+ " WHERE resource_id IS NULL "
+ " AND accessed <= ?1 "
+ ") ") };
+ // clang-format on
+ resourceQuery.bind(1, accessed);
+ resourceQuery.run();
+ const uint64_t resourceChanges = resourceQuery.changes();
+
+ // clang-format off
+ mapbox::sqlite::Query tileQuery{ getStatement(
+ "DELETE FROM tiles "
+ "WHERE id IN ( "
+ " SELECT id FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tile_id = tiles.id "
+ " WHERE tile_id IS NULL "
+ " AND accessed <= ?1 "
+ ") ") };
+ // clang-format on
+ tileQuery.bind(1, accessed);
+ tileQuery.run();
+ const uint64_t tileChanges = tileQuery.changes();
+
+ // The cached value of offlineTileCount does not need to be updated
+ // here because only non-offline tiles can be removed by eviction.
+
+ if (resourceChanges == 0 && tileChanges == 0) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
+ offlineMapboxTileCountLimit = limit;
+}
+
+uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
+ return offlineMapboxTileCountLimit;
+}
+
+bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
+ return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
+}
+
+uint64_t OfflineDatabase::getOfflineMapboxTileCount() try {
+ // Calculating this on every call would be much simpler than caching and
+ // manually updating the value, but it would make offline downloads an O(n²)
+ // operation, because the database query below involves an index scan of
+ // region_tiles.
+
+ if (offlineMapboxTileCount) {
+ return *offlineMapboxTileCount;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(DISTINCT id) "
+ "FROM region_tiles, tiles "
+ "WHERE tile_id = tiles.id "
+ "AND url_template LIKE 'mapbox://%' ") };
+ // clang-format on
+
+ query.run();
+
+ offlineMapboxTileCount = query.get<int64_t>(0);
+ return *offlineMapboxTileCount;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "get offline Mapbox tile count");
+ return std::numeric_limits<uint64_t>::max();
+}
+
+bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) {
+ return resource.kind == Resource::Kind::Tile
+ && util::mapbox::isMapboxURL(resource.url)
+ && offlineMapboxTileCountLimitExceeded();
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/offline_download.cpp b/platform/default/src/mbgl/storage/offline_download.cpp
new file mode 100644
index 0000000000..c97797a5a2
--- /dev/null
+++ b/platform/default/src/mbgl/storage/offline_download.cpp
@@ -0,0 +1,453 @@
+#include <mbgl/storage/online_file_source.hpp>
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/offline_download.hpp>
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/storage/http_file_source.hpp>
+#include <mbgl/style/parser.hpp>
+#include <mbgl/style/sources/vector_source.hpp>
+#include <mbgl/style/sources/raster_source.hpp>
+#include <mbgl/style/sources/raster_dem_source.hpp>
+#include <mbgl/style/sources/geojson_source.hpp>
+#include <mbgl/style/sources/image_source.hpp>
+#include <mbgl/style/conversion/json.hpp>
+#include <mbgl/style/conversion/tileset.hpp>
+#include <mbgl/text/glyph.hpp>
+#include <mbgl/util/mapbox.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/tile_cover.hpp>
+#include <mbgl/util/tileset.hpp>
+
+#include <set>
+
+namespace mbgl {
+
+using namespace style;
+
+// Generic functions
+
+template <class RegionDefinition>
+Range<uint8_t> coveringZoomRange(const RegionDefinition& definition,
+ style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) {
+ double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min);
+ double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max);
+
+ assert(minZ >= 0);
+ assert(maxZ >= 0);
+ assert(minZ < std::numeric_limits<uint8_t>::max());
+ assert(maxZ < std::numeric_limits<uint8_t>::max());
+ return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) };
+}
+
+template <class Geometry, class Fn>
+void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) {
+ util::TileCover cover(geometry, z);
+ while (cover.hasNext()) {
+ fn(cover.next()->canonical);
+ }
+}
+
+
+template <class Fn>
+void tileCover(const OfflineRegionDefinition& definition, style::SourceType type,
+ uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) {
+ const Range<uint8_t> clampedZoomRange =
+ definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
+
+ for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
+ definition.match(
+ [&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); },
+ [&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); }
+ );
+ }
+}
+
+uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type,
+ uint16_t tileSize, const Range<uint8_t>& zoomRange) {
+
+ const Range<uint8_t> clampedZoomRange =
+ definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
+
+ unsigned long result = 0;;
+ for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
+ result += definition.match(
+ [&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); },
+ [&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); }
+ );
+ }
+
+ return result;
+}
+
+// OfflineDownload
+
+OfflineDownload::OfflineDownload(int64_t id_,
+ OfflineRegionDefinition&& definition_,
+ OfflineDatabase& offlineDatabase_,
+ OnlineFileSource& onlineFileSource_)
+ : id(id_),
+ definition(definition_),
+ offlineDatabase(offlineDatabase_),
+ onlineFileSource(onlineFileSource_) {
+ setObserver(nullptr);
+}
+
+OfflineDownload::~OfflineDownload() = default;
+
+void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) {
+ observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>();
+}
+
+void OfflineDownload::setState(OfflineRegionDownloadState state) {
+ if (status.downloadState == state) {
+ return;
+ }
+
+ status.downloadState = state;
+
+ if (status.downloadState == OfflineRegionDownloadState::Active) {
+ activateDownload();
+ } else {
+ deactivateDownload();
+ }
+
+ observer->statusChanged(status);
+}
+
+OfflineRegionStatus OfflineDownload::getStatus() const {
+ if (status.downloadState == OfflineRegionDownloadState::Active) {
+ return status;
+ }
+
+ auto result = offlineDatabase.getRegionCompletedStatus(id);
+ if (!result) {
+ // We can't find this offline region because the database is unavailable, or the download
+ // does not exist.
+ return {};
+ }
+
+ result->requiredResourceCount++;
+ optional<Response> styleResponse =
+ offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })));
+ if (!styleResponse) {
+ return *result;
+ }
+
+ style::Parser parser;
+ parser.parse(*styleResponse->data);
+
+ result->requiredResourceCountIsPrecise = true;
+
+ for (const auto& source : parser.sources) {
+ SourceType type = source->getType();
+
+ auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
+ if (urlOrTileset.is<Tileset>()) {
+ result->requiredResourceCount +=
+ tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange);
+ } else {
+ result->requiredResourceCount += 1;
+ const auto& url = urlOrTileset.get<std::string>();
+ optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
+ if (sourceResponse) {
+ style::conversion::Error error;
+ optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
+ if (tileset) {
+ result->requiredResourceCount +=
+ tileCount(definition, type, tileSize, (*tileset).zoomRange);
+ }
+ } else {
+ result->requiredResourceCountIsPrecise = false;
+ }
+ }
+ };
+
+ switch (type) {
+ case SourceType::Vector: {
+ const auto& vectorSource = *source->as<VectorSource>();
+ handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
+ break;
+ }
+
+ case SourceType::Raster: {
+ const auto& rasterSource = *source->as<RasterSource>();
+ handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
+ break;
+ }
+
+ case SourceType::RasterDEM: {
+ const auto& rasterDEMSource = *source->as<RasterDEMSource>();
+ handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
+ break;
+ }
+
+ case SourceType::GeoJSON: {
+ const auto& geojsonSource = *source->as<GeoJSONSource>();
+ if (geojsonSource.getURL()) {
+ result->requiredResourceCount += 1;
+ }
+ break;
+ }
+
+ case SourceType::Image: {
+ const auto& imageSource = *source->as<ImageSource>();
+ if (imageSource.getURL()) {
+ result->requiredResourceCount += 1;
+ }
+ break;
+ }
+
+ case SourceType::Video:
+ case SourceType::Annotations:
+ case SourceType::CustomVector:
+ break;
+ }
+ }
+
+ if (!parser.glyphURL.empty()) {
+ result->requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK;
+ }
+
+ if (!parser.spriteURL.empty()) {
+ result->requiredResourceCount += 4;
+ }
+
+ return *result;
+}
+
+void OfflineDownload::activateDownload() {
+ status = OfflineRegionStatus();
+ status.downloadState = OfflineRegionDownloadState::Active;
+ status.requiredResourceCount++;
+ ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }), Resource::Priority::Low),
+ [&](Response styleResponse) {
+ status.requiredResourceCountIsPrecise = true;
+
+ style::Parser parser;
+ parser.parse(*styleResponse.data);
+
+ for (const auto& source : parser.sources) {
+ SourceType type = source->getType();
+
+ auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
+ if (urlOrTileset.is<Tileset>()) {
+ queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
+ } else {
+ const auto& url = urlOrTileset.get<std::string>();
+ status.requiredResourceCountIsPrecise = false;
+ status.requiredResourceCount++;
+ requiredSourceURLs.insert(url);
+
+ ensureResource(Resource::source(url, Resource::Priority::Low), [=](Response sourceResponse) {
+ style::conversion::Error error;
+ optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
+ if (tileset) {
+ util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
+ queueTiles(type, tileSize, *tileset);
+
+ requiredSourceURLs.erase(url);
+ if (requiredSourceURLs.empty()) {
+ status.requiredResourceCountIsPrecise = true;
+ }
+ }
+ });
+ }
+ };
+
+ switch (type) {
+ case SourceType::Vector: {
+ const auto& vectorSource = *source->as<VectorSource>();
+ handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
+ break;
+ }
+
+ case SourceType::Raster: {
+ const auto& rasterSource = *source->as<RasterSource>();
+ handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
+ break;
+ }
+
+ case SourceType::RasterDEM: {
+ const auto& rasterDEMSource = *source->as<RasterDEMSource>();
+ handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
+ break;
+ }
+
+ case SourceType::GeoJSON: {
+ const auto& geojsonSource = *source->as<GeoJSONSource>();
+ if (geojsonSource.getURL()) {
+ queueResource(Resource::source(*geojsonSource.getURL()));
+ }
+ break;
+ }
+
+ case SourceType::Image: {
+ const auto& imageSource = *source->as<ImageSource>();
+ auto imageUrl = imageSource.getURL();
+ if (imageUrl && !imageUrl->empty()) {
+ queueResource(Resource::image(*imageUrl));
+ }
+ break;
+ }
+
+ case SourceType::Video:
+ case SourceType::Annotations:
+ case SourceType::CustomVector:
+ break;
+ }
+ }
+
+ if (!parser.glyphURL.empty()) {
+ for (const auto& fontStack : parser.fontStacks()) {
+ for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
+ queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
+ }
+ }
+ }
+
+ if (!parser.spriteURL.empty()) {
+ // Always request 1x and @2x sprite images for portability.
+ queueResource(Resource::spriteImage(parser.spriteURL, 1));
+ queueResource(Resource::spriteImage(parser.spriteURL, 2));
+ queueResource(Resource::spriteJSON(parser.spriteURL, 1));
+ queueResource(Resource::spriteJSON(parser.spriteURL, 2));
+ }
+
+ continueDownload();
+ });
+}
+
+/*
+ Fill up our own request queue by requesting the next few resources. This is called
+ when activating the download, or when a request completes successfully.
+
+ Note "successfully"; it's not called when a requests receives an error. A request
+ that errors will be retried after some delay. So in that sense it's still "active"
+ and consuming resources, notably the request object, its timer, and network resources
+ when the timer fires.
+
+ We could try to squeeze in subsequent requests while we wait for the errored request
+ to retry. But that risks overloading the upstream request queue -- defeating our own
+ metering -- if there are a lot of errored requests that all come up for retry at the
+ same time. And many times, the cause of a request error will apply to many requests
+ of the same type. For instance if a server is unreachable, all the requests to that
+ host are going to error. In that case, continuing to try subsequent resources after
+ the first few errors is fruitless anyway.
+*/
+void OfflineDownload::continueDownload() {
+ if (resourcesRemaining.empty() && status.complete()) {
+ setState(OfflineRegionDownloadState::Inactive);
+ return;
+ }
+
+ while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) {
+ ensureResource(resourcesRemaining.front());
+ resourcesRemaining.pop_front();
+ }
+}
+
+void OfflineDownload::deactivateDownload() {
+ requiredSourceURLs.clear();
+ resourcesRemaining.clear();
+ requests.clear();
+}
+
+void OfflineDownload::queueResource(Resource resource) {
+ resource.setPriority(Resource::Priority::Low);
+ status.requiredResourceCount++;
+ resourcesRemaining.push_front(std::move(resource));
+}
+
+void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
+ tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) {
+ status.requiredResourceCount++;
+ resourcesRemaining.push_back(Resource::tile(
+ tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x,
+ tile.y, tile.z, tileset.scheme, Resource::Priority::Low));
+ });
+}
+
+void OfflineDownload::ensureResource(const Resource& resource,
+ std::function<void(Response)> callback) {
+ assert(resource.priority == Resource::Priority::Low);
+
+ auto workRequestsIt = requests.insert(requests.begin(), nullptr);
+ *workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
+ requests.erase(workRequestsIt);
+
+ auto getResourceSizeInDatabase = [&] () -> optional<int64_t> {
+ if (!callback) {
+ return offlineDatabase.hasRegionResource(id, resource);
+ }
+ optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource);
+ if (!response) {
+ return {};
+ }
+ callback(response->first);
+ return response->second;
+ };
+
+ optional<int64_t> offlineResponse = getResourceSizeInDatabase();
+ if (offlineResponse) {
+ status.completedResourceCount++;
+ status.completedResourceSize += *offlineResponse;
+ if (resource.kind == Resource::Kind::Tile) {
+ status.completedTileCount += 1;
+ status.completedTileSize += *offlineResponse;
+ }
+
+ observer->statusChanged(status);
+ continueDownload();
+ return;
+ }
+
+ if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
+ onMapboxTileCountLimitExceeded();
+ return;
+ }
+
+ auto fileRequestsIt = requests.insert(requests.begin(), nullptr);
+ *fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) {
+ if (onlineResponse.error) {
+ observer->responseError(*onlineResponse.error);
+ return;
+ }
+
+ requests.erase(fileRequestsIt);
+
+ if (callback) {
+ callback(onlineResponse);
+ }
+
+ // Queue up for batched insertion
+ buffer.emplace_back(resource, onlineResponse);
+
+ // Flush buffer periodically
+ if (buffer.size() == 64 || resourcesRemaining.size() == 0) {
+ try {
+ offlineDatabase.putRegionResources(id, buffer, status);
+ } catch (const MapboxTileLimitExceededException&) {
+ onMapboxTileCountLimitExceeded();
+ return;
+ }
+
+ buffer.clear();
+ observer->statusChanged(status);
+ }
+
+ if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
+ onMapboxTileCountLimitExceeded();
+ return;
+ }
+
+ continueDownload();
+ });
+ });
+}
+
+void OfflineDownload::onMapboxTileCountLimitExceeded() {
+ observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
+ setState(OfflineRegionDownloadState::Inactive);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/online_file_source.cpp b/platform/default/src/mbgl/storage/online_file_source.cpp
new file mode 100644
index 0000000000..fce1c3e2b6
--- /dev/null
+++ b/platform/default/src/mbgl/storage/online_file_source.cpp
@@ -0,0 +1,488 @@
+#include <mbgl/storage/online_file_source.hpp>
+#include <mbgl/storage/http_file_source.hpp>
+#include <mbgl/storage/network_status.hpp>
+
+#include <mbgl/storage/resource_transform.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <mbgl/actor/mailbox.hpp>
+#include <mbgl/util/constants.hpp>
+#include <mbgl/util/mapbox.hpp>
+#include <mbgl/util/exception.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/async_task.hpp>
+#include <mbgl/util/noncopyable.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/timer.hpp>
+#include <mbgl/util/http_timeout.hpp>
+
+#include <algorithm>
+#include <cassert>
+#include <list>
+#include <unordered_set>
+#include <unordered_map>
+
+namespace mbgl {
+
+static uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20;
+
+class OnlineFileRequest : public AsyncRequest {
+public:
+ using Callback = std::function<void (Response)>;
+
+ OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&);
+ ~OnlineFileRequest() override;
+
+ void networkIsReachableAgain();
+ void schedule();
+ void schedule(optional<Timestamp> expires);
+ void completed(Response);
+
+ void setTransformedURL(const std::string&& url);
+ ActorRef<OnlineFileRequest> actor();
+
+ OnlineFileSource::Impl& impl;
+ Resource resource;
+ std::unique_ptr<AsyncRequest> request;
+ util::Timer timer;
+ Callback callback;
+
+ std::shared_ptr<Mailbox> mailbox;
+
+ // Counts the number of times a response was already expired when received. We're using
+ // this to add a delay when making a new request so we don't keep retrying immediately
+ // in case of a server serving expired tiles.
+ uint32_t expiredRequests = 0;
+
+ // Counts the number of subsequent failed requests. We're using this value for exponential
+ // backoff when retrying requests.
+ uint32_t failedRequests = 0;
+ Response::Error::Reason failedRequestReason = Response::Error::Reason::Success;
+ optional<Timestamp> retryAfter;
+};
+
+class OnlineFileSource::Impl {
+public:
+ Impl() {
+ NetworkStatus::Subscribe(&reachability);
+ setMaximumConcurrentRequests(DEFAULT_MAXIMUM_CONCURRENT_REQUESTS);
+ }
+
+ ~Impl() {
+ NetworkStatus::Unsubscribe(&reachability);
+ }
+
+ void add(OnlineFileRequest* request) {
+ allRequests.insert(request);
+ if (resourceTransform) {
+ // Request the ResourceTransform actor a new url and replace the resource url with the
+ // transformed one before proceeding to schedule the request.
+ resourceTransform->invoke(&ResourceTransform::transform, request->resource.kind,
+ std::move(request->resource.url), [ref = request->actor()](const std::string&& url) mutable {
+ ref.invoke(&OnlineFileRequest::setTransformedURL, std::move(url));
+ });
+ } else {
+ request->schedule();
+ }
+ }
+
+ void remove(OnlineFileRequest* request) {
+ allRequests.erase(request);
+ if (activeRequests.erase(request)) {
+ activatePendingRequest();
+ } else {
+ pendingRequests.remove(request);
+ }
+ }
+
+ void activateOrQueueRequest(OnlineFileRequest* request) {
+ assert(allRequests.find(request) != allRequests.end());
+ assert(activeRequests.find(request) == activeRequests.end());
+ assert(!request->request);
+
+ if (activeRequests.size() >= getMaximumConcurrentRequests()) {
+ queueRequest(request);
+ } else {
+ activateRequest(request);
+ }
+ }
+
+ void queueRequest(OnlineFileRequest* request) {
+ pendingRequests.insert(request);
+ }
+
+ void activateRequest(OnlineFileRequest* request) {
+ auto callback = [=](Response response) {
+ activeRequests.erase(request);
+ request->request.reset();
+ request->completed(response);
+ activatePendingRequest();
+ };
+
+ activeRequests.insert(request);
+
+ if (online) {
+ request->request = httpFileSource.request(request->resource, callback);
+ } else {
+ Response response;
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection,
+ "Online connectivity is disabled.");
+ callback(response);
+ }
+
+ }
+
+ void activatePendingRequest() {
+
+ auto request = pendingRequests.pop();
+
+ if (request) {
+ activateRequest(*request);
+ }
+ }
+
+ bool isPending(OnlineFileRequest* request) {
+ return pendingRequests.contains(request);
+ }
+
+ bool isActive(OnlineFileRequest* request) {
+ return activeRequests.find(request) != activeRequests.end();
+ }
+
+ void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ resourceTransform = std::move(transform);
+ }
+
+ void setOnlineStatus(const bool status) {
+ online = status;
+ networkIsReachableAgain();
+ }
+
+ uint32_t getMaximumConcurrentRequests() const {
+ return maximumConcurrentRequests;
+ }
+
+ void setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) {
+ maximumConcurrentRequests = maximumConcurrentRequests_;
+ }
+
+private:
+
+ void networkIsReachableAgain() {
+ for (auto& request : allRequests) {
+ request->networkIsReachableAgain();
+ }
+ }
+
+ // Using Pending Requests as an priority queue which processes
+ // file requests in a FIFO manner but prefers regular requests
+ // over offline requests with a low priority such that low priority
+ // requests do not throttle regular requests.
+ //
+ // The order of a queue is therefore:
+ //
+ // hi0 -- hi1 -- hi2 -- hi3 -- lo0 -- lo1 --lo2
+ // ^
+ // firstLowPriorityRequest
+
+ struct PendingRequests {
+ PendingRequests() : queue(), firstLowPriorityRequest(queue.begin()) {}
+
+ std::list<OnlineFileRequest*> queue;
+ std::list<OnlineFileRequest*>::iterator firstLowPriorityRequest;
+
+ void remove(const OnlineFileRequest* request) {
+ auto it = std::find(queue.begin(), queue.end(), request);
+ if (it != queue.end()) {
+ if (it == firstLowPriorityRequest) {
+ firstLowPriorityRequest++;
+ }
+ queue.erase(it);
+ }
+ }
+
+ void insert(OnlineFileRequest* request) {
+ if (request->resource.priority == Resource::Priority::Regular) {
+ firstLowPriorityRequest = queue.insert(firstLowPriorityRequest, request);
+ firstLowPriorityRequest++;
+ }
+ else {
+ if (firstLowPriorityRequest == queue.end()) {
+ firstLowPriorityRequest = queue.insert(queue.end(), request);
+ }
+ else {
+ queue.insert(queue.end(), request);
+ }
+ }
+ }
+
+
+ optional<OnlineFileRequest*> pop() {
+ if (queue.empty()) {
+ return optional<OnlineFileRequest*>();
+ }
+
+ if (queue.begin() == firstLowPriorityRequest) {
+ firstLowPriorityRequest++;
+ }
+
+ OnlineFileRequest* next = queue.front();
+ queue.pop_front();
+ return optional<OnlineFileRequest*>(next);
+ }
+
+ bool contains(OnlineFileRequest* request) const {
+ return (std::find(queue.begin(), queue.end(), request) != queue.end());
+ }
+
+ };
+
+ optional<ActorRef<ResourceTransform>> resourceTransform;
+
+ /**
+ * The lifetime of a request is:
+ *
+ * 1. Waiting for timeout (revalidation or retry)
+ * 2. Pending (waiting for room in the active set)
+ * 3. Active (open network connection)
+ * 4. Back to #1
+ *
+ * Requests in any state are in `allRequests`. Requests in the pending state are in
+ * `pendingRequests`. Requests in the active state are in `activeRequests`.
+ */
+ std::unordered_set<OnlineFileRequest*> allRequests;
+
+ PendingRequests pendingRequests;
+
+ std::unordered_set<OnlineFileRequest*> activeRequests;
+
+ bool online = true;
+ uint32_t maximumConcurrentRequests;
+ HTTPFileSource httpFileSource;
+ util::AsyncTask reachability { std::bind(&Impl::networkIsReachableAgain, this) };
+};
+
+OnlineFileSource::OnlineFileSource()
+ : impl(std::make_unique<Impl>()) {
+}
+
+OnlineFileSource::~OnlineFileSource() = default;
+
+std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) {
+ Resource res = resource;
+
+ switch (resource.kind) {
+ case Resource::Kind::Unknown:
+ case Resource::Kind::Image:
+ break;
+
+ case Resource::Kind::Style:
+ res.url = mbgl::util::mapbox::normalizeStyleURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::Source:
+ res.url = util::mapbox::normalizeSourceURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::Glyphs:
+ res.url = util::mapbox::normalizeGlyphsURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::SpriteImage:
+ case Resource::Kind::SpriteJSON:
+ res.url = util::mapbox::normalizeSpriteURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::Tile:
+ res.url = util::mapbox::normalizeTileURL(apiBaseURL, resource.url, accessToken);
+ break;
+ }
+
+ return std::make_unique<OnlineFileRequest>(std::move(res), std::move(callback), *impl);
+}
+
+void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ impl->setResourceTransform(std::move(transform));
+}
+
+OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_)
+ : impl(impl_),
+ resource(std::move(resource_)),
+ callback(std::move(callback_)) {
+ impl.add(this);
+}
+
+void OnlineFileRequest::schedule() {
+ // Force an immediate first request if we don't have an expiration time.
+ if (resource.priorExpires) {
+ schedule(resource.priorExpires);
+ } else {
+ schedule(util::now());
+ }
+}
+
+OnlineFileRequest::~OnlineFileRequest() {
+ impl.remove(this);
+}
+
+Timestamp interpolateExpiration(const Timestamp& current,
+ optional<Timestamp> prior,
+ bool& expired) {
+ auto now = util::now();
+ if (current > now) {
+ return current;
+ }
+
+ if (!bool(prior)) {
+ expired = true;
+ return current;
+ }
+
+ // Expiring date is going backwards,
+ // fallback to exponential backoff.
+ if (current < *prior) {
+ expired = true;
+ return current;
+ }
+
+ auto delta = current - *prior;
+
+ // Server is serving the same expired resource
+ // over and over, fallback to exponential backoff.
+ if (delta == Duration::zero()) {
+ expired = true;
+ return current;
+ }
+
+ // Assume that either the client or server clock is wrong and
+ // try to interpolate a valid expiration date (from the client POV)
+ // observing a minimum timeout.
+ return now + std::max<Seconds>(delta, util::CLOCK_SKEW_RETRY_TIMEOUT);
+}
+
+void OnlineFileRequest::schedule(optional<Timestamp> expires) {
+ if (impl.isPending(this) || impl.isActive(this)) {
+ // There's already a request in progress; don't start another one.
+ return;
+ }
+
+ // If we're not being asked for a forced refresh, calculate a timeout that depends on how many
+ // consecutive errors we've encountered, and on the expiration time, if present.
+ Duration timeout = std::min(
+ http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter),
+ http::expirationTimeout(expires, expiredRequests));
+
+ if (timeout == Duration::max()) {
+ return;
+ }
+
+ // Emulate a Connection error when the Offline mode is forced with
+ // a really long timeout. The request will get re-triggered when
+ // the NetworkStatus is set back to Online.
+ if (NetworkStatus::Get() == NetworkStatus::Status::Offline) {
+ failedRequestReason = Response::Error::Reason::Connection;
+ failedRequests = 1;
+ timeout = Duration::max();
+ }
+
+ timer.start(timeout, Duration::zero(), [&] {
+ impl.activateOrQueueRequest(this);
+ });
+}
+
+void OnlineFileRequest::completed(Response response) {
+ // If we didn't get various caching headers in the response, continue using the
+ // previous values. Otherwise, update the previous values to the new values.
+
+ if (!response.modified) {
+ response.modified = resource.priorModified;
+ } else {
+ resource.priorModified = response.modified;
+ }
+
+ if (response.notModified && resource.priorData) {
+ // When the priorData field is set, it indicates that we had to revalidate the request and
+ // that the requestor hasn't gotten data yet. If we get a 304 response, this means that we
+ // have send the cached data to give the requestor a chance to actually obtain the data.
+ response.data = std::move(resource.priorData);
+ response.notModified = false;
+ }
+
+ bool isExpired = false;
+
+ if (response.expires) {
+ auto prior = resource.priorExpires;
+ resource.priorExpires = response.expires;
+ response.expires = interpolateExpiration(*response.expires, prior, isExpired);
+ }
+
+ if (isExpired) {
+ expiredRequests++;
+ } else {
+ expiredRequests = 0;
+ }
+
+ if (!response.etag) {
+ response.etag = resource.priorEtag;
+ } else {
+ resource.priorEtag = response.etag;
+ }
+
+ if (response.error) {
+ failedRequests++;
+ failedRequestReason = response.error->reason;
+ retryAfter = response.error->retryAfter;
+ } else {
+ failedRequests = 0;
+ failedRequestReason = Response::Error::Reason::Success;
+ }
+
+ schedule(response.expires);
+
+ // Calling the callback may result in `this` being deleted. It needs to be done last,
+ // and needs to make a local copy of the callback to ensure that it remains valid for
+ // the duration of the call.
+ auto callback_ = callback;
+ callback_(response);
+}
+
+void OnlineFileRequest::networkIsReachableAgain() {
+ // We need all requests to fail at least once before we are going to start retrying
+ // them, and we only immediately restart request that failed due to connection issues.
+ if (failedRequestReason == Response::Error::Reason::Connection) {
+ schedule(util::now());
+ }
+}
+
+void OnlineFileRequest::setTransformedURL(const std::string&& url) {
+ resource.url = std::move(url);
+ schedule();
+}
+
+ActorRef<OnlineFileRequest> OnlineFileRequest::actor() {
+ if (!mailbox) {
+ // Lazy constructed because this can be costly and
+ // the ResourceTransform is not used by many apps.
+ mailbox = std::make_shared<Mailbox>(*Scheduler::GetCurrent());
+ }
+
+ return ActorRef<OnlineFileRequest>(*this, mailbox);
+}
+
+void OnlineFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) {
+ impl->setMaximumConcurrentRequests(maximumConcurrentRequests_);
+}
+
+uint32_t OnlineFileSource::getMaximumConcurrentRequests() const {
+ return impl->getMaximumConcurrentRequests();
+}
+
+
+// For testing only:
+
+void OnlineFileSource::setOnlineStatus(const bool status) {
+ impl->setOnlineStatus(status);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/sqlite3.cpp b/platform/default/src/mbgl/storage/sqlite3.cpp
new file mode 100644
index 0000000000..0017dc45db
--- /dev/null
+++ b/platform/default/src/mbgl/storage/sqlite3.cpp
@@ -0,0 +1,494 @@
+#include <mbgl/storage/sqlite3.hpp>
+#include <sqlite3.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <cstdio>
+#include <chrono>
+#include <experimental/optional>
+
+#include <mbgl/util/traits.hpp>
+#include <mbgl/util/logging.hpp>
+
+namespace mapbox {
+namespace sqlite {
+
+static_assert(mbgl::underlying_type(ResultCode::OK) == SQLITE_OK, "error");
+static_assert(mbgl::underlying_type(ResultCode::Error) == SQLITE_ERROR, "error");
+static_assert(mbgl::underlying_type(ResultCode::Internal) == SQLITE_INTERNAL, "error");
+static_assert(mbgl::underlying_type(ResultCode::Perm) == SQLITE_PERM, "error");
+static_assert(mbgl::underlying_type(ResultCode::Abort) == SQLITE_ABORT, "error");
+static_assert(mbgl::underlying_type(ResultCode::Busy) == SQLITE_BUSY, "error");
+static_assert(mbgl::underlying_type(ResultCode::Locked) == SQLITE_LOCKED, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoMem) == SQLITE_NOMEM, "error");
+static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == SQLITE_READONLY, "error");
+static_assert(mbgl::underlying_type(ResultCode::Interrupt) == SQLITE_INTERRUPT, "error");
+static_assert(mbgl::underlying_type(ResultCode::IOErr) == SQLITE_IOERR, "error");
+static_assert(mbgl::underlying_type(ResultCode::Corrupt) == SQLITE_CORRUPT, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotFound) == SQLITE_NOTFOUND, "error");
+static_assert(mbgl::underlying_type(ResultCode::Full) == SQLITE_FULL, "error");
+static_assert(mbgl::underlying_type(ResultCode::CantOpen) == SQLITE_CANTOPEN, "error");
+static_assert(mbgl::underlying_type(ResultCode::Protocol) == SQLITE_PROTOCOL, "error");
+static_assert(mbgl::underlying_type(ResultCode::Schema) == SQLITE_SCHEMA, "error");
+static_assert(mbgl::underlying_type(ResultCode::TooBig) == SQLITE_TOOBIG, "error");
+static_assert(mbgl::underlying_type(ResultCode::Constraint) == SQLITE_CONSTRAINT, "error");
+static_assert(mbgl::underlying_type(ResultCode::Mismatch) == SQLITE_MISMATCH, "error");
+static_assert(mbgl::underlying_type(ResultCode::Misuse) == SQLITE_MISUSE, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoLFS) == SQLITE_NOLFS, "error");
+static_assert(mbgl::underlying_type(ResultCode::Auth) == SQLITE_AUTH, "error");
+static_assert(mbgl::underlying_type(ResultCode::Range) == SQLITE_RANGE, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotADB) == SQLITE_NOTADB, "error");
+
+void setTempPath(const std::string& path) {
+ sqlite3_temp_directory = sqlite3_mprintf("%s", path.c_str());
+}
+
+class DatabaseImpl {
+public:
+ DatabaseImpl(sqlite3* db_)
+ : db(db_)
+ {
+ const int error = sqlite3_extended_result_codes(db, true);
+ if (error != SQLITE_OK) {
+ mbgl::Log::Warning(mbgl::Event::Database, error, "Failed to enable extended result codes: %s", sqlite3_errmsg(db));
+ }
+ }
+
+ ~DatabaseImpl()
+ {
+ const int error = sqlite3_close(db);
+ if (error != SQLITE_OK) {
+ mbgl::Log::Error(mbgl::Event::Database, error, "Failed to close database: %s", sqlite3_errmsg(db));
+ }
+ }
+
+ void setBusyTimeout(std::chrono::milliseconds timeout);
+ void exec(const std::string& sql);
+
+ sqlite3* db;
+};
+
+class StatementImpl {
+public:
+ StatementImpl(sqlite3* db, const char* sql)
+ {
+ const int error = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
+ if (error != SQLITE_OK) {
+ stmt = nullptr;
+ throw Exception { error, sqlite3_errmsg(db) };
+ }
+ }
+
+ ~StatementImpl()
+ {
+ if (!stmt) return;
+
+ sqlite3_finalize(stmt);
+ }
+
+ void check(int err) {
+ if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt)) };
+ }
+ }
+
+ sqlite3_stmt* stmt = nullptr;
+ int64_t lastInsertRowId = 0;
+ int64_t changes = 0;
+};
+
+template <typename T>
+using optional = std::experimental::optional<T>;
+
+
+#ifndef NDEBUG
+void logSqlMessage(void *, const int err, const char *msg) {
+ mbgl::Log::Record(mbgl::EventSeverity::Debug, mbgl::Event::Database, err, "%s", msg);
+}
+#endif
+
+__attribute__((constructor))
+static void initalize() {
+ if (sqlite3_libversion_number() / 1000000 != SQLITE_VERSION_NUMBER / 1000000) {
+ char message[96];
+ snprintf(message, 96,
+ "sqlite3 libversion mismatch: headers report %d, but library reports %d",
+ SQLITE_VERSION_NUMBER, sqlite3_libversion_number());
+ throw std::runtime_error(message);
+ }
+
+#ifndef NDEBUG
+ // Enable SQLite logging before initializing the database.
+ sqlite3_config(SQLITE_CONFIG_LOG, &logSqlMessage, nullptr);
+#endif
+}
+
+mapbox::util::variant<Database, Exception> Database::tryOpen(const std::string &filename, int flags) {
+ sqlite3* db = nullptr;
+ const int error = sqlite3_open_v2(filename.c_str(), &db, flags | SQLITE_OPEN_URI, nullptr);
+ if (error != SQLITE_OK) {
+ const auto message = sqlite3_errmsg(db);
+ return Exception { error, message };
+ }
+ return Database(std::make_unique<DatabaseImpl>(db));
+}
+
+Database Database::open(const std::string &filename, int flags) {
+ auto result = tryOpen(filename, flags);
+ if (result.is<Exception>()) {
+ throw result.get<Exception>();
+ } else {
+ return std::move(result.get<Database>());
+ }
+}
+
+Database::Database(std::unique_ptr<DatabaseImpl> impl_)
+ : impl(std::move(impl_))
+{}
+
+Database::Database(Database &&other)
+ : impl(std::move(other.impl)) {}
+
+Database &Database::operator=(Database &&other) {
+ std::swap(impl, other.impl);
+ return *this;
+}
+
+Database::~Database() = default;
+
+void Database::setBusyTimeout(std::chrono::milliseconds timeout) {
+ assert(impl);
+ impl->setBusyTimeout(timeout);
+}
+
+void DatabaseImpl::setBusyTimeout(std::chrono::milliseconds timeout) {
+ const int err = sqlite3_busy_timeout(db,
+ int(std::min<std::chrono::milliseconds::rep>(timeout.count(), std::numeric_limits<int>::max())));
+ if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(db) };
+ }
+}
+
+void Database::exec(const std::string &sql) {
+ assert(impl);
+ impl->exec(sql);
+}
+
+void DatabaseImpl::exec(const std::string& sql) {
+ char *msg = nullptr;
+ const int err = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &msg);
+ if (msg) {
+ const std::string message = msg;
+ sqlite3_free(msg);
+ throw Exception { err, message };
+ } else if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(db) };
+ }
+}
+
+Statement::Statement(Database& db, const char* sql)
+ : impl(std::make_unique<StatementImpl>(db.impl->db, sql)) {
+}
+
+Statement::~Statement() {
+#ifndef NDEBUG
+ // Crash if we're destructing this object while we know a Query object references this.
+ assert(!used);
+#endif
+}
+
+Query::Query(Statement& stmt_) : stmt(stmt_) {
+ assert(stmt.impl);
+
+#ifndef NDEBUG
+ assert(!stmt.used);
+ stmt.used = true;
+#endif
+}
+
+Query::~Query() {
+ reset();
+ clearBindings();
+
+#ifndef NDEBUG
+ stmt.used = false;
+#endif
+}
+
+template <> void Query::bind(int offset, std::nullptr_t) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_null(stmt.impl->stmt, offset));
+}
+
+template <> void Query::bind(int offset, int8_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, int16_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, int32_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, int64_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, uint8_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, uint16_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, uint32_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, float value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, double value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, bool value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, const char *value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, -1, SQLITE_STATIC));
+}
+
+// We currently cannot use sqlite3_bind_blob64 / sqlite3_bind_text64 because they
+// were introduced in SQLite 3.8.7, and we need to support earlier versions:
+// Android 11: 3.7
+// Android 21: 3.8
+// Android 24: 3.9
+// Per https://developer.android.com/reference/android/database/sqlite/package-summary.
+// The first iOS version with 3.8.7+ was 9.0, with 3.8.8.
+
+void Query::bind(int offset, const char * value, std::size_t length, bool retain) {
+ assert(stmt.impl);
+ if (length > std::numeric_limits<int>::max()) {
+ throw std::range_error("value too long for sqlite3_bind_text");
+ }
+ stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, int(length),
+ retain ? SQLITE_TRANSIENT : SQLITE_STATIC));
+}
+
+void Query::bind(int offset, const std::string& value, bool retain) {
+ bind(offset, value.data(), value.size(), retain);
+}
+
+void Query::bindBlob(int offset, const void * value, std::size_t length, bool retain) {
+ assert(stmt.impl);
+ if (length > std::numeric_limits<int>::max()) {
+ throw std::range_error("value too long for sqlite3_bind_text");
+ }
+ stmt.impl->check(sqlite3_bind_blob(stmt.impl->stmt, offset, value, int(length),
+ retain ? SQLITE_TRANSIENT : SQLITE_STATIC));
+}
+
+void Query::bindBlob(int offset, const std::vector<uint8_t>& value, bool retain) {
+ bindBlob(offset, value.data(), value.size(), retain);
+}
+
+template <>
+void Query::bind(
+ int offset, std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds> value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, std::chrono::system_clock::to_time_t(value)));
+}
+
+template <> void Query::bind(int offset, optional<std::string> value) {
+ if (!value) {
+ bind(offset, nullptr);
+ } else {
+ bind(offset, *value);
+ }
+}
+
+template <>
+void Query::bind(
+ int offset,
+ optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>> value) {
+ if (!value) {
+ bind(offset, nullptr);
+ } else {
+ bind(offset, *value);
+ }
+}
+
+bool Query::run() {
+ assert(stmt.impl);
+ const int err = sqlite3_step(stmt.impl->stmt);
+ stmt.impl->lastInsertRowId = sqlite3_last_insert_rowid(sqlite3_db_handle(stmt.impl->stmt));
+ stmt.impl->changes = sqlite3_changes(sqlite3_db_handle(stmt.impl->stmt));
+ if (err == SQLITE_DONE) {
+ return false;
+ } else if (err == SQLITE_ROW) {
+ return true;
+ } else if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt.impl->stmt)) };
+ } else {
+ return false;
+ }
+}
+
+template <> bool Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_int(stmt.impl->stmt, offset);
+}
+
+template <> int Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_int(stmt.impl->stmt, offset);
+}
+
+template <> int64_t Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_int64(stmt.impl->stmt, offset);
+}
+
+template <> double Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_double(stmt.impl->stmt, offset);
+}
+
+template <> std::string Query::get(int offset) {
+ assert(stmt.impl);
+ return {
+ reinterpret_cast<const char *>(sqlite3_column_blob(stmt.impl->stmt, offset)),
+ size_t(sqlite3_column_bytes(stmt.impl->stmt, offset))
+ };
+}
+
+template <> std::vector<uint8_t> Query::get(int offset) {
+ assert(stmt.impl);
+ const auto* begin = reinterpret_cast<const uint8_t*>(sqlite3_column_blob(stmt.impl->stmt, offset));
+ const uint8_t* end = begin + sqlite3_column_bytes(stmt.impl->stmt, offset);
+ return { begin, end };
+}
+
+template <>
+std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>
+Query::get(int offset) {
+ assert(stmt.impl);
+ return std::chrono::time_point_cast<std::chrono::seconds>(
+ std::chrono::system_clock::from_time_t(sqlite3_column_int64(stmt.impl->stmt, offset)));
+}
+
+template <> optional<int64_t> Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return optional<int64_t>();
+ } else {
+ return get<int64_t>(offset);
+ }
+}
+
+template <> optional<double> Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return optional<double>();
+ } else {
+ return get<double>(offset);
+ }
+}
+
+template <> optional<std::string> Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return optional<std::string>();
+ } else {
+ return get<std::string>(offset);
+ }
+}
+
+template <>
+optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>>
+Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return {};
+ } else {
+ return get<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>>(
+ offset);
+ }
+}
+
+void Query::reset() {
+ assert(stmt.impl);
+ sqlite3_reset(stmt.impl->stmt);
+}
+
+void Query::clearBindings() {
+ assert(stmt.impl);
+ sqlite3_clear_bindings(stmt.impl->stmt);
+}
+
+int64_t Query::lastInsertRowId() const {
+ assert(stmt.impl);
+ return stmt.impl->lastInsertRowId;
+}
+
+uint64_t Query::changes() const {
+ assert(stmt.impl);
+ auto changes_ = stmt.impl->changes;
+ return (changes_ < 0 ? 0 : changes_);
+}
+
+Transaction::Transaction(Database& db_, Mode mode)
+ : dbImpl(*db_.impl) {
+ switch (mode) {
+ case Deferred:
+ dbImpl.exec("BEGIN DEFERRED TRANSACTION");
+ break;
+ case Immediate:
+ dbImpl.exec("BEGIN IMMEDIATE TRANSACTION");
+ break;
+ case Exclusive:
+ dbImpl.exec("BEGIN EXCLUSIVE TRANSACTION");
+ break;
+ }
+}
+
+Transaction::~Transaction() {
+ if (needRollback) {
+ try {
+ rollback();
+ } catch (...) {
+ // Ignore failed rollbacks in destructor.
+ }
+ }
+}
+
+void Transaction::commit() {
+ needRollback = false;
+ dbImpl.exec("COMMIT TRANSACTION");
+}
+
+void Transaction::rollback() {
+ needRollback = false;
+ dbImpl.exec("ROLLBACK TRANSACTION");
+}
+
+} // namespace sqlite
+} // namespace mapbox
diff --git a/platform/default/src/mbgl/test/main.cpp b/platform/default/src/mbgl/test/main.cpp
new file mode 100644
index 0000000000..d01cf75ffc
--- /dev/null
+++ b/platform/default/src/mbgl/test/main.cpp
@@ -0,0 +1,20 @@
+#include <mbgl/test.hpp>
+#include <unistd.h>
+#include <cstring>
+#include <cerrno>
+#include <cstdio>
+
+#define xstr(s) str(s)
+#define str(s) #s
+
+int main(int argc, char *argv[]) {
+#ifdef WORK_DIRECTORY
+ const int result = chdir(xstr(WORK_DIRECTORY));
+ if (result != 0) {
+ fprintf(stderr, "failed to change directory: %s\n", strerror(errno));
+ return errno;
+ }
+#endif
+
+ return mbgl::runTests(argc, argv);
+}
diff --git a/platform/default/src/mbgl/text/bidi.cpp b/platform/default/src/mbgl/text/bidi.cpp
new file mode 100644
index 0000000000..32a3dc23ef
--- /dev/null
+++ b/platform/default/src/mbgl/text/bidi.cpp
@@ -0,0 +1,239 @@
+#include <mbgl/text/bidi.hpp>
+#include <mbgl/util/traits.hpp>
+
+#include <unicode/ubidi.h>
+#include <unicode/ushape.h>
+
+#include <memory>
+
+namespace mbgl {
+
+class BiDiImpl {
+public:
+ BiDiImpl() : bidiText(ubidi_open()), bidiLine(ubidi_open()) {
+ }
+ ~BiDiImpl() {
+ ubidi_close(bidiText);
+ ubidi_close(bidiLine);
+ }
+
+ UBiDi* bidiText = nullptr;
+ UBiDi* bidiLine = nullptr;
+};
+
+BiDi::BiDi() : impl(std::make_unique<BiDiImpl>()) {}
+BiDi::~BiDi() = default;
+
+// Takes UTF16 input in logical order and applies Arabic shaping to the input while maintaining
+// logical order. Output won't be intelligible until the bidirectional algorithm is applied
+std::u16string applyArabicShaping(const std::u16string& input) {
+ UErrorCode errorCode = U_ZERO_ERROR;
+
+ const int32_t outputLength =
+ u_shapeArabic(mbgl::utf16char_cast<const UChar*>(input.c_str()), static_cast<int32_t>(input.size()), nullptr, 0,
+ (U_SHAPE_LETTERS_SHAPE & U_SHAPE_LETTERS_MASK) |
+ (U_SHAPE_TEXT_DIRECTION_LOGICAL & U_SHAPE_TEXT_DIRECTION_MASK),
+ &errorCode);
+
+ // Pre-flighting will always set U_BUFFER_OVERFLOW_ERROR
+ errorCode = U_ZERO_ERROR;
+
+ std::u16string outputText(outputLength, 0);
+
+ u_shapeArabic(mbgl::utf16char_cast<const UChar*>(input.c_str()), static_cast<int32_t>(input.size()), mbgl::utf16char_cast<UChar*>(&outputText[0]), outputLength,
+ (U_SHAPE_LETTERS_SHAPE & U_SHAPE_LETTERS_MASK) |
+ (U_SHAPE_TEXT_DIRECTION_LOGICAL & U_SHAPE_TEXT_DIRECTION_MASK),
+ &errorCode);
+
+ // If the algorithm fails for any reason, fall back to non-transformed text
+ if (U_FAILURE(errorCode))
+ return input;
+
+ return outputText;
+}
+
+void BiDi::mergeParagraphLineBreaks(std::set<size_t>& lineBreakPoints) {
+ int32_t paragraphCount = ubidi_countParagraphs(impl->bidiText);
+ for (int32_t i = 0; i < paragraphCount; i++) {
+ UErrorCode errorCode = U_ZERO_ERROR;
+ int32_t paragraphEndIndex;
+ ubidi_getParagraphByIndex(impl->bidiText, i, nullptr, &paragraphEndIndex, nullptr, &errorCode);
+
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("ProcessedBiDiText::mergeParagraphLineBreaks: ") +
+ u_errorName(errorCode));
+ }
+
+ lineBreakPoints.insert(static_cast<std::size_t>(paragraphEndIndex));
+ }
+}
+
+std::vector<std::u16string> BiDi::applyLineBreaking(std::set<std::size_t> lineBreakPoints) {
+ // BiDi::getLine will error if called across a paragraph boundary, so we need to ensure that all
+ // paragraph boundaries are included in the set of line break points. The calling code might not
+ // include the line break because it didn't need to wrap at that point, or because the text was
+ // separated with a more exotic code point such as (U+001C)
+ mergeParagraphLineBreaks(lineBreakPoints);
+
+ std::vector<std::u16string> transformedLines;
+ transformedLines.reserve(lineBreakPoints.size());
+
+ std::size_t start = 0;
+ for (std::size_t lineBreakPoint : lineBreakPoints) {
+ transformedLines.push_back(getLine(start, lineBreakPoint));
+ start = lineBreakPoint;
+ }
+
+ return transformedLines;
+}
+
+std::vector<std::u16string> BiDi::processText(const std::u16string& input,
+ std::set<std::size_t> lineBreakPoints) {
+ UErrorCode errorCode = U_ZERO_ERROR;
+
+ ubidi_setPara(impl->bidiText, mbgl::utf16char_cast<const UChar*>(input.c_str()), static_cast<int32_t>(input.size()),
+ UBIDI_DEFAULT_LTR, nullptr, &errorCode);
+
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::processText: ") + u_errorName(errorCode));
+ }
+
+ return applyLineBreaking(lineBreakPoints);
+}
+
+std::vector<StyledText> BiDi::processStyledText(const StyledText& input, std::set<std::size_t> lineBreakPoints) {
+ std::vector<StyledText> lines;
+ const auto& inputText = input.first;
+ const auto& styleIndices = input.second;
+
+ UErrorCode errorCode = U_ZERO_ERROR;
+
+ ubidi_setPara(impl->bidiText, mbgl::utf16char_cast<const UChar*>(inputText.c_str()), static_cast<int32_t>(inputText.size()),
+ UBIDI_DEFAULT_LTR, nullptr, &errorCode);
+
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::processStyledText: ") + u_errorName(errorCode));
+ }
+
+ mergeParagraphLineBreaks(lineBreakPoints);
+
+ std::size_t lineStartIndex = 0;
+
+ for (std::size_t lineBreakPoint : lineBreakPoints) {
+ StyledText line;
+ line.second.reserve(lineBreakPoint - lineStartIndex);
+
+ errorCode = U_ZERO_ERROR;
+ ubidi_setLine(impl->bidiText, static_cast<int32_t>(lineStartIndex), static_cast<int32_t>(lineBreakPoint), impl->bidiLine, &errorCode);
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::processStyledText (setLine): ") + u_errorName(errorCode));
+ }
+
+ errorCode = U_ZERO_ERROR;
+ uint32_t runCount = ubidi_countRuns(impl->bidiLine, &errorCode);
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::processStyledText (countRuns): ") + u_errorName(errorCode));
+ }
+
+ for (uint32_t runIndex = 0; runIndex < runCount; runIndex++) {
+ int32_t runLogicalStart;
+ int32_t runLength;
+ UBiDiDirection direction = ubidi_getVisualRun(impl->bidiLine, runIndex, &runLogicalStart, &runLength);
+ const bool isReversed = direction == UBIDI_RTL;
+
+ std::size_t logicalStart = lineStartIndex + runLogicalStart;
+ std::size_t logicalEnd = logicalStart + runLength;
+ if (isReversed) {
+ // Within this reversed section, iterate logically backwards
+ // Each time we see a change in style, render a reversed chunk
+ // of everything since the last change
+ std::size_t styleRunStart = logicalEnd;
+ uint8_t currentStyleIndex = styleIndices.at(styleRunStart - 1);
+ for (std::size_t i = logicalEnd - 1; i >= logicalStart; i--) {
+ if (currentStyleIndex != styleIndices.at(i) || i == logicalStart) {
+ std::size_t styleRunEnd = i == logicalStart ? i : i + 1;
+ std::u16string reversed = writeReverse(inputText, styleRunEnd, styleRunStart);
+ line.first += reversed;
+ for (std::size_t j = 0; j < reversed.size(); j++) {
+ line.second.push_back(currentStyleIndex);
+ }
+ currentStyleIndex = styleIndices.at(i);
+ styleRunStart = styleRunEnd;
+ }
+ if (i == 0) {
+ break;
+ }
+ }
+
+ } else {
+ line.first += input.first.substr(logicalStart, runLength);
+ line.second.insert(line.second.end(), styleIndices.begin() + logicalStart, styleIndices.begin() + logicalStart + runLength);
+ }
+ }
+
+ lines.push_back(line);
+ lineStartIndex = lineBreakPoint;
+ }
+
+ return lines;
+}
+
+std::u16string BiDi::writeReverse(const std::u16string& input, std::size_t logicalStart, std::size_t logicalEnd) {
+ UErrorCode errorCode = U_ZERO_ERROR;
+ int32_t logicalLength = static_cast<int32_t>(logicalEnd - logicalStart);
+ std::u16string outputText(logicalLength + 1, 0);
+
+ // UBIDI_DO_MIRRORING: Apply unicode mirroring of characters like parentheses
+ // UBIDI_REMOVE_BIDI_CONTROLS: Now that all the lines are set, remove control characters so that
+ // they don't show up on screen (some fonts have glyphs representing them)
+ int32_t outputLength =
+ ubidi_writeReverse(mbgl::utf16char_cast<const UChar*>(&input[logicalStart]),
+ logicalLength,
+ mbgl::utf16char_cast<UChar*>(&outputText[0]),
+ logicalLength + 1, // Extra room for null terminator, although we don't really need to have ICU write it for us
+ UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS,
+ &errorCode);
+
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::writeReverse: ") + u_errorName(errorCode));
+ }
+
+ outputText.resize(outputLength); // REMOVE_BIDI_CONTROLS may have shrunk the string
+
+ return outputText;
+}
+
+std::u16string BiDi::getLine(std::size_t start, std::size_t end) {
+ UErrorCode errorCode = U_ZERO_ERROR;
+ ubidi_setLine(impl->bidiText, static_cast<int32_t>(start), static_cast<int32_t>(end), impl->bidiLine, &errorCode);
+
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::getLine (setLine): ") + u_errorName(errorCode));
+ }
+
+ // Because we set UBIDI_REMOVE_BIDI_CONTROLS, the output may be smaller than what we reserve
+ // Setting UBIDI_INSERT_LRM_FOR_NUMERIC would require
+ // ubidi_getLength(pBiDi)+2*ubidi_countRuns(pBiDi)
+ const int32_t outputLength = ubidi_getProcessedLength(impl->bidiLine);
+ std::u16string outputText(outputLength, 0);
+
+ // UBIDI_DO_MIRRORING: Apply unicode mirroring of characters like parentheses
+ // UBIDI_REMOVE_BIDI_CONTROLS: Now that all the lines are set, remove control characters so that
+ // they don't show up on screen (some fonts have glyphs representing them)
+ int32_t finalLength = ubidi_writeReordered(impl->bidiLine,
+ mbgl::utf16char_cast<UChar*>(&outputText[0]),
+ outputLength,
+ UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS,
+ &errorCode);
+
+ outputText.resize(finalLength); // REMOVE_BIDI_CONTROLS may have shrunk the string
+
+ if (U_FAILURE(errorCode)) {
+ throw std::runtime_error(std::string("BiDi::getLine (writeReordered): ") +
+ u_errorName(errorCode));
+ }
+
+ return outputText;
+}
+
+} // end namespace mbgl
diff --git a/platform/default/src/mbgl/text/collator.cpp b/platform/default/src/mbgl/text/collator.cpp
new file mode 100644
index 0000000000..400fa4d94d
--- /dev/null
+++ b/platform/default/src/mbgl/text/collator.cpp
@@ -0,0 +1,79 @@
+#include <mbgl/style/expression/collator.hpp>
+#include <mbgl/util/platform.hpp>
+#include <libnu/strcoll.h>
+#include <mbgl/text/unaccent.hpp>
+
+/*
+ The default implementation of Collator ignores locale.
+ Case sensitivity and collation order are based on
+ Default Unicode Collation Element Table (DUCET).
+
+ Diacritic-insensitivity is implemented with nunicode's
+ non-standard "unaccent" functionality, which is tailored
+ to European languages.
+
+ It would be possible to implement locale awareness using ICU,
+ but would require bundling locale data.
+*/
+
+namespace mbgl {
+namespace style {
+namespace expression {
+
+class Collator::Impl {
+public:
+ Impl(bool caseSensitive_, bool diacriticSensitive_, optional<std::string>)
+ : caseSensitive(caseSensitive_)
+ , diacriticSensitive(diacriticSensitive_)
+ {}
+
+ bool operator==(const Impl& other) const {
+ return caseSensitive == other.caseSensitive &&
+ diacriticSensitive == other.diacriticSensitive;
+ }
+
+ int compare(const std::string& lhs, const std::string& rhs) const {
+ if (caseSensitive && diacriticSensitive) {
+ return nu_strcoll(lhs.c_str(), rhs.c_str(),
+ nu_utf8_read, nu_utf8_read);
+ } else if (!caseSensitive && diacriticSensitive) {
+ return nu_strcasecoll(lhs.c_str(), rhs.c_str(),
+ nu_utf8_read, nu_utf8_read);
+ } else if (caseSensitive && !diacriticSensitive) {
+ return nu_strcoll(platform::unaccent(lhs).c_str(), platform::unaccent(rhs).c_str(),
+ nu_utf8_read, nu_utf8_read);
+ } else {
+ return nu_strcasecoll(platform::unaccent(lhs).c_str(), platform::unaccent(rhs).c_str(),
+ nu_utf8_read, nu_utf8_read);
+ }
+ }
+
+ std::string resolvedLocale() const {
+ return "";
+ }
+private:
+ bool caseSensitive;
+ bool diacriticSensitive;
+};
+
+
+Collator::Collator(bool caseSensitive, bool diacriticSensitive, optional<std::string> locale_)
+ : impl(std::make_shared<Impl>(caseSensitive, diacriticSensitive, std::move(locale_)))
+{}
+
+bool Collator::operator==(const Collator& other) const {
+ return *impl == *(other.impl);
+}
+
+int Collator::compare(const std::string& lhs, const std::string& rhs) const {
+ return impl->compare(lhs, rhs);
+}
+
+std::string Collator::resolvedLocale() const {
+ return impl->resolvedLocale();
+}
+
+
+} // namespace expression
+} // namespace style
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp b/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp
new file mode 100644
index 0000000000..7866f29420
--- /dev/null
+++ b/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp
@@ -0,0 +1,22 @@
+#include <mbgl/text/local_glyph_rasterizer.hpp>
+
+namespace mbgl {
+
+class LocalGlyphRasterizer::Impl {
+};
+
+LocalGlyphRasterizer::LocalGlyphRasterizer(const optional<std::string>)
+{}
+
+LocalGlyphRasterizer::~LocalGlyphRasterizer()
+{}
+
+bool LocalGlyphRasterizer::canRasterizeGlyph(const FontStack&, GlyphID) {
+ return false;
+}
+
+Glyph LocalGlyphRasterizer::rasterizeGlyph(const FontStack&, GlyphID) {
+ return Glyph();
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/text/unaccent.cpp b/platform/default/src/mbgl/text/unaccent.cpp
new file mode 100644
index 0000000000..37b9a0d9ca
--- /dev/null
+++ b/platform/default/src/mbgl/text/unaccent.cpp
@@ -0,0 +1,43 @@
+#include <mbgl/util/platform.hpp>
+#include <libnu/unaccent.h>
+#include <mbgl/text/unaccent.hpp>
+
+#include <cstring>
+#include <sstream>
+
+namespace mbgl { namespace platform {
+
+std::string unaccent(const std::string& str)
+{
+ std::stringstream output;
+ char const *itr = str.c_str(), *nitr;
+ char const *end = itr + str.length();
+ char lo[5] = { 0 };
+
+ for (; itr < end; itr = nitr)
+ {
+ uint32_t code_point = 0;
+ char const* buf = nullptr;
+
+ nitr = _nu_tounaccent(itr, end, nu_utf8_read, &code_point, &buf, nullptr);
+ if (buf != nullptr)
+ {
+ do
+ {
+ buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point);
+ if (code_point == 0) break;
+ output.write(lo, nu_utf8_write(code_point, lo) - lo);
+ }
+ while (code_point != 0);
+ }
+ else
+ {
+ output.write(itr, nitr - itr);
+ }
+ }
+
+ return output.str();
+}
+
+} // namespace platform
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/async_task.cpp b/platform/default/src/mbgl/util/async_task.cpp
new file mode 100644
index 0000000000..50891056d8
--- /dev/null
+++ b/platform/default/src/mbgl/util/async_task.cpp
@@ -0,0 +1,66 @@
+#include <mbgl/util/async_task.hpp>
+
+#include <mbgl/util/run_loop.hpp>
+
+#include <atomic>
+#include <functional>
+
+#include <uv.h>
+
+namespace mbgl {
+namespace util {
+
+class AsyncTask::Impl {
+public:
+ Impl(std::function<void()>&& fn)
+ : async(new uv_async_t),
+ task(std::move(fn)) {
+
+ auto* loop = reinterpret_cast<uv_loop_t*>(RunLoop::getLoopHandle());
+ if (uv_async_init(loop, async, asyncCallback) != 0) {
+ throw std::runtime_error("Failed to initialize async.");
+ }
+
+ handle()->data = this;
+ uv_unref(handle());
+ }
+
+ ~Impl() {
+ uv_close(handle(), [](uv_handle_t* h) {
+ delete reinterpret_cast<uv_async_t*>(h);
+ });
+ }
+
+ void maySend() {
+ // uv_async_send will do the call coalescing for us.
+ if (uv_async_send(async) != 0) {
+ throw std::runtime_error("Failed to async send.");
+ }
+ }
+
+private:
+ static void asyncCallback(uv_async_t* handle) {
+ reinterpret_cast<Impl*>(handle->data)->task();
+ }
+
+ uv_handle_t* handle() {
+ return reinterpret_cast<uv_handle_t*>(async);
+ }
+
+ uv_async_t* async;
+
+ std::function<void()> task;
+};
+
+AsyncTask::AsyncTask(std::function<void()>&& fn)
+ : impl(std::make_unique<Impl>(std::move(fn))) {
+}
+
+AsyncTask::~AsyncTask() = default;
+
+void AsyncTask::send() {
+ impl->maySend();
+}
+
+} // namespace util
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/default_thread_pool.cpp b/platform/default/src/mbgl/util/default_thread_pool.cpp
new file mode 100644
index 0000000000..d3950bb8aa
--- /dev/null
+++ b/platform/default/src/mbgl/util/default_thread_pool.cpp
@@ -0,0 +1,57 @@
+#include <mbgl/util/default_thread_pool.hpp>
+#include <mbgl/actor/mailbox.hpp>
+#include <mbgl/util/platform.hpp>
+#include <mbgl/util/string.hpp>
+
+namespace mbgl {
+
+ThreadPool::ThreadPool(std::size_t count) {
+ threads.reserve(count);
+ for (std::size_t i = 0; i < count; ++i) {
+ threads.emplace_back([this, i]() {
+ platform::setCurrentThreadName(std::string{ "Worker " } + util::toString(i + 1));
+
+ while (true) {
+ std::unique_lock<std::mutex> lock(mutex);
+
+ cv.wait(lock, [this] {
+ return !queue.empty() || terminate;
+ });
+
+ if (terminate) {
+ return;
+ }
+
+ auto mailbox = queue.front();
+ queue.pop();
+ lock.unlock();
+
+ Mailbox::maybeReceive(mailbox);
+ }
+ });
+ }
+}
+
+ThreadPool::~ThreadPool() {
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ terminate = true;
+ }
+
+ cv.notify_all();
+
+ for (auto& thread : threads) {
+ thread.join();
+ }
+}
+
+void ThreadPool::schedule(std::weak_ptr<Mailbox> mailbox) {
+ {
+ std::lock_guard<std::mutex> lock(mutex);
+ queue.push(mailbox);
+ }
+
+ cv.notify_one();
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/image.cpp b/platform/default/src/mbgl/util/image.cpp
new file mode 100644
index 0000000000..25063892b7
--- /dev/null
+++ b/platform/default/src/mbgl/util/image.cpp
@@ -0,0 +1,31 @@
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/premultiply.hpp>
+
+namespace mbgl {
+
+PremultipliedImage decodePNG(const uint8_t*, size_t);
+PremultipliedImage decodeJPEG(const uint8_t*, size_t);
+
+PremultipliedImage decodeImage(const std::string& string) {
+ const auto* data = reinterpret_cast<const uint8_t*>(string.data());
+ const size_t size = string.size();
+
+ if (size >= 4) {
+ uint32_t magic = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
+ if (magic == 0x89504E47U) {
+ return decodePNG(data, size);
+ }
+ }
+
+ if (size >= 2) {
+ uint16_t magic = ((data[0] << 8) | data[1]) & 0xffff;
+ if (magic == 0xFFD8) {
+ return decodeJPEG(data, size);
+ }
+ }
+
+ throw std::runtime_error("unsupported image type");
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/jpeg_reader.cpp b/platform/default/src/mbgl/util/jpeg_reader.cpp
new file mode 100644
index 0000000000..5f613f9423
--- /dev/null
+++ b/platform/default/src/mbgl/util/jpeg_reader.cpp
@@ -0,0 +1,151 @@
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/char_array_buffer.hpp>
+
+#include <istream>
+#include <sstream>
+#include <array>
+
+extern "C"
+{
+#include <jpeglib.h>
+}
+
+namespace mbgl {
+
+const static unsigned BUF_SIZE = 4096;
+
+struct jpeg_stream_wrapper {
+ jpeg_source_mgr manager;
+ std::istream* stream;
+ std::array<JOCTET, BUF_SIZE> buffer;
+};
+
+static void init_source(j_decompress_ptr cinfo) {
+ auto* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
+ wrap->stream->seekg(0, std::ios_base::beg);
+}
+
+static boolean fill_input_buffer(j_decompress_ptr cinfo) {
+ auto* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
+ wrap->stream->read(reinterpret_cast<char*>(&wrap->buffer[0]), BUF_SIZE);
+ std::streamsize size = wrap->stream->gcount();
+ wrap->manager.next_input_byte = wrap->buffer.data();
+ wrap->manager.bytes_in_buffer = BUF_SIZE;
+ return (size > 0) ? TRUE : FALSE;
+}
+
+static void skip(j_decompress_ptr cinfo, long count) {
+ if (count <= 0) return; // A zero or negative skip count should be treated as a no-op.
+ auto* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src);
+
+ if (wrap->manager.bytes_in_buffer > 0 && count < static_cast<long>(wrap->manager.bytes_in_buffer))
+ {
+ wrap->manager.bytes_in_buffer -= count;
+ wrap->manager.next_input_byte = &wrap->buffer[BUF_SIZE - wrap->manager.bytes_in_buffer];
+ }
+ else
+ {
+ wrap->stream->seekg(count - wrap->manager.bytes_in_buffer, std::ios_base::cur);
+ // trigger buffer fill
+ wrap->manager.next_input_byte = nullptr;
+ wrap->manager.bytes_in_buffer = 0; // bytes_in_buffer may be zero on return.
+ }
+}
+
+static void term(j_decompress_ptr) {}
+
+static void attach_stream(j_decompress_ptr cinfo, std::istream* in) {
+ if (cinfo->src == nullptr) {
+ cinfo->src = (struct jpeg_source_mgr *)
+ (*cinfo->mem->alloc_small) ((j_common_ptr) cinfo, JPOOL_PERMANENT, sizeof(jpeg_stream_wrapper));
+ }
+ auto * src = reinterpret_cast<jpeg_stream_wrapper*> (cinfo->src);
+ src->manager.init_source = init_source;
+ src->manager.fill_input_buffer = fill_input_buffer;
+ src->manager.skip_input_data = skip;
+ src->manager.resync_to_restart = jpeg_resync_to_restart;
+ src->manager.term_source = term;
+ src->manager.bytes_in_buffer = 0;
+ src->manager.next_input_byte = nullptr;
+ src->stream = in;
+}
+
+static void on_error(j_common_ptr) {}
+
+static void on_error_message(j_common_ptr cinfo) {
+ char buffer[JMSG_LENGTH_MAX];
+ (*cinfo->err->format_message)(cinfo, buffer);
+ throw std::runtime_error(std::string("JPEG Reader: libjpeg could not read image: ") + buffer);
+}
+
+struct jpeg_info_guard {
+ jpeg_info_guard(jpeg_decompress_struct* cinfo)
+ : i_(cinfo) {}
+
+ ~jpeg_info_guard() {
+ jpeg_destroy_decompress(i_);
+ }
+
+ jpeg_decompress_struct* i_;
+};
+
+PremultipliedImage decodeJPEG(const uint8_t* data, size_t size) {
+ util::CharArrayBuffer dataBuffer { reinterpret_cast<const char*>(data), size };
+ std::istream stream(&dataBuffer);
+
+ jpeg_decompress_struct cinfo;
+ jpeg_info_guard iguard(&cinfo);
+ jpeg_error_mgr jerr;
+ cinfo.err = jpeg_std_error(&jerr);
+ jerr.error_exit = on_error;
+ jerr.output_message = on_error_message;
+ jpeg_create_decompress(&cinfo);
+ attach_stream(&cinfo, &stream);
+
+ int ret = jpeg_read_header(&cinfo, TRUE);
+ if (ret != JPEG_HEADER_OK)
+ throw std::runtime_error("JPEG Reader: failed to read header");
+
+ jpeg_start_decompress(&cinfo);
+
+ if (cinfo.out_color_space == JCS_UNKNOWN)
+ throw std::runtime_error("JPEG Reader: failed to read unknown color space");
+
+ if (cinfo.output_width == 0 || cinfo.output_height == 0)
+ throw std::runtime_error("JPEG Reader: failed to read image size");
+
+ size_t width = cinfo.output_width;
+ size_t height = cinfo.output_height;
+ size_t components = cinfo.output_components;
+ size_t rowStride = components * width;
+
+ PremultipliedImage image({ static_cast<uint32_t>(width), static_cast<uint32_t>(height) });
+ uint8_t* dst = image.data.get();
+
+ JSAMPARRAY buffer = (*cinfo.mem->alloc_sarray)((j_common_ptr) &cinfo, JPOOL_IMAGE, rowStride, 1);
+
+ while (cinfo.output_scanline < cinfo.output_height) {
+ jpeg_read_scanlines(&cinfo, buffer, 1);
+
+ for (size_t i = 0; i < width; ++i) {
+ dst[0] = buffer[0][components * i];
+ dst[3] = 0xFF;
+
+ if (components > 2) {
+ dst[1] = buffer[0][components * i + 1];
+ dst[2] = buffer[0][components * i + 2];
+ } else {
+ dst[1] = dst[0];
+ dst[2] = dst[0];
+ }
+
+ dst += 4;
+ }
+ }
+
+ jpeg_finish_decompress(&cinfo);
+
+ return image;
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/logging_stderr.cpp b/platform/default/src/mbgl/util/logging_stderr.cpp
new file mode 100644
index 0000000000..41585fb7bb
--- /dev/null
+++ b/platform/default/src/mbgl/util/logging_stderr.cpp
@@ -0,0 +1,12 @@
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/enum.hpp>
+
+#include <iostream>
+
+namespace mbgl {
+
+void Log::platformRecord(EventSeverity severity, const std::string &msg) {
+ std::cerr << "[" << Enum<EventSeverity>::toString(severity) << "] " << msg << std::endl;
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/png_reader.cpp b/platform/default/src/mbgl/util/png_reader.cpp
new file mode 100644
index 0000000000..4d4ee29d1f
--- /dev/null
+++ b/platform/default/src/mbgl/util/png_reader.cpp
@@ -0,0 +1,142 @@
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/premultiply.hpp>
+#include <mbgl/util/char_array_buffer.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <istream>
+#include <sstream>
+
+extern "C"
+{
+#include <png.h>
+}
+
+template<size_t max, typename... Args>
+static std::string sprintf(const char *msg, Args... args) {
+ char res[max];
+ int len = snprintf(res, sizeof(res), msg, args...);
+ return std::string(res, len);
+}
+
+const static bool png_version_check __attribute__((unused)) = []() {
+ const png_uint_32 version = png_access_version_number();
+ if (version != PNG_LIBPNG_VER) {
+ throw std::runtime_error(sprintf<96>(
+ "libpng version mismatch: headers report %d.%d.%d, but library reports %d.%d.%d",
+ PNG_LIBPNG_VER / 10000, (PNG_LIBPNG_VER / 100) % 100, PNG_LIBPNG_VER % 100,
+ version / 10000, (version / 100) % 100, version % 100));
+ }
+ return true;
+}();
+
+namespace mbgl {
+
+static void user_error_fn(png_structp, png_const_charp error_msg) {
+ throw std::runtime_error(std::string("failed to read invalid png: '") + error_msg + "'");
+}
+
+static void user_warning_fn(png_structp, png_const_charp warning_msg) {
+ Log::Warning(Event::Image, "ImageReader (PNG): %s", warning_msg);
+}
+
+static void png_read_data(png_structp png_ptr, png_bytep data, png_size_t length) {
+ auto* fin = reinterpret_cast<std::istream*>(png_get_io_ptr(png_ptr));
+ fin->read(reinterpret_cast<char*>(data), length);
+ std::streamsize read_count = fin->gcount();
+ if (read_count < 0 || static_cast<png_size_t>(read_count) != length)
+ {
+ png_error(png_ptr, "Read Error");
+ }
+}
+
+struct png_struct_guard {
+ png_struct_guard(png_structpp png_ptr_ptr, png_infopp info_ptr_ptr)
+ : p_(png_ptr_ptr),
+ i_(info_ptr_ptr) {}
+
+ ~png_struct_guard() {
+ png_destroy_read_struct(p_,i_,nullptr);
+ }
+
+ png_structpp p_;
+ png_infopp i_;
+};
+
+PremultipliedImage decodePNG(const uint8_t* data, size_t size) {
+ util::CharArrayBuffer dataBuffer { reinterpret_cast<const char*>(data), size };
+ std::istream stream(&dataBuffer);
+
+ png_byte header[8] = { 0 };
+ stream.read(reinterpret_cast<char*>(header), 8);
+ if (stream.gcount() != 8)
+ throw std::runtime_error("PNG reader: Could not read image");
+
+ int is_png = !png_sig_cmp(header, 0, 8);
+ if (!is_png)
+ throw std::runtime_error("File or stream is not a png");
+
+ png_structp png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr);
+ if (!png_ptr)
+ throw std::runtime_error("failed to allocate png_ptr");
+
+ // catch errors in a custom way to avoid the need for setjmp
+ png_set_error_fn(png_ptr, png_get_error_ptr(png_ptr), user_error_fn, user_warning_fn);
+
+ png_infop info_ptr;
+ png_struct_guard sguard(&png_ptr, &info_ptr);
+ info_ptr = png_create_info_struct(png_ptr);
+ if (!info_ptr)
+ throw std::runtime_error("failed to create info_ptr");
+
+ png_set_read_fn(png_ptr, &stream, png_read_data);
+ png_set_sig_bytes(png_ptr, 8);
+ png_read_info(png_ptr, info_ptr);
+
+ png_uint_32 width = 0;
+ png_uint_32 height = 0;
+ int bit_depth = 0;
+ int color_type = 0;
+ png_get_IHDR(png_ptr, info_ptr, &width, &height, &bit_depth, &color_type, nullptr, nullptr, nullptr);
+
+ UnassociatedImage image({ static_cast<uint32_t>(width), static_cast<uint32_t>(height) });
+
+ if (color_type == PNG_COLOR_TYPE_PALETTE)
+ png_set_expand(png_ptr);
+
+ if (color_type == PNG_COLOR_TYPE_GRAY && bit_depth < 8)
+ png_set_expand(png_ptr);
+
+ if (png_get_valid(png_ptr, info_ptr, PNG_INFO_tRNS))
+ png_set_expand(png_ptr);
+
+ if (bit_depth == 16)
+ png_set_strip_16(png_ptr);
+
+ if (color_type == PNG_COLOR_TYPE_GRAY ||
+ color_type == PNG_COLOR_TYPE_GRAY_ALPHA)
+ png_set_gray_to_rgb(png_ptr);
+
+ png_set_add_alpha(png_ptr, 0xff, PNG_FILLER_AFTER);
+
+ if (png_get_interlace_type(png_ptr,info_ptr) == PNG_INTERLACE_ADAM7) {
+ png_set_interlace_handling(png_ptr); // FIXME: libpng bug?
+ // according to docs png_read_image
+ // "..automatically handles interlacing,
+ // so you don't need to call png_set_interlace_handling()"
+ }
+
+ png_read_update_info(png_ptr, info_ptr);
+
+ // we can read whole image at once
+ // alloc row pointers
+ const std::unique_ptr<png_bytep[]> rows(new png_bytep[height]);
+ for (unsigned row = 0; row < height; ++row)
+ rows[row] = image.data.get() + row * width * 4;
+ png_read_image(png_ptr, rows.get());
+
+ png_read_end(png_ptr, nullptr);
+
+ return util::premultiply(std::move(image));
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/png_writer.cpp b/platform/default/src/mbgl/util/png_writer.cpp
new file mode 100644
index 0000000000..b89e253f85
--- /dev/null
+++ b/platform/default/src/mbgl/util/png_writer.cpp
@@ -0,0 +1,77 @@
+#include <mbgl/util/compression.hpp>
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/premultiply.hpp>
+
+#include <boost/crc.hpp>
+
+#include <cassert>
+#include <cstring>
+
+#define NETWORK_BYTE_UINT32(value) \
+ char(value >> 24), char(value >> 16), char(value >> 8), char(value >> 0)
+
+namespace {
+
+void addChunk(std::string& png, const char* type, const char* data = "", const uint32_t size = 0) {
+ assert(strlen(type) == 4);
+
+ // Checksum encompasses type + data
+ boost::crc_32_type checksum;
+ checksum.process_bytes(type, 4);
+ checksum.process_bytes(data, size);
+
+ const char length[4] = { NETWORK_BYTE_UINT32(size) };
+ const char crc[4] = { NETWORK_BYTE_UINT32(checksum.checksum()) };
+
+ png.reserve(png.size() + 4 /* length */ + 4 /* type */ + size + 4 /* CRC */);
+ png.append(length, 4);
+ png.append(type, 4);
+ png.append(data, size);
+ png.append(crc, 4);
+}
+
+} // namespace
+
+namespace mbgl {
+
+// Encode PNGs without libpng.
+std::string encodePNG(const PremultipliedImage& pre) {
+ // Make copy of the image so that we can unpremultiply it.
+ const auto src = util::unpremultiply(pre.clone());
+
+ // PNG magic bytes
+ const char preamble[8] = { char(0x89), 'P', 'N', 'G', '\r', '\n', 0x1a, '\n' };
+
+ // IHDR chunk for our RGBA image.
+ const char ihdr[13] = {
+ NETWORK_BYTE_UINT32(src.size.width), // width
+ NETWORK_BYTE_UINT32(src.size.height), // height
+ 8, // bit depth == 8 bits
+ 6, // color type == RGBA
+ 0, // compression method == deflate
+ 0, // filter method == default
+ 0, // interlace method == none
+ };
+
+ // Prepare the (compressed) data chunk.
+ const auto stride = src.stride();
+ std::string idat;
+ for (uint32_t y = 0; y < src.size.height; y++) {
+ // Every scanline needs to be prefixed with one byte that indicates the filter type.
+ idat.append(1, 0); // filter type 0
+ idat.append((const char*)(src.data.get() + y * stride), stride);
+ }
+ idat = util::compress(idat);
+
+ // Assemble the PNG.
+ std::string png;
+ png.reserve((8 /* preamble */) + (12 + 13 /* IHDR */) +
+ (12 + idat.size() /* IDAT */) + (12 /* IEND */));
+ png.append(preamble, 8);
+ addChunk(png, "IHDR", ihdr, 13);
+ addChunk(png, "IDAT", idat.data(), static_cast<uint32_t>(idat.size()));
+ addChunk(png, "IEND");
+ return png;
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/run_loop.cpp b/platform/default/src/mbgl/util/run_loop.cpp
new file mode 100644
index 0000000000..868ee72114
--- /dev/null
+++ b/platform/default/src/mbgl/util/run_loop.cpp
@@ -0,0 +1,219 @@
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/async_task.hpp>
+#include <mbgl/util/thread_local.hpp>
+#include <mbgl/actor/scheduler.hpp>
+
+#include <uv.h>
+
+#include <cassert>
+#include <functional>
+#include <unordered_map>
+
+namespace {
+
+void dummyCallback(uv_async_t*) {}
+
+} // namespace
+
+namespace mbgl {
+namespace util {
+
+struct Watch {
+ static void onEvent(uv_poll_t* poll, int, int event) {
+ auto watch = reinterpret_cast<Watch*>(poll->data);
+
+ RunLoop::Event watchEvent = RunLoop::Event::None;
+ switch (event) {
+ case UV_READABLE:
+ watchEvent = RunLoop::Event::Read;
+ break;
+ case UV_WRITABLE:
+ watchEvent = RunLoop::Event::Write;
+ break;
+ case UV_READABLE | UV_WRITABLE:
+ watchEvent = RunLoop::Event::ReadWrite;
+ break;
+ }
+
+ watch->eventCallback(watch->fd, watchEvent);
+ };
+
+ static void onClose(uv_handle_t *poll) {
+ auto watch = reinterpret_cast<Watch*>(poll->data);
+ watch->closeCallback();
+ };
+
+ uv_poll_t poll;
+ int fd;
+
+ std::function<void(int, RunLoop::Event)> eventCallback;
+ std::function<void()> closeCallback;
+};
+
+RunLoop* RunLoop::Get() {
+ assert(static_cast<RunLoop*>(Scheduler::GetCurrent()));
+ return static_cast<RunLoop*>(Scheduler::GetCurrent());
+}
+
+class RunLoop::Impl {
+public:
+ void closeHolder() {
+ uv_close(holderHandle(), [](uv_handle_t* h) {
+ delete reinterpret_cast<uv_async_t*>(h);
+ });
+ }
+
+ uv_handle_t* holderHandle() {
+ return reinterpret_cast<uv_handle_t*>(holder);
+ }
+
+ uv_loop_t *loop = nullptr;
+ uv_async_t* holder = new uv_async_t;
+
+ RunLoop::Type type;
+ std::unique_ptr<AsyncTask> async;
+
+ std::unordered_map<int, std::unique_ptr<Watch>> watchPoll;
+};
+
+RunLoop::RunLoop(Type type) : impl(std::make_unique<Impl>()) {
+ switch (type) {
+ case Type::New:
+ impl->loop = new uv_loop_t;
+ if (uv_loop_init(impl->loop) != 0) {
+ throw std::runtime_error("Failed to initialize loop.");
+ }
+ break;
+ case Type::Default:
+ impl->loop = uv_default_loop();
+ break;
+ }
+
+ // Just for holding a ref to the main loop and keep
+ // it alive as required by libuv.
+ if (uv_async_init(impl->loop, impl->holder, dummyCallback) != 0) {
+ throw std::runtime_error("Failed to initialize async.");
+ }
+
+ impl->type = type;
+
+ Scheduler::SetCurrent(this);
+ impl->async = std::make_unique<AsyncTask>(std::bind(&RunLoop::process, this));
+}
+
+RunLoop::~RunLoop() {
+ Scheduler::SetCurrent(nullptr);
+
+ // Close the dummy handle that we have
+ // just to keep the main loop alive.
+ impl->closeHolder();
+
+ if (impl->type == Type::Default) {
+ return;
+ }
+
+ // Run the loop again to ensure that async
+ // close callbacks have been called. Not needed
+ // for the default main loop because it is only
+ // closed when the application exits.
+ impl->async.reset();
+ runOnce();
+
+ if (uv_loop_close(impl->loop) == UV_EBUSY) {
+ assert(false && "Failed to close loop.");
+ }
+ delete impl->loop;
+}
+
+LOOP_HANDLE RunLoop::getLoopHandle() {
+ return Get()->impl->loop;
+}
+
+void RunLoop::wake() {
+ impl->async->send();
+}
+
+void RunLoop::run() {
+ MBGL_VERIFY_THREAD(tid);
+
+ uv_ref(impl->holderHandle());
+ uv_run(impl->loop, UV_RUN_DEFAULT);
+}
+
+void RunLoop::runOnce() {
+ MBGL_VERIFY_THREAD(tid);
+
+ uv_run(impl->loop, UV_RUN_NOWAIT);
+}
+
+void RunLoop::stop() {
+ invoke([&] { uv_unref(impl->holderHandle()); });
+}
+
+void RunLoop::addWatch(int fd, Event event, std::function<void(int, Event)>&& callback) {
+ MBGL_VERIFY_THREAD(tid);
+
+ Watch *watch = nullptr;
+ auto watchPollIter = impl->watchPoll.find(fd);
+
+ if (watchPollIter == impl->watchPoll.end()) {
+ std::unique_ptr<Watch> watchPtr = std::make_unique<Watch>();
+
+ watch = watchPtr.get();
+ impl->watchPoll[fd] = std::move(watchPtr);
+
+ if (uv_poll_init(impl->loop, &watch->poll, fd)) {
+ throw std::runtime_error("Failed to init poll on file descriptor.");
+ }
+ } else {
+ watch = watchPollIter->second.get();
+ }
+
+ watch->poll.data = watch;
+ watch->fd = fd;
+ watch->eventCallback = std::move(callback);
+
+ int pollEvent = 0;
+ switch (event) {
+ case Event::Read:
+ pollEvent = UV_READABLE;
+ break;
+ case Event::Write:
+ pollEvent = UV_WRITABLE;
+ break;
+ case Event::ReadWrite:
+ pollEvent = UV_READABLE | UV_WRITABLE;
+ break;
+ default:
+ throw std::runtime_error("Unhandled event.");
+ }
+
+ if (uv_poll_start(&watch->poll, pollEvent, &Watch::onEvent)) {
+ throw std::runtime_error("Failed to start poll on file descriptor.");
+ }
+}
+
+void RunLoop::removeWatch(int fd) {
+ MBGL_VERIFY_THREAD(tid);
+
+ auto watchPollIter = impl->watchPoll.find(fd);
+ if (watchPollIter == impl->watchPoll.end()) {
+ return;
+ }
+
+ Watch* watch = watchPollIter->second.release();
+ impl->watchPoll.erase(watchPollIter);
+
+ watch->closeCallback = [watch] {
+ delete watch;
+ };
+
+ if (uv_poll_stop(&watch->poll)) {
+ throw std::runtime_error("Failed to stop poll on file descriptor.");
+ }
+
+ uv_close(reinterpret_cast<uv_handle_t*>(&watch->poll), &Watch::onClose);
+}
+
+} // namespace util
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/shared_thread_pool.cpp b/platform/default/src/mbgl/util/shared_thread_pool.cpp
new file mode 100644
index 0000000000..d7facbab94
--- /dev/null
+++ b/platform/default/src/mbgl/util/shared_thread_pool.cpp
@@ -0,0 +1,14 @@
+#include <mbgl/util/shared_thread_pool.hpp>
+
+namespace mbgl {
+
+std::shared_ptr<ThreadPool> sharedThreadPool() {
+ static std::weak_ptr<ThreadPool> weak;
+ auto pool = weak.lock();
+ if (!pool) {
+ weak = pool = std::make_shared<ThreadPool>(4);
+ }
+ return pool;
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/string_stdlib.cpp b/platform/default/src/mbgl/util/string_stdlib.cpp
new file mode 100644
index 0000000000..103444df1c
--- /dev/null
+++ b/platform/default/src/mbgl/util/string_stdlib.cpp
@@ -0,0 +1,74 @@
+#include <mbgl/util/platform.hpp>
+#include <libnu/casemap.h>
+#include <cstring>
+#include <sstream>
+
+namespace mbgl { namespace platform {
+
+std::string uppercase(const std::string& str)
+{
+ std::stringstream output;
+ char const *itr = str.c_str(), *nitr;
+ char const *end = itr + str.length();
+ char lo[5] = { 0 };
+
+ for (; itr < end; itr = nitr)
+ {
+ uint32_t code_point = 0;
+ char const* buf = nullptr;
+
+ nitr = _nu_toupper(itr, end, nu_utf8_read, &code_point, &buf, nullptr);
+ if (buf != nullptr)
+ {
+ do
+ {
+ buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point);
+ if (code_point == 0) break;
+ output.write(lo, nu_utf8_write(code_point, lo) - lo);
+ }
+ while (code_point != 0);
+ }
+ else
+ {
+ output.write(itr, nitr - itr);
+ }
+ }
+
+ return output.str();
+
+}
+
+std::string lowercase(const std::string& str)
+{
+ std::stringstream output;
+ char const *itr = str.c_str(), *nitr;
+ char const *end = itr + str.length();
+ char lo[5] = { 0 };
+
+ for (; itr < end; itr = nitr)
+ {
+ uint32_t code_point = 0;
+ char const* buf = nullptr;
+
+ nitr = _nu_tolower(itr, end, nu_utf8_read, &code_point, &buf, nullptr);
+ if (buf != nullptr)
+ {
+ do
+ {
+ buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point);
+ if (code_point == 0) break;
+ output.write(lo, nu_utf8_write(code_point, lo) - lo);
+ }
+ while (code_point != 0);
+ }
+ else
+ {
+ output.write(itr, nitr - itr);
+ }
+ }
+
+ return output.str();
+}
+
+} // namespace platform
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/thread.cpp b/platform/default/src/mbgl/util/thread.cpp
new file mode 100644
index 0000000000..c7c79b4fb0
--- /dev/null
+++ b/platform/default/src/mbgl/util/thread.cpp
@@ -0,0 +1,37 @@
+#include <mbgl/util/platform.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <string>
+
+#include <pthread.h>
+#include <sched.h>
+
+namespace mbgl {
+namespace platform {
+
+std::string getCurrentThreadName() {
+ char name[32] = "unknown";
+ pthread_getname_np(pthread_self(), name, sizeof(name));
+
+ return name;
+}
+
+void setCurrentThreadName(const std::string& name) {
+ if (name.size() > 15) { // Linux hard limit (see manpages).
+ pthread_setname_np(pthread_self(), name.substr(0, 15).c_str());
+ } else {
+ pthread_setname_np(pthread_self(), name.c_str());
+ }
+}
+
+void makeThreadLowPriority() {
+ struct sched_param param;
+ param.sched_priority = 0;
+
+ if (sched_setscheduler(0, SCHED_IDLE, &param) != 0) {
+ Log::Warning(Event::General, "Couldn't set thread scheduling policy");
+ }
+}
+
+} // namespace platform
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/thread_local.cpp b/platform/default/src/mbgl/util/thread_local.cpp
new file mode 100644
index 0000000000..db70773c12
--- /dev/null
+++ b/platform/default/src/mbgl/util/thread_local.cpp
@@ -0,0 +1,66 @@
+#include <mbgl/util/thread_local.hpp>
+
+#include <mbgl/renderer/backend_scope.hpp>
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/run_loop.hpp>
+
+#include <stdexcept>
+#include <cassert>
+
+#include <pthread.h>
+
+namespace mbgl {
+namespace util {
+
+template <class T>
+class ThreadLocal<T>::Impl {
+public:
+ pthread_key_t key;
+};
+
+template <class T>
+ThreadLocal<T>::ThreadLocal() : impl(std::make_unique<Impl>()) {
+ int ret = pthread_key_create(&impl->key, [](void *) {});
+
+ if (ret) {
+ throw std::runtime_error("Failed to init local storage key.");
+ }
+}
+
+template <class T>
+ThreadLocal<T>::~ThreadLocal() {
+ // ThreadLocal will not take ownership
+ // of the pointer it is managing. The pointer
+ // needs to be explicitly cleared before we
+ // destroy this object.
+ assert(!get());
+
+ if (pthread_key_delete(impl->key)) {
+ Log::Error(Event::General, "Failed to delete local storage key.");
+ assert(false);
+ }
+}
+
+template <class T>
+T* ThreadLocal<T>::get() {
+ auto* ret = reinterpret_cast<T*>(pthread_getspecific(impl->key));
+ if (!ret) {
+ return nullptr;
+ }
+
+ return ret;
+}
+
+template <class T>
+void ThreadLocal<T>::set(T* ptr) {
+ if (pthread_setspecific(impl->key, ptr)) {
+ throw std::runtime_error("Failed to set local storage.");
+ }
+}
+
+template class ThreadLocal<BackendScope>;
+template class ThreadLocal<Scheduler>;
+template class ThreadLocal<int>; // For unit tests
+
+} // namespace util
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/timer.cpp b/platform/default/src/mbgl/util/timer.cpp
new file mode 100644
index 0000000000..90a85bfc1f
--- /dev/null
+++ b/platform/default/src/mbgl/util/timer.cpp
@@ -0,0 +1,73 @@
+#include <mbgl/util/timer.hpp>
+
+#include <mbgl/util/run_loop.hpp>
+
+#include <uv.h>
+
+namespace mbgl {
+namespace util {
+
+class Timer::Impl {
+public:
+ Impl() : timer(new uv_timer_t) {
+ auto* loop = reinterpret_cast<uv_loop_t*>(RunLoop::getLoopHandle());
+ if (uv_timer_init(loop, timer) != 0) {
+ throw std::runtime_error("Failed to initialize timer.");
+ }
+
+ handle()->data = this;
+ uv_unref(handle());
+ }
+
+ ~Impl() {
+ uv_close(handle(), [](uv_handle_t* h) {
+ delete reinterpret_cast<uv_timer_t*>(h);
+ });
+ }
+
+ void start(uint64_t timeout, uint64_t repeat, std::function<void ()>&& cb_) {
+ cb = std::move(cb_);
+ if (uv_timer_start(timer, timerCallback, timeout, repeat) != 0) {
+ throw std::runtime_error("Failed to start timer.");
+ }
+ }
+
+ void stop() {
+ cb = nullptr;
+ if (uv_timer_stop(timer) != 0) {
+ throw std::runtime_error("Failed to stop timer.");
+ }
+ }
+
+private:
+ static void timerCallback(uv_timer_t* handle) {
+ reinterpret_cast<Impl*>(handle->data)->cb();
+ }
+
+ uv_handle_t* handle() {
+ return reinterpret_cast<uv_handle_t*>(timer);
+ }
+
+ uv_timer_t* timer;
+
+ std::function<void()> cb;
+};
+
+Timer::Timer()
+ : impl(std::make_unique<Impl>()) {
+}
+
+Timer::~Timer() = default;
+
+void Timer::start(Duration timeout, Duration repeat, std::function<void()>&& cb) {
+ impl->start(std::chrono::duration_cast<Milliseconds>(timeout).count(),
+ std::chrono::duration_cast<Milliseconds>(repeat).count(),
+ std::move(cb));
+}
+
+void Timer::stop() {
+ impl->stop();
+}
+
+} // namespace util
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/util/utf.cpp b/platform/default/src/mbgl/util/utf.cpp
new file mode 100644
index 0000000000..f0f9d3e67a
--- /dev/null
+++ b/platform/default/src/mbgl/util/utf.cpp
@@ -0,0 +1,17 @@
+#include <mbgl/util/utf.hpp>
+
+#include <boost/locale/encoding_utf.hpp>
+
+namespace mbgl {
+namespace util {
+
+std::u16string convertUTF8ToUTF16(const std::string& str) {
+ return boost::locale::conv::utf_to_utf<char16_t>(str);
+}
+
+std::string convertUTF16ToUTF8(const std::u16string& str) {
+ return boost::locale::conv::utf_to_utf<char>(str);
+}
+
+} // namespace util
+} // namespace mbgl