diff options
author | Konstantin Käfer <mail@kkaefer.com> | 2018-12-13 18:45:29 +0100 |
---|---|---|
committer | Konstantin Käfer <mail@kkaefer.com> | 2018-12-14 11:03:03 +0100 |
commit | 1d8235f5b899a2cd8414522b2d72b96fab91577b (patch) | |
tree | 2ab56dce064de872525db7f24ba150a9065c4757 /platform/default/src | |
parent | c2a4a8822ce9577c972975da61034a30fb0fe3e9 (diff) | |
download | qtlocation-mapboxgl-1d8235f5b899a2cd8414522b2d72b96fab91577b.tar.gz |
[build] rework platform/default directory and add -files.txt for vendored libs
Diffstat (limited to 'platform/default/src')
34 files changed, 5750 insertions, 0 deletions
diff --git a/platform/default/src/mbgl/gl/headless_backend.cpp b/platform/default/src/mbgl/gl/headless_backend.cpp new file mode 100644 index 0000000000..ba08aecab7 --- /dev/null +++ b/platform/default/src/mbgl/gl/headless_backend.cpp @@ -0,0 +1,85 @@ +#include <mbgl/gl/headless_backend.hpp> +#include <mbgl/gl/context.hpp> +#include <mbgl/renderer/backend_scope.hpp> + +#include <cassert> +#include <stdexcept> +#include <type_traits> + +namespace mbgl { + +class HeadlessBackend::View { +public: + View(gl::Context& context, Size size_) + : color(context.createRenderbuffer<gl::RenderbufferType::RGBA>(size_)), + depthStencil(context.createRenderbuffer<gl::RenderbufferType::DepthStencil>(size_)), + framebuffer(context.createFramebuffer(color, depthStencil)) { + } + + gl::Renderbuffer<gl::RenderbufferType::RGBA> color; + gl::Renderbuffer<gl::RenderbufferType::DepthStencil> depthStencil; + gl::Framebuffer framebuffer; +}; + +HeadlessBackend::HeadlessBackend(Size size_) + : size(size_) { +} + +HeadlessBackend::~HeadlessBackend() { + BackendScope guard { *this }; + view.reset(); + context.reset(); +} + +gl::ProcAddress HeadlessBackend::getExtensionFunctionPointer(const char* name) { + assert(impl); + return impl->getExtensionFunctionPointer(name); +} + +void HeadlessBackend::activate() { + active = true; + + if (!impl) { + createImpl(); + } + + assert(impl); + impl->activateContext(); +} + +void HeadlessBackend::deactivate() { + assert(impl); + impl->deactivateContext(); + active = false; +} + +void HeadlessBackend::bind() { + gl::Context& context_ = getContext(); + + if (!view) { + view = std::make_unique<View>(context_, size); + } + + context_.bindFramebuffer = view->framebuffer.framebuffer; + context_.scissorTest = false; + context_.viewport = { 0, 0, size }; +} + +Size HeadlessBackend::getFramebufferSize() const { + return size; +} + +void HeadlessBackend::updateAssumedState() { + // no-op +} + +void HeadlessBackend::setSize(Size size_) { + size = size_; + view.reset(); +} + +PremultipliedImage HeadlessBackend::readStillImage() { + return getContext().readFramebuffer<PremultipliedImage>(size); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/gl/headless_backend_osmesa.cpp b/platform/default/src/mbgl/gl/headless_backend_osmesa.cpp new file mode 100644 index 0000000000..0da1caf9af --- /dev/null +++ b/platform/default/src/mbgl/gl/headless_backend_osmesa.cpp @@ -0,0 +1,47 @@ +#include <mbgl/gl/headless_backend.hpp> +#include <mbgl/util/logging.hpp> + +#include <GL/osmesa.h> + +#include <cassert> + +namespace mbgl { + +class OSMesaBackendImpl : public HeadlessBackend::Impl { +public: + OSMesaBackendImpl() { +#if OSMESA_MAJOR_VERSION * 100 + OSMESA_MINOR_VERSION >= 305 + glContext = OSMesaCreateContextExt(OSMESA_RGBA, 16, 0, 0, nullptr); +#else + glContext = OSMesaCreateContext(OSMESA_RGBA, nullptr); +#endif + if (glContext == nullptr) { + throw std::runtime_error("Error creating GL context object."); + } + } + + ~OSMesaBackendImpl() final { + OSMesaDestroyContext(glContext); + } + + gl::ProcAddress getExtensionFunctionPointer(const char* name) final { + return OSMesaGetProcAddress(name); + } + + void activateContext() final { + if (!OSMesaMakeCurrent(glContext, &fakeBuffer, GL_UNSIGNED_BYTE, 1, 1)) { + throw std::runtime_error("Switching OpenGL context failed.\n"); + } + } + +private: + OSMesaContext glContext = nullptr; + GLubyte fakeBuffer = 0; +}; + +void HeadlessBackend::createImpl() { + assert(!impl); + impl = std::make_unique<OSMesaBackendImpl>(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/gl/headless_frontend.cpp b/platform/default/src/mbgl/gl/headless_frontend.cpp new file mode 100644 index 0000000000..37b0f91f32 --- /dev/null +++ b/platform/default/src/mbgl/gl/headless_frontend.cpp @@ -0,0 +1,144 @@ +#include <mbgl/gl/headless_frontend.hpp> +#include <mbgl/renderer/renderer.hpp> +#include <mbgl/renderer/renderer_state.hpp> +#include <mbgl/renderer/update_parameters.hpp> +#include <mbgl/map/map.hpp> +#include <mbgl/map/transform_state.hpp> +#include <mbgl/util/run_loop.hpp> + +namespace mbgl { + +HeadlessFrontend::HeadlessFrontend(float pixelRatio_, FileSource& fileSource, Scheduler& scheduler, const optional<std::string> programCacheDir, GLContextMode mode, const optional<std::string> localFontFamily) + : HeadlessFrontend({ 256, 256 }, pixelRatio_, fileSource, scheduler, programCacheDir, mode, localFontFamily) { +} + +HeadlessFrontend::HeadlessFrontend(Size size_, float pixelRatio_, FileSource& fileSource, Scheduler& scheduler, const optional<std::string> programCacheDir, GLContextMode mode, const optional<std::string> localFontFamily) + : size(size_), + pixelRatio(pixelRatio_), + backend({ static_cast<uint32_t>(size.width * pixelRatio), + static_cast<uint32_t>(size.height * pixelRatio) }), + asyncInvalidate([this] { + if (renderer && updateParameters) { + mbgl::BackendScope guard { backend }; + renderer->render(*updateParameters); + } + }), + renderer(std::make_unique<Renderer>(backend, pixelRatio, fileSource, scheduler, mode, programCacheDir, localFontFamily)) { +} + +HeadlessFrontend::~HeadlessFrontend() = default; + +void HeadlessFrontend::reset() { + assert(renderer); + renderer.reset(); +} + +void HeadlessFrontend::update(std::shared_ptr<UpdateParameters> updateParameters_) { + updateParameters = updateParameters_; + asyncInvalidate.send(); +} + +void HeadlessFrontend::setObserver(RendererObserver& observer_) { + assert(renderer); + renderer->setObserver(&observer_); +} + +Size HeadlessFrontend::getSize() const { + return size; +} + +Renderer* HeadlessFrontend::getRenderer() { + assert(renderer); + return renderer.get(); +} + +RendererBackend* HeadlessFrontend::getBackend() { + return &backend; +} + +CameraOptions HeadlessFrontend::getCameraOptions() { + if (updateParameters) + return RendererState::getCameraOptions(*updateParameters); + + static CameraOptions nullCamera; + return nullCamera; +} + +bool HeadlessFrontend::hasImage(const std::string& id) { + if (updateParameters) { + return RendererState::hasImage(*updateParameters, id); + } + + return false; +} + +bool HeadlessFrontend::hasLayer(const std::string& id) { + if (updateParameters) { + return RendererState::hasLayer(*updateParameters, id); + } + + return false; +} + +bool HeadlessFrontend::hasSource(const std::string& id) { + if (updateParameters) { + return RendererState::hasSource(*updateParameters, id); + } + + return false; +} +ScreenCoordinate HeadlessFrontend::pixelForLatLng(const LatLng& coordinate) { + if (updateParameters) { + return RendererState::pixelForLatLng(*updateParameters, coordinate); + } + + return ScreenCoordinate {}; +} + +LatLng HeadlessFrontend::latLngForPixel(const ScreenCoordinate& point) { + if (updateParameters) { + return RendererState::latLngForPixel(*updateParameters, point); + } + + return LatLng {}; +} + +void HeadlessFrontend::setSize(Size size_) { + if (size != size_) { + size = size_; + backend.setSize({ static_cast<uint32_t>(size_.width * pixelRatio), + static_cast<uint32_t>(size_.height * pixelRatio) }); + } +} + +PremultipliedImage HeadlessFrontend::readStillImage() { + return backend.readStillImage(); +} + +PremultipliedImage HeadlessFrontend::render(Map& map) { + PremultipliedImage result; + + map.renderStill([&](std::exception_ptr error) { + if (error) { + std::rethrow_exception(error); + } else { + result = backend.readStillImage(); + } + }); + + while (!result.valid()) { + util::RunLoop::Get()->runOnce(); + } + + return result; +} + +optional<TransformState> HeadlessFrontend::getTransformState() const { + if (updateParameters) { + return updateParameters->transformState; + } else { + return {}; + } +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/layermanager/layer_manager.cpp b/platform/default/src/mbgl/layermanager/layer_manager.cpp new file mode 100644 index 0000000000..05d0f4d1ae --- /dev/null +++ b/platform/default/src/mbgl/layermanager/layer_manager.cpp @@ -0,0 +1,79 @@ +#include <mbgl/layermanager/layer_manager.hpp> + +#include <mbgl/layermanager/background_layer_factory.hpp> +#include <mbgl/layermanager/circle_layer_factory.hpp> +#include <mbgl/layermanager/custom_layer_factory.hpp> +#include <mbgl/layermanager/fill_extrusion_layer_factory.hpp> +#include <mbgl/layermanager/fill_layer_factory.hpp> +#include <mbgl/layermanager/heatmap_layer_factory.hpp> +#include <mbgl/layermanager/hillshade_layer_factory.hpp> +#include <mbgl/layermanager/line_layer_factory.hpp> +#include <mbgl/layermanager/raster_layer_factory.hpp> +#include <mbgl/layermanager/symbol_layer_factory.hpp> + +#include <map> +#include <memory> +#include <vector> + +namespace mbgl { + +class LayerManagerDefault final : public LayerManager { +public: + LayerManagerDefault(); + +private: + void addLayerType(std::unique_ptr<LayerFactory>); + // LayerManager overrides. + LayerFactory* getFactory(const std::string& type) noexcept final; + LayerFactory* getFactory(const style::LayerTypeInfo*) noexcept final; + + std::vector<std::unique_ptr<LayerFactory>> factories; + std::map<std::string, LayerFactory*> typeToFactory; +}; + +LayerManagerDefault::LayerManagerDefault() { + addLayerType(std::make_unique<FillLayerFactory>()); + addLayerType(std::make_unique<LineLayerFactory>()); + addLayerType(std::make_unique<CircleLayerFactory>()); + addLayerType(std::make_unique<SymbolLayerFactory>()); + addLayerType(std::make_unique<RasterLayerFactory>()); + addLayerType(std::make_unique<BackgroundLayerFactory>()); + addLayerType(std::make_unique<HillshadeLayerFactory>()); + addLayerType(std::make_unique<FillExtrusionLayerFactory>()); + addLayerType(std::make_unique<HeatmapLayerFactory>()); + addLayerType(std::make_unique<CustomLayerFactory>()); +} + +void LayerManagerDefault::addLayerType(std::unique_ptr<LayerFactory> factory) { + std::string type{factory->getTypeInfo()->type}; + if (!type.empty()) { + typeToFactory.emplace(std::make_pair(std::move(type), factory.get())); + } + factories.emplace_back(std::move(factory)); +} + +LayerFactory* LayerManagerDefault::getFactory(const mbgl::style::LayerTypeInfo* typeInfo) noexcept { + assert(typeInfo); + for (const auto& factory: factories) { + if (factory->getTypeInfo() == typeInfo) { + return factory.get(); + } + } + assert(false); + return nullptr; +} + +LayerFactory* LayerManagerDefault::getFactory(const std::string& type) noexcept { + auto search = typeToFactory.find(type); + return (search != typeToFactory.end()) ? search->second : nullptr; +} + +// static +LayerManager* LayerManager::get() noexcept { + static LayerManagerDefault instance; + return &instance; +} + +const bool LayerManager::annotationsEnabled = true; + +} // namespace mbgl diff --git a/platform/default/src/mbgl/map/map_snapshotter.cpp b/platform/default/src/mbgl/map/map_snapshotter.cpp new file mode 100644 index 0000000000..ae14b20721 --- /dev/null +++ b/platform/default/src/mbgl/map/map_snapshotter.cpp @@ -0,0 +1,224 @@ +#include <mbgl/map/map_snapshotter.hpp> + +#include <mbgl/actor/actor_ref.hpp> +#include <mbgl/gl/headless_frontend.hpp> +#include <mbgl/map/map.hpp> +#include <mbgl/map/transform_state.hpp> +#include <mbgl/storage/file_source.hpp> +#include <mbgl/style/style.hpp> +#include <mbgl/util/event.hpp> +#include <mbgl/map/transform.hpp> + +namespace mbgl { + +class MapSnapshotter::Impl { +public: + Impl(FileSource*, + std::shared_ptr<Scheduler>, + const std::pair<bool, std::string> style, + const Size&, + const float pixelRatio, + const optional<CameraOptions> cameraOptions, + const optional<LatLngBounds> region, + const optional<std::string> programCacheDir, + const optional<std::string> localFontFamily = {}); + + void setStyleURL(std::string styleURL); + std::string getStyleURL() const; + + void setStyleJSON(std::string styleJSON); + std::string getStyleJSON() const; + + void setSize(Size); + Size getSize() const; + + void setCameraOptions(CameraOptions); + CameraOptions getCameraOptions() const; + + void setRegion(LatLngBounds); + LatLngBounds getRegion() const; + + void snapshot(ActorRef<MapSnapshotter::Callback>); + +private: + std::shared_ptr<Scheduler> scheduler; + HeadlessFrontend frontend; + Map map; +}; + +MapSnapshotter::Impl::Impl(FileSource* fileSource, + std::shared_ptr<Scheduler> scheduler_, + const std::pair<bool, std::string> style, + const Size& size, + const float pixelRatio, + const optional<CameraOptions> cameraOptions, + const optional<LatLngBounds> region, + const optional<std::string> programCacheDir, + const optional<std::string> localFontFamily) + : scheduler(std::move(scheduler_)) + , frontend(size, pixelRatio, *fileSource, *scheduler, programCacheDir, GLContextMode::Unique, localFontFamily) + , map(frontend, MapObserver::nullObserver(), size, pixelRatio, *fileSource, *scheduler, MapMode::Static) { + + if (style.first) { + map.getStyle().loadJSON(style.second); + } else{ + map.getStyle().loadURL(style.second); + } + + if (cameraOptions) { + map.jumpTo(*cameraOptions); + } + + // Set region, if specified + if (region) { + this->setRegion(*region); + } +} + +void MapSnapshotter::Impl::snapshot(ActorRef<MapSnapshotter::Callback> callback) { + map.renderStill([this, callback = std::move(callback)] (std::exception_ptr error) mutable { + + // Create lambda that captures the current transform state + // and can be used to translate for geographic to screen + // coordinates + assert (frontend.getTransformState()); + PointForFn pointForFn { [=, center=map.getLatLng(), transformState = *frontend.getTransformState()] (const LatLng& latLng) { + LatLng unwrappedLatLng = latLng.wrapped(); + unwrappedLatLng.unwrapForShortestPath(center); + Transform transform { transformState }; + return transform.latLngToScreenCoordinate(unwrappedLatLng); + }}; + + // Create lambda that captures the current transform state + // and can be used to translate for geographic to screen + // coordinates + assert (frontend.getTransformState()); + LatLngForFn latLngForFn { [=, transformState = *frontend.getTransformState()] (const ScreenCoordinate& screenCoordinate) { + Transform transform { transformState }; + return transform.screenCoordinateToLatLng(screenCoordinate); + }}; + + // Collect all source attributions + std::vector<std::string> attributions; + for (auto source : map.getStyle().getSources()) { + auto attribution = source->getAttribution(); + if (attribution) { + attributions.push_back(*attribution); + } + } + + // Invoke callback + callback.invoke( + &MapSnapshotter::Callback::operator(), + error, + error ? PremultipliedImage() : frontend.readStillImage(), + std::move(attributions), + std::move(pointForFn), + std::move(latLngForFn) + ); + }); +} + +void MapSnapshotter::Impl::setStyleURL(std::string styleURL) { + map.getStyle().loadURL(styleURL); +} + +std::string MapSnapshotter::Impl::getStyleURL() const { + return map.getStyle().getURL(); +} + +void MapSnapshotter::Impl::setStyleJSON(std::string styleJSON) { + map.getStyle().loadJSON(styleJSON); +} + +std::string MapSnapshotter::Impl::getStyleJSON() const { + return map.getStyle().getJSON(); +} + +void MapSnapshotter::Impl::setSize(Size size) { + map.setSize(size); + frontend.setSize(size); +} + +Size MapSnapshotter::Impl::getSize() const { + return map.getSize(); +} + +void MapSnapshotter::Impl::setCameraOptions(CameraOptions cameraOptions) { + map.jumpTo(cameraOptions); +} + +CameraOptions MapSnapshotter::Impl::getCameraOptions() const { + EdgeInsets insets; + return map.getCameraOptions(insets); +} + +void MapSnapshotter::Impl::setRegion(LatLngBounds region) { + mbgl::EdgeInsets insets = { 0, 0, 0, 0 }; + std::vector<LatLng> latLngs = { region.southwest(), region.northeast() }; + map.jumpTo(map.cameraForLatLngs(latLngs, insets)); +} + +LatLngBounds MapSnapshotter::Impl::getRegion() const { + return map.latLngBoundsForCamera(getCameraOptions()); +} + +MapSnapshotter::MapSnapshotter(FileSource* fileSource, + std::shared_ptr<Scheduler> scheduler, + const std::pair<bool, std::string> style, + const Size& size, + const float pixelRatio, + const optional<CameraOptions> cameraOptions, + const optional<LatLngBounds> region, + const optional<std::string> programCacheDir, + const optional<std::string> localFontFamily) + : impl(std::make_unique<util::Thread<MapSnapshotter::Impl>>("Map Snapshotter", fileSource, std::move(scheduler), style, size, pixelRatio, cameraOptions, region, programCacheDir, localFontFamily)) { +} + +MapSnapshotter::~MapSnapshotter() = default; + +void MapSnapshotter::snapshot(ActorRef<MapSnapshotter::Callback> callback) { + impl->actor().invoke(&Impl::snapshot, std::move(callback)); +} + +void MapSnapshotter::setStyleURL(const std::string& styleURL) { + impl->actor().invoke(&Impl::setStyleURL, styleURL); +} + +std::string MapSnapshotter::getStyleURL() const { + return impl->actor().ask(&Impl::getStyleURL).get(); +} + +void MapSnapshotter::setStyleJSON(const std::string& styleJSON) { + impl->actor().invoke(&Impl::setStyleJSON, styleJSON); +} + +std::string MapSnapshotter::getStyleJSON() const { + return impl->actor().ask(&Impl::getStyleJSON).get(); +} + +void MapSnapshotter::setSize(const Size& size) { + impl->actor().invoke(&Impl::setSize, size); +} + +Size MapSnapshotter::getSize() const { + return impl->actor().ask(&Impl::getSize).get(); +} + +void MapSnapshotter::setCameraOptions(const CameraOptions& options) { + impl->actor().invoke(&Impl::setCameraOptions, options); +} + +CameraOptions MapSnapshotter::getCameraOptions() const { + return impl->actor().ask(&Impl::getCameraOptions).get(); +} + +void MapSnapshotter::setRegion(const LatLngBounds& bounds) { + impl->actor().invoke(&Impl::setRegion, std::move(bounds)); +} + +LatLngBounds MapSnapshotter::getRegion() const { + return impl->actor().ask(&Impl::getRegion).get(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/asset_file_source.cpp b/platform/default/src/mbgl/storage/asset_file_source.cpp new file mode 100644 index 0000000000..7988654ae5 --- /dev/null +++ b/platform/default/src/mbgl/storage/asset_file_source.cpp @@ -0,0 +1,81 @@ +#include <mbgl/storage/asset_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/util.hpp> +#include <mbgl/util/io.hpp> + +#include <sys/types.h> +#include <sys/stat.h> + +namespace { + +const std::string assetProtocol = "asset://"; + +} // namespace + +namespace mbgl { + +class AssetFileSource::Impl { +public: + Impl(ActorRef<Impl>, std::string root_) + : root(std::move(root_)) { + } + + void request(const std::string& url, ActorRef<FileSourceRequest> req) { + Response response; + + if (!acceptsURL(url)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, + "Invalid asset URL"); + req.invoke(&FileSourceRequest::setResponse, response); + return; + } + + // Cut off the protocol and prefix with path. + const auto path = root + "/" + mbgl::util::percentDecode(url.substr(assetProtocol.size())); + struct stat buf; + int result = stat(path.c_str(), &buf); + + if (result == 0 && (S_IFDIR & buf.st_mode)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (result == -1 && errno == ENOENT) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else { + try { + response.data = std::make_shared<std::string>(util::read_file(path)); + } catch (...) { + response.error = std::make_unique<Response::Error>( + Response::Error::Reason::Other, + util::toString(std::current_exception())); + } + } + + req.invoke(&FileSourceRequest::setResponse, response); + } + +private: + std::string root; +}; + +AssetFileSource::AssetFileSource(const std::string& root) + : impl(std::make_unique<util::Thread<Impl>>("AssetFileSource", root)) { +} + +AssetFileSource::~AssetFileSource() = default; + +std::unique_ptr<AsyncRequest> AssetFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + impl->actor().invoke(&Impl::request, resource.url, req->actor()); + + return std::move(req); +} + +bool AssetFileSource::acceptsURL(const std::string& url) { + return 0 == url.rfind(assetProtocol, 0); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/default_file_source.cpp b/platform/default/src/mbgl/storage/default_file_source.cpp new file mode 100644 index 0000000000..cad68e7de9 --- /dev/null +++ b/platform/default/src/mbgl/storage/default_file_source.cpp @@ -0,0 +1,316 @@ +#include <mbgl/storage/default_file_source.hpp> +#include <mbgl/storage/asset_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/local_file_source.hpp> +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/resource_transform.hpp> + +#include <mbgl/util/platform.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/work_request.hpp> +#include <mbgl/util/stopwatch.hpp> + +#include <cassert> + +namespace mbgl { + +class DefaultFileSource::Impl { +public: + Impl(std::shared_ptr<FileSource> assetFileSource_, std::string cachePath, uint64_t maximumCacheSize) + : assetFileSource(assetFileSource_) + , localFileSource(std::make_unique<LocalFileSource>()) + , offlineDatabase(std::make_unique<OfflineDatabase>(cachePath, maximumCacheSize)) { + } + + void setAPIBaseURL(const std::string& url) { + onlineFileSource.setAPIBaseURL(url); + } + + std::string getAPIBaseURL() const{ + return onlineFileSource.getAPIBaseURL(); + } + + void setAccessToken(const std::string& accessToken) { + onlineFileSource.setAccessToken(accessToken); + } + + std::string getAccessToken() const { + return onlineFileSource.getAccessToken(); + } + + void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + onlineFileSource.setResourceTransform(std::move(transform)); + } + + void listRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + callback(offlineDatabase->listRegions()); + } + + void createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) { + callback(offlineDatabase->createRegion(definition, metadata)); + } + + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + callback(offlineDatabase->mergeDatabase(sideDatabasePath)); + } + + void updateMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) { + callback(offlineDatabase->updateMetadata(regionID, metadata)); + } + + void getRegionStatus(int64_t regionID, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) { + if (auto download = getDownload(regionID)) { + callback(download.value()->getStatus()); + } else { + callback(unexpected<std::exception_ptr>(download.error())); + } + } + + void deleteRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) { + downloads.erase(region.getID()); + callback(offlineDatabase->deleteRegion(std::move(region))); + } + + void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) { + if (auto download = getDownload(regionID)) { + download.value()->setObserver(std::move(observer)); + } + } + + void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { + if (auto download = getDownload(regionID)) { + download.value()->setState(state); + } + } + + void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) { + auto callback = [ref] (const Response& res) mutable { + ref.invoke(&FileSourceRequest::setResponse, res); + }; + + if (AssetFileSource::acceptsURL(resource.url)) { + //Asset request + tasks[req] = assetFileSource->request(resource, callback); + } else if (LocalFileSource::acceptsURL(resource.url)) { + //Local file request + tasks[req] = localFileSource->request(resource, callback); + } else { + // Try the offline database + if (resource.hasLoadingMethod(Resource::LoadingMethod::Cache)) { + auto offlineResponse = offlineDatabase->get(resource); + + if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) { + if (!offlineResponse) { + // Ensure there's always a response that we can send, so the caller knows that + // there's no optional data available in the cache, when it's the only place + // we're supposed to load from. + offlineResponse.emplace(); + offlineResponse->noContent = true; + offlineResponse->error = std::make_unique<Response::Error>( + Response::Error::Reason::NotFound, "Not found in offline database"); + } else if (!offlineResponse->isUsable()) { + // Don't return resources the server requested not to show when they're stale. + // Even if we can't directly use the response, we may still use it to send a + // conditional HTTP request, which is why we're saving it above. + offlineResponse->error = std::make_unique<Response::Error>( + Response::Error::Reason::NotFound, "Cached resource is unusable"); + } + callback(*offlineResponse); + } else if (offlineResponse) { + // Copy over the fields so that we can use them when making a refresh request. + resource.priorModified = offlineResponse->modified; + resource.priorExpires = offlineResponse->expires; + resource.priorEtag = offlineResponse->etag; + resource.priorData = offlineResponse->data; + + if (offlineResponse->isUsable()) { + callback(*offlineResponse); + } + } + } + + // Get from the online file source + if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) { + MBGL_TIMING_START(watch); + tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) mutable { + this->offlineDatabase->put(resource, onlineResponse); + if (resource.kind == Resource::Kind::Tile) { + // onlineResponse.data will be null if data not modified + MBGL_TIMING_FINISH(watch, + " Action: " << "Requesting," << + " URL: " << resource.url.c_str() << + " Size: " << (onlineResponse.data != nullptr ? onlineResponse.data->size() : 0) << "B," << + " Time") + } + callback(onlineResponse); + }); + } + } + } + + void cancel(AsyncRequest* req) { + tasks.erase(req); + } + + void setOfflineMapboxTileCountLimit(uint64_t limit) { + offlineDatabase->setOfflineMapboxTileCountLimit(limit); + } + + void setOnlineStatus(const bool status) { + onlineFileSource.setOnlineStatus(status); + } + + void put(const Resource& resource, const Response& response) { + offlineDatabase->put(resource, response); + } + +private: + expected<OfflineDownload*, std::exception_ptr> getDownload(int64_t regionID) { + auto it = downloads.find(regionID); + if (it != downloads.end()) { + return it->second.get(); + } + auto definition = offlineDatabase->getRegionDefinition(regionID); + if (!definition) { + return unexpected<std::exception_ptr>(definition.error()); + } + auto download = std::make_unique<OfflineDownload>(regionID, std::move(definition.value()), + *offlineDatabase, onlineFileSource); + return downloads.emplace(regionID, std::move(download)).first->second.get(); + } + + // shared so that destruction is done on the creating thread + const std::shared_ptr<FileSource> assetFileSource; + const std::unique_ptr<FileSource> localFileSource; + std::unique_ptr<OfflineDatabase> offlineDatabase; + OnlineFileSource onlineFileSource; + std::unordered_map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks; + std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads; +}; + +DefaultFileSource::DefaultFileSource(const std::string& cachePath, + const std::string& assetRoot, + uint64_t maximumCacheSize) + : DefaultFileSource(cachePath, std::make_unique<AssetFileSource>(assetRoot), maximumCacheSize) { +} + +DefaultFileSource::DefaultFileSource(const std::string& cachePath, + std::unique_ptr<FileSource>&& assetFileSource_, + uint64_t maximumCacheSize) + : assetFileSource(std::move(assetFileSource_)) + , impl(std::make_unique<util::Thread<Impl>>("DefaultFileSource", assetFileSource, cachePath, maximumCacheSize)) { +} + +DefaultFileSource::~DefaultFileSource() = default; + +void DefaultFileSource::setAPIBaseURL(const std::string& baseURL) { + impl->actor().invoke(&Impl::setAPIBaseURL, baseURL); + + { + std::lock_guard<std::mutex> lock(cachedBaseURLMutex); + cachedBaseURL = baseURL; + } +} + +std::string DefaultFileSource::getAPIBaseURL() { + std::lock_guard<std::mutex> lock(cachedBaseURLMutex); + return cachedBaseURL; +} + +void DefaultFileSource::setAccessToken(const std::string& accessToken) { + impl->actor().invoke(&Impl::setAccessToken, accessToken); + + { + std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); + cachedAccessToken = accessToken; + } +} + +std::string DefaultFileSource::getAccessToken() { + std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); + return cachedAccessToken; +} + +void DefaultFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + impl->actor().invoke(&Impl::setResourceTransform, std::move(transform)); +} + +std::unique_ptr<AsyncRequest> DefaultFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + req->onCancel([fs = impl->actor(), req = req.get()] () mutable { fs.invoke(&Impl::cancel, req); }); + + impl->actor().invoke(&Impl::request, req.get(), resource, req->actor()); + + return std::move(req); +} + +void DefaultFileSource::listOfflineRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::listRegions, callback); +} + +void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::createRegion, definition, metadata, callback); +} + +void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback); +} + +void DefaultFileSource::updateOfflineMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, + std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback); +} + +void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::deleteRegion, std::move(region), callback); +} + +void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr<OfflineRegionObserver> observer) { + impl->actor().invoke(&Impl::setRegionObserver, region.getID(), std::move(observer)); +} + +void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { + impl->actor().invoke(&Impl::setRegionDownloadState, region.getID(), state); +} + +void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) const { + impl->actor().invoke(&Impl::getRegionStatus, region.getID(), callback); +} + +void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const { + impl->actor().invoke(&Impl::setOfflineMapboxTileCountLimit, limit); +} + +void DefaultFileSource::pause() { + impl->pause(); +} + +void DefaultFileSource::resume() { + impl->resume(); +} + +void DefaultFileSource::put(const Resource& resource, const Response& response) { + impl->actor().invoke(&Impl::put, resource, response); +} + +// For testing only: + +void DefaultFileSource::setOnlineStatus(const bool status) { + impl->actor().invoke(&Impl::setOnlineStatus, status); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source_request.cpp b/platform/default/src/mbgl/storage/file_source_request.cpp new file mode 100644 index 0000000000..09ea8cc32a --- /dev/null +++ b/platform/default/src/mbgl/storage/file_source_request.cpp @@ -0,0 +1,37 @@ +#include <mbgl/storage/file_source_request.hpp> + +#include <mbgl/actor/mailbox.hpp> +#include <mbgl/actor/scheduler.hpp> + +namespace mbgl { + +FileSourceRequest::FileSourceRequest(FileSource::Callback&& callback) + : responseCallback(callback) + , mailbox(std::make_shared<Mailbox>(*Scheduler::GetCurrent())) { +} + +FileSourceRequest::~FileSourceRequest() { + if (cancelCallback) { + cancelCallback(); + } + + mailbox->close(); +} + +void FileSourceRequest::onCancel(std::function<void()>&& callback) { + cancelCallback = std::move(callback); +} + +void FileSourceRequest::setResponse(const Response& response) { + // Copy, because calling the callback will sometimes self + // destroy this object. We cannot move because this method + // can be called more than one. + auto callback = responseCallback; + callback(response); +} + +ActorRef<FileSourceRequest> FileSourceRequest::actor() { + return ActorRef<FileSourceRequest>(*this, mailbox); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/http_file_source.cpp b/platform/default/src/mbgl/storage/http_file_source.cpp new file mode 100644 index 0000000000..213b53de98 --- /dev/null +++ b/platform/default/src/mbgl/storage/http_file_source.cpp @@ -0,0 +1,495 @@ +#include <mbgl/storage/http_file_source.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/logging.hpp> + +#include <mbgl/util/util.hpp> +#include <mbgl/util/optional.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/timer.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/http_header.hpp> + +#include <curl/curl.h> + +// Dynamically load all cURL functions. Debian-derived systems upgraded the OpenSSL version linked +// to in https://salsa.debian.org/debian/curl/commit/95c94957bb7e89e36e78b995fed468c42f64d18d +// They state: +// Rename libcurl3 to libcurl4, because libcurl exposes an SSL_CTX via +// CURLOPT_SSL_CTX_FUNCTION, and this object changes incompatibly between +// openssl 1.0 and openssl 1.1. +// Since we are not accessing the underlying OpenSSL context, we don't care whether we're linking +// against libcurl3 or libcurl4; both use the ABI version 4 which hasn't changed since 2006 +// (see https://curl.haxx.se/libcurl/abi.html). In fact, cURL's ABI compatibility is very good as +// shown on https://abi-laboratory.pro/tracker/timeline/curl/ +// Therefore, we're dynamically loading the cURL symbols we need to avoid linking against versioned +// symbols. +#include <dlfcn.h> + +namespace curl { + +#define CURL_FUNCTIONS \ + X(global_init) \ + X(getdate) \ + X(easy_strerror) \ + X(easy_init) \ + X(easy_setopt) \ + X(easy_cleanup) \ + X(easy_getinfo) \ + X(easy_reset) \ + X(multi_init) \ + X(multi_add_handle) \ + X(multi_remove_handle) \ + X(multi_cleanup) \ + X(multi_info_read) \ + X(multi_strerror) \ + X(multi_socket_action) \ + X(multi_setopt) \ + X(share_init) \ + X(share_cleanup) \ + X(slist_append) \ + X(slist_free_all) + +#define X(name) static decltype(&curl_ ## name) name = nullptr; +CURL_FUNCTIONS +#undef X + +static void* handle = nullptr; + +static void* load(const char* name) { + void* symbol = dlsym(handle, name); + if (const char* error = dlerror()) { + fprintf(stderr, "Cannot load symbol '%s': %s\n", name, error); + dlclose(handle); + handle = nullptr; + abort(); + } + return symbol; +} + +__attribute__((constructor)) +static void load() { + assert(!handle); + handle = dlopen("libcurl.so.4", RTLD_LAZY | RTLD_LOCAL); + if (!handle) { + fprintf(stderr, "Could not open shared library '%s'\n", "libcurl.so.4"); + abort(); + } + + #define X(name) name = (decltype(&curl_ ## name))load("curl_" #name); + CURL_FUNCTIONS + #undef X +} + +__attribute__((constructor)) +static void unload() { + if (handle) { + dlclose(handle); + } +} + +} // namespace curl + + +#include <queue> +#include <map> +#include <cassert> +#include <cstring> +#include <cstdio> + +static void handleError(CURLMcode code) { + if (code != CURLM_OK) { + throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(code)); + } +} + +static void handleError(CURLcode code) { + if (code != CURLE_OK) { + throw std::runtime_error(std::string("CURL easy error: ") + curl::easy_strerror(code)); + } +} + +namespace mbgl { + +class HTTPFileSource::Impl { +public: + Impl(); + ~Impl(); + + static int handleSocket(CURL *handle, curl_socket_t s, int action, void *userp, void *socketp); + static int startTimeout(CURLM *multi, long timeout_ms, void *userp); + static void onTimeout(HTTPFileSource::Impl *context); + + void perform(curl_socket_t s, util::RunLoop::Event event); + CURL *getHandle(); + void returnHandle(CURL *handle); + void checkMultiInfo(); + + // Used as the CURL timer function to periodically check for socket updates. + util::Timer timeout; + + // CURL multi handle that we use to request multiple URLs at the same time, without having to + // block and spawn threads. + CURLM *multi = nullptr; + + // CURL share handles are used for sharing session state (e.g.) + CURLSH *share = nullptr; + + // A queue that we use for storing resuable CURL easy handles to avoid creating and destroying + // them all the time. + std::queue<CURL *> handles; +}; + +class HTTPRequest : public AsyncRequest { +public: + HTTPRequest(HTTPFileSource::Impl*, Resource, FileSource::Callback); + ~HTTPRequest() override; + + void handleResult(CURLcode code); + +private: + static size_t headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp); + static size_t writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp); + + HTTPFileSource::Impl* context = nullptr; + Resource resource; + FileSource::Callback callback; + + // Will store the current response. + std::shared_ptr<std::string> data; + std::unique_ptr<Response> response; + + optional<std::string> retryAfter; + optional<std::string> xRateLimitReset; + + CURL *handle = nullptr; + curl_slist *headers = nullptr; + + char error[CURL_ERROR_SIZE] = { 0 }; +}; + +HTTPFileSource::Impl::Impl() { + if (curl::global_init(CURL_GLOBAL_ALL)) { + throw std::runtime_error("Could not init cURL"); + } + + share = curl::share_init(); + + multi = curl::multi_init(); + handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETFUNCTION, handleSocket)); + handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETDATA, this)); + handleError(curl::multi_setopt(multi, CURLMOPT_TIMERFUNCTION, startTimeout)); + handleError(curl::multi_setopt(multi, CURLMOPT_TIMERDATA, this)); +} + +HTTPFileSource::Impl::~Impl() { + while (!handles.empty()) { + curl::easy_cleanup(handles.front()); + handles.pop(); + } + + curl::multi_cleanup(multi); + multi = nullptr; + + curl::share_cleanup(share); + share = nullptr; + + timeout.stop(); +} + +CURL *HTTPFileSource::Impl::getHandle() { + if (!handles.empty()) { + auto handle = handles.front(); + handles.pop(); + return handle; + } else { + return curl::easy_init(); + } +} + +void HTTPFileSource::Impl::returnHandle(CURL *handle) { + curl::easy_reset(handle); + handles.push(handle); +} + +void HTTPFileSource::Impl::checkMultiInfo() { + CURLMsg *message = nullptr; + int pending = 0; + + while ((message = curl::multi_info_read(multi, &pending))) { + switch (message->msg) { + case CURLMSG_DONE: { + HTTPRequest *baton = nullptr; + curl::easy_getinfo(message->easy_handle, CURLINFO_PRIVATE, (char *)&baton); + assert(baton); + baton->handleResult(message->data.result); + } break; + + default: + // This should never happen, because there are no other message types. + throw std::runtime_error("CURLMsg returned unknown message type"); + } + } +} + +void HTTPFileSource::Impl::perform(curl_socket_t s, util::RunLoop::Event events) { + int flags = 0; + + if (events == util::RunLoop::Event::Read) { + flags |= CURL_CSELECT_IN; + } + if (events == util::RunLoop::Event::Write) { + flags |= CURL_CSELECT_OUT; + } + + + int running_handles = 0; + curl::multi_socket_action(multi, s, flags, &running_handles); + checkMultiInfo(); +} + +int HTTPFileSource::Impl::handleSocket(CURL * /* handle */, curl_socket_t s, int action, void *userp, + void * /* socketp */) { + assert(userp); + auto context = reinterpret_cast<Impl *>(userp); + + switch (action) { + case CURL_POLL_IN: { + using namespace std::placeholders; + util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Read, + std::bind(&Impl::perform, context, _1, _2)); + break; + } + case CURL_POLL_OUT: { + using namespace std::placeholders; + util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Write, + std::bind(&Impl::perform, context, _1, _2)); + break; + } + case CURL_POLL_REMOVE: + util::RunLoop::Get()->removeWatch(s); + break; + default: + throw std::runtime_error("Unhandled CURL socket action"); + } + + return 0; +} + +void HTTPFileSource::Impl::onTimeout(Impl *context) { + int running_handles; + CURLMcode error = curl::multi_socket_action(context->multi, CURL_SOCKET_TIMEOUT, 0, &running_handles); + if (error != CURLM_OK) { + throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(error)); + } + context->checkMultiInfo(); +} + +int HTTPFileSource::Impl::startTimeout(CURLM * /* multi */, long timeout_ms, void *userp) { + assert(userp); + auto context = reinterpret_cast<Impl *>(userp); + + if (timeout_ms < 0) { + // A timeout of 0 ms means that the timer will invoked in the next loop iteration. + timeout_ms = 0; + } + + context->timeout.stop(); + context->timeout.start(mbgl::Milliseconds(timeout_ms), Duration::zero(), + std::bind(&Impl::onTimeout, context)); + + return 0; +} + +HTTPRequest::HTTPRequest(HTTPFileSource::Impl* context_, Resource resource_, FileSource::Callback callback_) + : context(context_), + resource(std::move(resource_)), + callback(std::move(callback_)), + handle(context->getHandle()) { + + // If there's already a response, set the correct etags/modified headers to make sure we are + // getting a 304 response if possible. This avoids redownloading unchanged data. + if (resource.priorEtag) { + const std::string header = std::string("If-None-Match: ") + *resource.priorEtag; + headers = curl::slist_append(headers, header.c_str()); + } else if (resource.priorModified) { + const std::string time = + std::string("If-Modified-Since: ") + util::rfc1123(*resource.priorModified); + headers = curl::slist_append(headers, time.c_str()); + } + + if (headers) { + curl::easy_setopt(handle, CURLOPT_HTTPHEADER, headers); + } + + handleError(curl::easy_setopt(handle, CURLOPT_PRIVATE, this)); + handleError(curl::easy_setopt(handle, CURLOPT_ERRORBUFFER, error)); + handleError(curl::easy_setopt(handle, CURLOPT_CAINFO, "ca-bundle.crt")); + handleError(curl::easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1)); + handleError(curl::easy_setopt(handle, CURLOPT_URL, resource.url.c_str())); + handleError(curl::easy_setopt(handle, CURLOPT_WRITEFUNCTION, writeCallback)); + handleError(curl::easy_setopt(handle, CURLOPT_WRITEDATA, this)); + handleError(curl::easy_setopt(handle, CURLOPT_HEADERFUNCTION, headerCallback)); + handleError(curl::easy_setopt(handle, CURLOPT_HEADERDATA, this)); +#if LIBCURL_VERSION_NUM >= ((7) << 16 | (21) << 8 | 6) // Renamed in 7.21.6 + handleError(curl::easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate")); +#else + handleError(curl::easy_setopt(handle, CURLOPT_ENCODING, "gzip, deflate")); +#endif + handleError(curl::easy_setopt(handle, CURLOPT_USERAGENT, "MapboxGL/1.0")); + handleError(curl::easy_setopt(handle, CURLOPT_SHARE, context->share)); + + // Start requesting the information. + handleError(curl::multi_add_handle(context->multi, handle)); +} + +HTTPRequest::~HTTPRequest() { + handleError(curl::multi_remove_handle(context->multi, handle)); + context->returnHandle(handle); + handle = nullptr; + + if (headers) { + curl::slist_free_all(headers); + headers = nullptr; + } +} + +// This function is called when we have new data for a request. We just append it to the string +// containing the previous data. +size_t HTTPRequest::writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp) { + assert(userp); + auto impl = reinterpret_cast<HTTPRequest *>(userp); + + if (!impl->data) { + impl->data = std::make_shared<std::string>(); + } + + impl->data->append((char *)contents, size * nmemb); + return size * nmemb; +} + +// Compares the beginning of the (non-zero-terminated!) data buffer with the (zero-terminated!) +// header string. If the data buffer contains the header string at the beginning, it returns +// the length of the header string == begin of the value, otherwise it returns npos. +// The comparison of the header is ASCII-case-insensitive. +size_t headerMatches(const char *const header, const char *const buffer, const size_t length) { + const size_t headerLength = strlen(header); + if (length < headerLength) { + return std::string::npos; + } + size_t i = 0; + while (i < length && i < headerLength && std::tolower(buffer[i]) == std::tolower(header[i])) { + i++; + } + return i == headerLength ? i : std::string::npos; +} + +size_t HTTPRequest::headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp) { + assert(userp); + auto baton = reinterpret_cast<HTTPRequest *>(userp); + + if (!baton->response) { + baton->response = std::make_unique<Response>(); + } + + const size_t length = size * nmemb; + size_t begin = std::string::npos; + if ((begin = headerMatches("last-modified: ", buffer, length)) != std::string::npos) { + // Always overwrite the modification date; We might already have a value here from the + // Date header, but this one is more accurate. + const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n + baton->response->modified = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) }; + } else if ((begin = headerMatches("etag: ", buffer, length)) != std::string::npos) { + baton->response->etag = std::string(buffer + begin, length - begin - 2); // remove \r\n + } else if ((begin = headerMatches("cache-control: ", buffer, length)) != std::string::npos) { + const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n + const auto cc = http::CacheControl::parse(value.c_str()); + baton->response->expires = cc.toTimePoint(); + baton->response->mustRevalidate = cc.mustRevalidate; + } else if ((begin = headerMatches("expires: ", buffer, length)) != std::string::npos) { + const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n + baton->response->expires = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) }; + } else if ((begin = headerMatches("retry-after: ", buffer, length)) != std::string::npos) { + baton->retryAfter = std::string(buffer + begin, length - begin - 2); // remove \r\n + } else if ((begin = headerMatches("x-rate-limit-reset: ", buffer, length)) != std::string::npos) { + baton->xRateLimitReset = std::string(buffer + begin, length - begin - 2); // remove \r\n + } + + return length; +} + +void HTTPRequest::handleResult(CURLcode code) { + // Make sure a response object exists in case we haven't got any headers or content. + if (!response) { + response = std::make_unique<Response>(); + } + + using Error = Response::Error; + + // Add human-readable error code + if (code != CURLE_OK) { + switch (code) { + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_CONNECT: + case CURLE_OPERATION_TIMEDOUT: + + response->error = std::make_unique<Error>( + Error::Reason::Connection, std::string{ curl::easy_strerror(code) } + ": " + error); + break; + + default: + response->error = std::make_unique<Error>( + Error::Reason::Other, std::string{ curl::easy_strerror(code) } + ": " + error); + break; + } + } else { + long responseCode = 0; + curl::easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode); + + if (responseCode == 200) { + if (data) { + response->data = std::move(data); + } else { + response->data = std::make_shared<std::string>(); + } + } else if (responseCode == 204 || (responseCode == 404 && resource.kind == Resource::Kind::Tile)) { + response->noContent = true; + } else if (responseCode == 304) { + response->notModified = true; + } else if (responseCode == 404) { + response->error = + std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404"); + } else if (responseCode == 429) { + response->error = + std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429", + http::parseRetryHeaders(retryAfter, xRateLimitReset)); + } else if (responseCode >= 500 && responseCode < 600) { + response->error = + std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } + + util::toString(responseCode)); + } else { + response->error = + std::make_unique<Error>(Error::Reason::Other, std::string{ "HTTP status code " } + + util::toString(responseCode)); + } + } + + // Calling `callback` may result in deleting `this`. Copy data to temporaries first. + auto callback_ = callback; + auto response_ = *response; + callback_(response_); +} + +HTTPFileSource::HTTPFileSource() + : impl(std::make_unique<Impl>()) { +} + +HTTPFileSource::~HTTPFileSource() = default; + +std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource, Callback callback) { + return std::make_unique<HTTPRequest>(impl.get(), resource, callback); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp new file mode 100644 index 0000000000..1b7b7b9278 --- /dev/null +++ b/platform/default/src/mbgl/storage/local_file_source.cpp @@ -0,0 +1,81 @@ +#include <mbgl/storage/local_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/util.hpp> +#include <mbgl/util/io.hpp> + +#include <sys/types.h> +#include <sys/stat.h> + +#if defined(_WINDOWS) && !defined(S_ISDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + +namespace { + +const std::string fileProtocol = "file://"; + +} // namespace + +namespace mbgl { + +class LocalFileSource::Impl { +public: + Impl(ActorRef<Impl>) {} + + void request(const std::string& url, ActorRef<FileSourceRequest> req) { + Response response; + + if (!acceptsURL(url)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, + "Invalid file URL"); + req.invoke(&FileSourceRequest::setResponse, response); + return; + } + + // Cut off the protocol and prefix with path. + const auto path = mbgl::util::percentDecode(url.substr(fileProtocol.size())); + struct stat buf; + int result = stat(path.c_str(), &buf); + + if (result == 0 && S_ISDIR(buf.st_mode)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (result == -1 && errno == ENOENT) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else { + try { + response.data = std::make_shared<std::string>(util::read_file(path)); + } catch (...) { + response.error = std::make_unique<Response::Error>( + Response::Error::Reason::Other, + util::toString(std::current_exception())); + } + } + + req.invoke(&FileSourceRequest::setResponse, response); + } + +}; + +LocalFileSource::LocalFileSource() + : impl(std::make_unique<util::Thread<Impl>>("LocalFileSource")) { +} + +LocalFileSource::~LocalFileSource() = default; + +std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + impl->actor().invoke(&Impl::request, resource.url, req->actor()); + + return std::move(req); +} + +bool LocalFileSource::acceptsURL(const std::string& url) { + return 0 == url.rfind(fileProtocol, 0); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline.cpp b/platform/default/src/mbgl/storage/offline.cpp new file mode 100644 index 0000000000..e1ec0acb31 --- /dev/null +++ b/platform/default/src/mbgl/storage/offline.cpp @@ -0,0 +1,158 @@ +#include <mbgl/storage/offline.hpp> +#include <mbgl/util/tileset.hpp> +#include <mbgl/util/projection.hpp> + +#include <mapbox/geojson.hpp> +#include <mapbox/geojson/rapidjson.hpp> + +#include <rapidjson/document.h> +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> + +#include <cmath> + +namespace mbgl { + +// OfflineTilePyramidRegionDefinition + +OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( + std::string styleURL_, LatLngBounds bounds_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(std::move(styleURL_)), + bounds(std::move(bounds_)), + minZoom(minZoom_), + maxZoom(maxZoom_), + pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + + +// OfflineGeometryRegionDefinition + +OfflineGeometryRegionDefinition::OfflineGeometryRegionDefinition(std::string styleURL_, Geometry<double> geometry_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(styleURL_) + , geometry(std::move(geometry_)) + , minZoom(minZoom_) + , maxZoom(maxZoom_) + , pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.Parse<0>(region.c_str()); + + // validation + + auto hasValidBounds = [&] { + return doc.HasMember("bounds") && doc["bounds"].IsArray() && doc["bounds"].Size() == 4 + && doc["bounds"][0].IsDouble() && doc["bounds"][1].IsDouble() + && doc["bounds"][2].IsDouble() && doc["bounds"][3].IsDouble(); + }; + + auto hasValidGeometry = [&] { + return doc.HasMember("geometry") && doc["geometry"].IsObject(); + }; + + if (doc.HasParseError() + || !doc.HasMember("style_url") || !doc["style_url"].IsString() + || !(hasValidBounds() || hasValidGeometry()) + || !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() + || (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) + || !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { + throw std::runtime_error("Malformed offline region definition"); + } + + // Common properties + + std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() }; + double minZoom = doc["min_zoom"].GetDouble(); + double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY; + float pixelRatio = doc["pixel_ratio"].GetDouble(); + + if (doc.HasMember("bounds")) { + return OfflineTilePyramidRegionDefinition{ + styleURL, + LatLngBounds::hull( + LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), + LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())), + minZoom, maxZoom, pixelRatio }; + } else { + return OfflineGeometryRegionDefinition{ + styleURL, + mapbox::geojson::convert<Geometry<double>>(doc["geometry"].GetObject()), + minZoom, maxZoom, pixelRatio }; + }; + +} + +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.SetObject(); + + // Encode common properties + region.match([&](auto& _region) { + doc.AddMember("style_url", rapidjson::StringRef(_region.styleURL.data(), _region.styleURL.length()), doc.GetAllocator()); + doc.AddMember("min_zoom", _region.minZoom, doc.GetAllocator()); + if (std::isfinite(_region.maxZoom)) { + doc.AddMember("max_zoom", _region.maxZoom, doc.GetAllocator()); + } + + doc.AddMember("pixel_ratio", _region.pixelRatio, doc.GetAllocator()); + }); + + // Encode specific properties + region.match( + [&] (const OfflineTilePyramidRegionDefinition& _region) { + rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); + bounds.PushBack(_region.bounds.south(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.west(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.north(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.east(), doc.GetAllocator()); + doc.AddMember("bounds", bounds, doc.GetAllocator()); + + }, + [&] (const OfflineGeometryRegionDefinition& _region) { + doc.AddMember("geometry", mapbox::geojson::convert(_region.geometry, doc.GetAllocator()), doc.GetAllocator()); + + } + ); + + rapidjson::StringBuffer buffer; + rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); + doc.Accept(writer); + + return buffer.GetString(); +} + + +// OfflineRegion + +OfflineRegion::OfflineRegion(int64_t id_, + OfflineRegionDefinition definition_, + OfflineRegionMetadata metadata_) + : id(id_), + definition(std::move(definition_)), + metadata(std::move(metadata_)) { +} + +OfflineRegion::OfflineRegion(OfflineRegion&&) = default; +OfflineRegion::~OfflineRegion() = default; + +const OfflineRegionDefinition& OfflineRegion::getDefinition() const { + return definition; +} + +const OfflineRegionMetadata& OfflineRegion::getMetadata() const { + return metadata; +} + +int64_t OfflineRegion::getID() const { + return id; +} +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline_database.cpp b/platform/default/src/mbgl/storage/offline_database.cpp new file mode 100644 index 0000000000..7732076991 --- /dev/null +++ b/platform/default/src/mbgl/storage/offline_database.cpp @@ -0,0 +1,1129 @@ +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/storage/sqlite3.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/logging.hpp> + +#include <mbgl/storage/offline_schema.hpp> +#include <mbgl/storage/merge_sideloaded.hpp> + + +namespace mbgl { + +OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_) + : path(std::move(path_)), + maximumCacheSize(maximumCacheSize_) { + try { + initialize(); + } catch (const util::IOException& ex) { + handleError(ex, "open database"); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "open database"); + } + // Assume that we can't open the database right now and work with an empty database object. +} + +OfflineDatabase::~OfflineDatabase() { + // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we + // can't throw anything. + try { + statements.clear(); + db.reset(); + } catch (const util::IOException& ex) { + handleError(ex, "close database"); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "close database"); + } +} + +void OfflineDatabase::initialize() { + assert(!db); + assert(statements.empty()); + + db = std::make_unique<mapbox::sqlite::Database>( + mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate)); + db->setBusyTimeout(Milliseconds::max()); + db->exec("PRAGMA foreign_keys = ON"); + + const auto userVersion = getPragma<int64_t>("PRAGMA user_version"); + switch (userVersion) { + case 0: + case 1: + // Newly created database, or old cache-only database; remove old table if it exists. + removeOldCacheTable(); + createSchema(); + return; + case 2: + migrateToVersion3(); + // fall through + case 3: + // Removed migration, see below. + // fall through + case 4: + migrateToVersion5(); + // fall through + case 5: + migrateToVersion6(); + // fall through + case 6: + // Happy path; we're done + return; + default: + // Downgrade: delete the database and try to reinitialize. + removeExisting(); + initialize(); + } +} + +void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex, const char* action) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt || + (ex.code == mapbox::sqlite::ResultCode::ReadOnly && + ex.extendedCode == mapbox::sqlite::ExtendedResultCode::ReadOnlyDBMoved)) { + // The database was corruped, moved away, or deleted. We're going to start fresh with a + // clean slate for the next operation. + Log::Error(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what()); + try { + removeExisting(); + } catch (const util::IOException& ioEx) { + handleError(ioEx, action); + } + } else { + // We treat the error as temporary, and pretend we have an inaccessible DB. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what()); + } +} + +void OfflineDatabase::handleError(const util::IOException& ex, const char* action) { + // We failed to delete the database file. + Log::Error(Event::Database, ex.code, "Can't %s: %s", action, ex.what()); +} + +void OfflineDatabase::removeExisting() { + Log::Warning(Event::Database, "Removing existing incompatible offline database"); + + statements.clear(); + db.reset(); + + util::deleteFile(path); +} + +void OfflineDatabase::removeOldCacheTable() { + assert(db); + db->exec("DROP TABLE IF EXISTS http_cache"); + db->exec("VACUUM"); +} + +void OfflineDatabase::createSchema() { + assert(db); + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + mapbox::sqlite::Transaction transaction(*db); + db->exec(offlineDatabaseSchema); + db->exec("PRAGMA user_version = 6"); + transaction.commit(); +} + +void OfflineDatabase::migrateToVersion3() { + assert(db); + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("VACUUM"); + db->exec("PRAGMA user_version = 3"); +} + +// Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre- +// release development and the migration was removed entirely to avoid potential +// conflicts from quickly (and needlessly) switching journal and sync modes. +// +// See: https://github.com/mapbox/mapbox-gl-native/pull/6320 + +void OfflineDatabase::migrateToVersion5() { + assert(db); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + db->exec("PRAGMA user_version = 5"); +} + +void OfflineDatabase::migrateToVersion6() { + assert(db); + mapbox::sqlite::Transaction transaction(*db); + db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); + db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); + db->exec("PRAGMA user_version = 6"); + transaction.commit(); +} + +mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { + if (!db) { + initialize(); + } + auto it = statements.find(sql); + if (it == statements.end()) { + it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first; + } + return *it->second; +} + +optional<Response> OfflineDatabase::get(const Resource& resource) try { + auto result = getInternal(resource); + return result ? optional<Response>{ result->first } : nullopt; +} catch (const util::IOException& ex) { + handleError(ex, "read resource"); + return nullopt; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read resource"); + return nullopt; +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return getTile(*resource.tileData); + } else { + return getResource(resource); + } +} + +optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return hasTile(*resource.tileData); + } else { + return hasResource(resource); + } +} + +std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try { + if (!db) { + initialize(); + } + mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); + auto result = putInternal(resource, response, true); + transaction.commit(); + return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write resource"); + return { false, 0 }; +} + +std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) { + if (response.error) { + return { false, 0 }; + } + + std::string compressedData; + bool compressed = false; + uint64_t size = 0; + + if (response.data) { + compressedData = util::compress(*response.data); + compressed = compressedData.size() < response.data->size(); + size = compressed ? compressedData.size() : response.data->size(); + } + + if (evict_ && !evict(size)) { + Log::Info(Event::Database, "Unable to make space for entry"); + return { false, 0 }; + } + + bool inserted; + + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + inserted = putTile(*resource.tileData, response, + compressed ? compressedData : response.data ? *response.data : "", + compressed); + } else { + inserted = putResource(resource, response, + compressed ? compressedData : response.data ? *response.data : "", + compressed); + } + + return { inserted, size }; +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) { + // Update accessed timestamp used for LRU eviction. + try { + mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") }; + accessedQuery.bind(1, util::now()); + accessedQuery.bind(2, resource.url); + accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); + } + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + // 0 1 2 3 4 5 + "SELECT etag, expires, must_revalidate, modified, data, compressed " + "FROM resources " + "WHERE url = ?") }; + // clang-format on + + query.bind(1, resource.url); + + if (!query.run()) { + return nullopt; + } + + Response response; + uint64_t size = 0; + + response.etag = query.get<optional<std::string>>(0); + response.expires = query.get<optional<Timestamp>>(1); + response.mustRevalidate = query.get<bool>(2); + response.modified = query.get<optional<Timestamp>>(3); + + auto data = query.get<optional<std::string>>(4); + if (!data) { + response.noContent = true; + } else if (query.get<bool>(5)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + size = data->length(); + } else { + response.data = std::make_shared<std::string>(*data); + size = data->length(); + } + + return std::make_pair(response, size); +} + +optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) { + mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") }; + query.bind(1, resource.url); + if (!query.run()) { + return nullopt; + } + + return query.get<optional<int64_t>>(0); +} + +bool OfflineDatabase::putResource(const Resource& resource, + const Response& response, + const std::string& data, + bool compressed) { + if (response.notModified) { + // clang-format off + mapbox::sqlite::Query notModifiedQuery{ getStatement( + "UPDATE resources " + "SET accessed = ?1, " + " expires = ?2, " + " must_revalidate = ?3 " + "WHERE url = ?4 ") }; + // clang-format on + + notModifiedQuery.bind(1, util::now()); + notModifiedQuery.bind(2, response.expires); + notModifiedQuery.bind(3, response.mustRevalidate); + notModifiedQuery.bind(4, resource.url); + notModifiedQuery.run(); + return false; + } + + // We can't use REPLACE because it would change the id value. + // clang-format off + mapbox::sqlite::Query updateQuery{ getStatement( + "UPDATE resources " + "SET kind = ?1, " + " etag = ?2, " + " expires = ?3, " + " must_revalidate = ?4, " + " modified = ?5, " + " accessed = ?6, " + " data = ?7, " + " compressed = ?8 " + "WHERE url = ?9 ") }; + // clang-format on + + updateQuery.bind(1, int(resource.kind)); + updateQuery.bind(2, response.etag); + updateQuery.bind(3, response.expires); + updateQuery.bind(4, response.mustRevalidate); + updateQuery.bind(5, response.modified); + updateQuery.bind(6, util::now()); + updateQuery.bind(9, resource.url); + + if (response.noContent) { + updateQuery.bind(7, nullptr); + updateQuery.bind(8, false); + } else { + updateQuery.bindBlob(7, data.data(), data.size(), false); + updateQuery.bind(8, compressed); + } + + updateQuery.run(); + if (updateQuery.changes() != 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") }; + // clang-format on + + insertQuery.bind(1, resource.url); + insertQuery.bind(2, int(resource.kind)); + insertQuery.bind(3, response.etag); + insertQuery.bind(4, response.expires); + insertQuery.bind(5, response.mustRevalidate); + insertQuery.bind(6, response.modified); + insertQuery.bind(7, util::now()); + + if (response.noContent) { + insertQuery.bind(8, nullptr); + insertQuery.bind(9, false); + } else { + insertQuery.bindBlob(8, data.data(), data.size(), false); + insertQuery.bind(9, compressed); + } + + insertQuery.run(); + + return true; +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) { + // Update accessed timestamp used for LRU eviction. + try { + // clang-format off + mapbox::sqlite::Query accessedQuery{ getStatement( + "UPDATE tiles " + "SET accessed = ?1 " + "WHERE url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 ") }; + // clang-format on + + accessedQuery.bind(1, util::now()); + accessedQuery.bind(2, tile.urlTemplate); + accessedQuery.bind(3, tile.pixelRatio); + accessedQuery.bind(4, tile.x); + accessedQuery.bind(5, tile.y); + accessedQuery.bind(6, tile.z); + accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); + } + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + // 0 1 2, 3, 4, 5 + "SELECT etag, expires, must_revalidate, modified, data, compressed " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 ") }; + // clang-format on + + query.bind(1, tile.urlTemplate); + query.bind(2, tile.pixelRatio); + query.bind(3, tile.x); + query.bind(4, tile.y); + query.bind(5, tile.z); + + if (!query.run()) { + return nullopt; + } + + Response response; + uint64_t size = 0; + + response.etag = query.get<optional<std::string>>(0); + response.expires = query.get<optional<Timestamp>>(1); + response.mustRevalidate = query.get<bool>(2); + response.modified = query.get<optional<Timestamp>>(3); + + optional<std::string> data = query.get<optional<std::string>>(4); + if (!data) { + response.noContent = true; + } else if (query.get<bool>(5)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + size = data->length(); + } else { + response.data = std::make_shared<std::string>(*data); + size = data->length(); + } + + return std::make_pair(response, size); +} + +optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) { + // clang-format off + mapbox::sqlite::Query size{ getStatement( + "SELECT length(data) " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 ") }; + // clang-format on + + size.bind(1, tile.urlTemplate); + size.bind(2, tile.pixelRatio); + size.bind(3, tile.x); + size.bind(4, tile.y); + size.bind(5, tile.z); + + if (!size.run()) { + return nullopt; + } + + return size.get<optional<int64_t>>(0); +} + +bool OfflineDatabase::putTile(const Resource::TileData& tile, + const Response& response, + const std::string& data, + bool compressed) { + if (response.notModified) { + // clang-format off + mapbox::sqlite::Query notModifiedQuery{ getStatement( + "UPDATE tiles " + "SET accessed = ?1, " + " expires = ?2, " + " must_revalidate = ?3 " + "WHERE url_template = ?4 " + " AND pixel_ratio = ?5 " + " AND x = ?6 " + " AND y = ?7 " + " AND z = ?8 ") }; + // clang-format on + + notModifiedQuery.bind(1, util::now()); + notModifiedQuery.bind(2, response.expires); + notModifiedQuery.bind(3, response.mustRevalidate); + notModifiedQuery.bind(4, tile.urlTemplate); + notModifiedQuery.bind(5, tile.pixelRatio); + notModifiedQuery.bind(6, tile.x); + notModifiedQuery.bind(7, tile.y); + notModifiedQuery.bind(8, tile.z); + notModifiedQuery.run(); + return false; + } + + // We can't use REPLACE because it would change the id value. + + // clang-format off + mapbox::sqlite::Query updateQuery{ getStatement( + "UPDATE tiles " + "SET modified = ?1, " + " etag = ?2, " + " expires = ?3, " + " must_revalidate = ?4, " + " accessed = ?5, " + " data = ?6, " + " compressed = ?7 " + "WHERE url_template = ?8 " + " AND pixel_ratio = ?9 " + " AND x = ?10 " + " AND y = ?11 " + " AND z = ?12 ") }; + // clang-format on + + updateQuery.bind(1, response.modified); + updateQuery.bind(2, response.etag); + updateQuery.bind(3, response.expires); + updateQuery.bind(4, response.mustRevalidate); + updateQuery.bind(5, util::now()); + updateQuery.bind(8, tile.urlTemplate); + updateQuery.bind(9, tile.pixelRatio); + updateQuery.bind(10, tile.x); + updateQuery.bind(11, tile.y); + updateQuery.bind(12, tile.z); + + if (response.noContent) { + updateQuery.bind(6, nullptr); + updateQuery.bind(7, false); + } else { + updateQuery.bindBlob(6, data.data(), data.size(), false); + updateQuery.bind(7, compressed); + } + + updateQuery.run(); + if (updateQuery.changes() != 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") }; + // clang-format on + + insertQuery.bind(1, tile.urlTemplate); + insertQuery.bind(2, tile.pixelRatio); + insertQuery.bind(3, tile.x); + insertQuery.bind(4, tile.y); + insertQuery.bind(5, tile.z); + insertQuery.bind(6, response.modified); + insertQuery.bind(7, response.mustRevalidate); + insertQuery.bind(8, response.etag); + insertQuery.bind(9, response.expires); + insertQuery.bind(10, util::now()); + + if (response.noContent) { + insertQuery.bind(11, nullptr); + insertQuery.bind(12, false); + } else { + insertQuery.bindBlob(11, data.data(), data.size(), false); + insertQuery.bind(12, compressed); + } + + insertQuery.run(); + + return true; +} + +expected<OfflineRegions, std::exception_ptr> OfflineDatabase::listRegions() try { + mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") }; + OfflineRegions result; + while (query.run()) { + const auto id = query.get<int64_t>(0); + const auto definition = query.get<std::string>(1); + const auto description = query.get<std::vector<uint8_t>>(2); + try { + // Construct, then move because this constructor is private. + OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description); + result.emplace_back(std::move(region)); + } catch (const std::exception& ex) { + // Catch errors from malformed offline region definitions + // and skip them. + Log::Error(Event::General, "%s", ex.what()); + } + } + // Explicit move to avoid triggering the copy constructor. + return { std::move(result) }; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "list regions"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +expected<OfflineRegion, std::exception_ptr> +OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "INSERT INTO regions (definition, description) " + "VALUES (?1, ?2) ") }; + // clang-format on + + query.bind(1, encodeOfflineRegionDefinition(definition)); + query.bindBlob(2, metadata); + query.run(); + return OfflineRegion(query.lastInsertRowId(), definition, metadata); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "create region"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +expected<OfflineRegions, std::exception_ptr> +OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) { + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") }; + // clang-format on + + query.bind(1, sideDatabasePath); + query.run(); + } catch (const mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, static_cast<int>(ex.code), "Can't attach database (%s) for merge: %s", sideDatabasePath.c_str(), ex.what()); + return unexpected<std::exception_ptr>(std::current_exception()); + } + try { + // Support sideloaded databases at user_version = 6. Future schema version + // changes will need to implement migration paths for sideloaded databases at + // version 6. + auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version")); + const auto mainUserVersion = getPragma<int64_t>("PRAGMA user_version"); + if (sideUserVersion < 6 || sideUserVersion != mainUserVersion) { + throw std::runtime_error("Merge database has incorrect user_version"); + } + + auto currentTileCount = getOfflineMapboxTileCount(); + // clang-format off + mapbox::sqlite::Query queryTiles{ getStatement( + "SELECT COUNT(DISTINCT st.id) " + "FROM side.tiles st " + //only consider region tiles, and not ambient tiles. + "JOIN side.region_tiles srt ON srt.tile_id = st.id " + "LEFT JOIN tiles t ON st.url_template = t.url_template AND " + "st.pixel_ratio = t.pixel_ratio AND " + "st.z = t.z AND " + "st.x = t.x AND " + "st.y = t.y " + "WHERE t.id IS NULL " + "AND st.url_template LIKE 'mapbox://%' ") }; + // clang-format on + queryTiles.run(); + auto countOfTilesToMerge = queryTiles.get<int64_t>(0); + if ((countOfTilesToMerge + currentTileCount) > offlineMapboxTileCountLimit) { + throw MapboxTileLimitExceededException(); + } + queryTiles.reset(); + + mapbox::sqlite::Transaction transaction(*db); + db->exec(mergeSideloadedDatabaseSQL); + transaction.commit(); + + // clang-format off + mapbox::sqlite::Query queryRegions{ getStatement( + "SELECT DISTINCT r.id, r.definition, r.description " + "FROM side.regions sr " + "JOIN regions r ON sr.definition = r.definition AND sr.description IS r.description") }; + // clang-format on + + OfflineRegions result; + while (queryRegions.run()) { + // Construct, then move because this constructor is private. + OfflineRegion region(queryRegions.get<int64_t>(0), + decodeOfflineRegionDefinition(queryRegions.get<std::string>(1)), + queryRegions.get<std::vector<uint8_t>>(2)); + result.emplace_back(std::move(region)); + } + db->exec("DETACH DATABASE side"); + // Explicit move to avoid triggering the copy constructor. + return { std::move(result) }; + } catch (const std::runtime_error& ex) { + db->exec("DETACH DATABASE side"); + Log::Error(Event::Database, "%s", ex.what()); + + return unexpected<std::exception_ptr>(std::current_exception()); + } + return {}; +} + +expected<OfflineRegionMetadata, std::exception_ptr> +OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "UPDATE regions SET description = ?1 " + "WHERE id = ?2") }; + // clang-format on + query.bindBlob(1, metadata); + query.bind(2, regionID); + query.run(); + + return metadata; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "update region metadata"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try { + { + mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") }; + query.bind(1, region.getID()); + query.run(); + } + + evict(0); + assert(db); + db->exec("PRAGMA incremental_vacuum"); + + // Ensure that the cached offlineTileCount value is recalculated. + offlineMapboxTileCount = {}; + return nullptr; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "delete region"); + return std::current_exception(); +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try { + auto response = getInternal(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read region resource"); + return nullopt; +} + +optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try { + auto response = hasInternal(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "query region resource"); + return nullopt; +} + +uint64_t OfflineDatabase::putRegionResource(int64_t regionID, + const Resource& resource, + const Response& response) try { + if (!db) { + initialize(); + } + mapbox::sqlite::Transaction transaction(*db); + auto size = putRegionResourceInternal(regionID, resource, response); + transaction.commit(); + return size; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resource"); + return 0; +} + +void OfflineDatabase::putRegionResources(int64_t regionID, + const std::list<std::tuple<Resource, Response>>& resources, + OfflineRegionStatus& status) try { + if (!db) { + initialize(); + } + mapbox::sqlite::Transaction transaction(*db); + + // Accumulate all statistics locally first before adding them to the OfflineRegionStatus object + // to ensure correctness when the transaction fails. + uint64_t completedResourceCount = 0; + uint64_t completedResourceSize = 0; + uint64_t completedTileCount = 0; + uint64_t completedTileSize = 0; + + for (const auto& elem : resources) { + const auto& resource = std::get<0>(elem); + const auto& response = std::get<1>(elem); + + try { + uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response); + completedResourceCount++; + completedResourceSize += resourceSize; + if (resource.kind == Resource::Kind::Tile) { + completedTileCount += 1; + completedTileSize += resourceSize; + } + } catch (const MapboxTileLimitExceededException&) { + // Commit the rest of the batch and rethrow + transaction.commit(); + throw; + } + } + + // Commit the completed batch + transaction.commit(); + + status.completedResourceCount += completedResourceCount; + status.completedResourceSize += completedResourceSize; + status.completedTileCount += completedTileCount; + status.completedTileSize += completedTileSize; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resources"); +} + +uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) { + if (exceedsOfflineMapboxTileCountLimit(resource)) { + throw MapboxTileLimitExceededException(); + } + + uint64_t size = putInternal(resource, response, false).second; + bool previouslyUnused = markUsed(regionID, resource); + + if (offlineMapboxTileCount + && resource.kind == Resource::Kind::Tile + && util::mapbox::isMapboxURL(resource.url) + && previouslyUnused) { + *offlineMapboxTileCount += 1; + } + + return size; +} + +bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) " + "SELECT ?1, tiles.id " + "FROM tiles " + "WHERE url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 ") }; + // clang-format on + + const Resource::TileData& tile = *resource.tileData; + insertQuery.bind(1, regionID); + insertQuery.bind(2, tile.urlTemplate); + insertQuery.bind(3, tile.pixelRatio); + insertQuery.bind(4, tile.x); + insertQuery.bind(5, tile.y); + insertQuery.bind(6, tile.z); + insertQuery.run(); + + if (insertQuery.changes() == 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query selectQuery{ getStatement( + "SELECT region_id " + "FROM region_tiles, tiles " + "WHERE region_id != ?1 " + " AND url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 " + "LIMIT 1 ") }; + // clang-format on + + selectQuery.bind(1, regionID); + selectQuery.bind(2, tile.urlTemplate); + selectQuery.bind(3, tile.pixelRatio); + selectQuery.bind(4, tile.x); + selectQuery.bind(5, tile.y); + selectQuery.bind(6, tile.z); + return !selectQuery.run(); + } else { + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT OR IGNORE INTO region_resources (region_id, resource_id) " + "SELECT ?1, resources.id " + "FROM resources " + "WHERE resources.url = ?2 ") }; + // clang-format on + + insertQuery.bind(1, regionID); + insertQuery.bind(2, resource.url); + insertQuery.run(); + + if (insertQuery.changes() == 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query selectQuery{ getStatement( + "SELECT region_id " + "FROM region_resources, resources " + "WHERE region_id != ?1 " + " AND resources.url = ?2 " + "LIMIT 1 ") }; + // clang-format on + + selectQuery.bind(1, regionID); + selectQuery.bind(2, resource.url); + return !selectQuery.run(); + } +} + +expected<OfflineRegionDefinition, std::exception_ptr> OfflineDatabase::getRegionDefinition(int64_t regionID) try { + mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") }; + query.bind(1, regionID); + query.run(); + + return decodeOfflineRegionDefinition(query.get<std::string>(0)); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "load region"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +expected<OfflineRegionStatus, std::exception_ptr> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try { + OfflineRegionStatus result; + + std::tie(result.completedResourceCount, result.completedResourceSize) + = getCompletedResourceCountAndSize(regionID); + std::tie(result.completedTileCount, result.completedTileSize) + = getCompletedTileCountAndSize(regionID); + + result.completedResourceCount += result.completedTileCount; + result.completedResourceSize += result.completedTileSize; + + return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get region status"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(*), SUM(LENGTH(data)) " + "FROM region_resources, resources " + "WHERE region_id = ?1 " + "AND resource_id = resources.id ") }; + // clang-format on + query.bind(1, regionID); + query.run(); + return { query.get<int64_t>(0), query.get<int64_t>(1) }; +} + +std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(*), SUM(LENGTH(data)) " + "FROM region_tiles, tiles " + "WHERE region_id = ?1 " + "AND tile_id = tiles.id ") }; + // clang-format on + query.bind(1, regionID); + query.run(); + return { query.get<int64_t>(0), query.get<int64_t>(1) }; +} + +template <class T> +T OfflineDatabase::getPragma(const char* sql) { + mapbox::sqlite::Query query{ getStatement(sql) }; + query.run(); + return query.get<T>(0); +} + +// Remove least-recently used resources and tiles until the used database size, +// as calculated by multiplying the number of in-use pages by the page size, is +// less than the maximum cache size. Returns false if this condition cannot be +// satisfied. +// +// SQLite database never shrinks in size unless we call VACCUM. We here +// are monitoring the soft limit (i.e. number of free pages in the file) +// and as it approaches to the hard limit (i.e. the actual file size) we +// delete an arbitrary number of old cache entries. The free pages approach saves +// us from calling VACCUM or keeping a running total, which can be costly. +bool OfflineDatabase::evict(uint64_t neededFreeSize) { + uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size"); + uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count"); + + auto usedSize = [&] { + return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count")); + }; + + // The addition of pageSize is a fudge factor to account for non `data` column + // size, and because pages can get fragmented on the database. + while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) { + // clang-format off + mapbox::sqlite::Query accessedQuery{ getStatement( + "SELECT max(accessed) " + "FROM ( " + " SELECT accessed " + " FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " UNION ALL " + " SELECT accessed " + " FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " ORDER BY accessed ASC LIMIT ?1 " + ") " + ) }; + accessedQuery.bind(1, 50); + // clang-format on + if (!accessedQuery.run()) { + return false; + } + Timestamp accessed = accessedQuery.get<Timestamp>(0); + + // clang-format off + mapbox::sqlite::Query resourceQuery{ getStatement( + "DELETE FROM resources " + "WHERE id IN ( " + " SELECT id FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " AND accessed <= ?1 " + ") ") }; + // clang-format on + resourceQuery.bind(1, accessed); + resourceQuery.run(); + const uint64_t resourceChanges = resourceQuery.changes(); + + // clang-format off + mapbox::sqlite::Query tileQuery{ getStatement( + "DELETE FROM tiles " + "WHERE id IN ( " + " SELECT id FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " AND accessed <= ?1 " + ") ") }; + // clang-format on + tileQuery.bind(1, accessed); + tileQuery.run(); + const uint64_t tileChanges = tileQuery.changes(); + + // The cached value of offlineTileCount does not need to be updated + // here because only non-offline tiles can be removed by eviction. + + if (resourceChanges == 0 && tileChanges == 0) { + return false; + } + } + + return true; +} + +void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) { + offlineMapboxTileCountLimit = limit; +} + +uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() { + return offlineMapboxTileCountLimit; +} + +bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() { + return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit; +} + +uint64_t OfflineDatabase::getOfflineMapboxTileCount() try { + // Calculating this on every call would be much simpler than caching and + // manually updating the value, but it would make offline downloads an O(n²) + // operation, because the database query below involves an index scan of + // region_tiles. + + if (offlineMapboxTileCount) { + return *offlineMapboxTileCount; + } + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(DISTINCT id) " + "FROM region_tiles, tiles " + "WHERE tile_id = tiles.id " + "AND url_template LIKE 'mapbox://%' ") }; + // clang-format on + + query.run(); + + offlineMapboxTileCount = query.get<int64_t>(0); + return *offlineMapboxTileCount; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get offline Mapbox tile count"); + return std::numeric_limits<uint64_t>::max(); +} + +bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) { + return resource.kind == Resource::Kind::Tile + && util::mapbox::isMapboxURL(resource.url) + && offlineMapboxTileCountLimitExceeded(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline_download.cpp b/platform/default/src/mbgl/storage/offline_download.cpp new file mode 100644 index 0000000000..c97797a5a2 --- /dev/null +++ b/platform/default/src/mbgl/storage/offline_download.cpp @@ -0,0 +1,453 @@ +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/storage/http_file_source.hpp> +#include <mbgl/style/parser.hpp> +#include <mbgl/style/sources/vector_source.hpp> +#include <mbgl/style/sources/raster_source.hpp> +#include <mbgl/style/sources/raster_dem_source.hpp> +#include <mbgl/style/sources/geojson_source.hpp> +#include <mbgl/style/sources/image_source.hpp> +#include <mbgl/style/conversion/json.hpp> +#include <mbgl/style/conversion/tileset.hpp> +#include <mbgl/text/glyph.hpp> +#include <mbgl/util/mapbox.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/tile_cover.hpp> +#include <mbgl/util/tileset.hpp> + +#include <set> + +namespace mbgl { + +using namespace style; + +// Generic functions + +template <class RegionDefinition> +Range<uint8_t> coveringZoomRange(const RegionDefinition& definition, + style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) { + double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min); + double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max); + + assert(minZ >= 0); + assert(maxZ >= 0); + assert(minZ < std::numeric_limits<uint8_t>::max()); + assert(maxZ < std::numeric_limits<uint8_t>::max()); + return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) }; +} + +template <class Geometry, class Fn> +void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) { + util::TileCover cover(geometry, z); + while (cover.hasNext()) { + fn(cover.next()->canonical); + } +} + + +template <class Fn> +void tileCover(const OfflineRegionDefinition& definition, style::SourceType type, + uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) { + const Range<uint8_t> clampedZoomRange = + definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }); + + for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { + definition.match( + [&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); }, + [&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); } + ); + } +} + +uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type, + uint16_t tileSize, const Range<uint8_t>& zoomRange) { + + const Range<uint8_t> clampedZoomRange = + definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }); + + unsigned long result = 0;; + for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { + result += definition.match( + [&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); }, + [&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); } + ); + } + + return result; +} + +// OfflineDownload + +OfflineDownload::OfflineDownload(int64_t id_, + OfflineRegionDefinition&& definition_, + OfflineDatabase& offlineDatabase_, + OnlineFileSource& onlineFileSource_) + : id(id_), + definition(definition_), + offlineDatabase(offlineDatabase_), + onlineFileSource(onlineFileSource_) { + setObserver(nullptr); +} + +OfflineDownload::~OfflineDownload() = default; + +void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) { + observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>(); +} + +void OfflineDownload::setState(OfflineRegionDownloadState state) { + if (status.downloadState == state) { + return; + } + + status.downloadState = state; + + if (status.downloadState == OfflineRegionDownloadState::Active) { + activateDownload(); + } else { + deactivateDownload(); + } + + observer->statusChanged(status); +} + +OfflineRegionStatus OfflineDownload::getStatus() const { + if (status.downloadState == OfflineRegionDownloadState::Active) { + return status; + } + + auto result = offlineDatabase.getRegionCompletedStatus(id); + if (!result) { + // We can't find this offline region because the database is unavailable, or the download + // does not exist. + return {}; + } + + result->requiredResourceCount++; + optional<Response> styleResponse = + offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }))); + if (!styleResponse) { + return *result; + } + + style::Parser parser; + parser.parse(*styleResponse->data); + + result->requiredResourceCountIsPrecise = true; + + for (const auto& source : parser.sources) { + SourceType type = source->getType(); + + auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) { + if (urlOrTileset.is<Tileset>()) { + result->requiredResourceCount += + tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange); + } else { + result->requiredResourceCount += 1; + const auto& url = urlOrTileset.get<std::string>(); + optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url)); + if (sourceResponse) { + style::conversion::Error error; + optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error); + if (tileset) { + result->requiredResourceCount += + tileCount(definition, type, tileSize, (*tileset).zoomRange); + } + } else { + result->requiredResourceCountIsPrecise = false; + } + } + }; + + switch (type) { + case SourceType::Vector: { + const auto& vectorSource = *source->as<VectorSource>(); + handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize); + break; + } + + case SourceType::Raster: { + const auto& rasterSource = *source->as<RasterSource>(); + handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize()); + break; + } + + case SourceType::RasterDEM: { + const auto& rasterDEMSource = *source->as<RasterDEMSource>(); + handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize()); + break; + } + + case SourceType::GeoJSON: { + const auto& geojsonSource = *source->as<GeoJSONSource>(); + if (geojsonSource.getURL()) { + result->requiredResourceCount += 1; + } + break; + } + + case SourceType::Image: { + const auto& imageSource = *source->as<ImageSource>(); + if (imageSource.getURL()) { + result->requiredResourceCount += 1; + } + break; + } + + case SourceType::Video: + case SourceType::Annotations: + case SourceType::CustomVector: + break; + } + } + + if (!parser.glyphURL.empty()) { + result->requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK; + } + + if (!parser.spriteURL.empty()) { + result->requiredResourceCount += 4; + } + + return *result; +} + +void OfflineDownload::activateDownload() { + status = OfflineRegionStatus(); + status.downloadState = OfflineRegionDownloadState::Active; + status.requiredResourceCount++; + ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }), Resource::Priority::Low), + [&](Response styleResponse) { + status.requiredResourceCountIsPrecise = true; + + style::Parser parser; + parser.parse(*styleResponse.data); + + for (const auto& source : parser.sources) { + SourceType type = source->getType(); + + auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) { + if (urlOrTileset.is<Tileset>()) { + queueTiles(type, tileSize, urlOrTileset.get<Tileset>()); + } else { + const auto& url = urlOrTileset.get<std::string>(); + status.requiredResourceCountIsPrecise = false; + status.requiredResourceCount++; + requiredSourceURLs.insert(url); + + ensureResource(Resource::source(url, Resource::Priority::Low), [=](Response sourceResponse) { + style::conversion::Error error; + optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error); + if (tileset) { + util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize); + queueTiles(type, tileSize, *tileset); + + requiredSourceURLs.erase(url); + if (requiredSourceURLs.empty()) { + status.requiredResourceCountIsPrecise = true; + } + } + }); + } + }; + + switch (type) { + case SourceType::Vector: { + const auto& vectorSource = *source->as<VectorSource>(); + handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize); + break; + } + + case SourceType::Raster: { + const auto& rasterSource = *source->as<RasterSource>(); + handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize()); + break; + } + + case SourceType::RasterDEM: { + const auto& rasterDEMSource = *source->as<RasterDEMSource>(); + handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize()); + break; + } + + case SourceType::GeoJSON: { + const auto& geojsonSource = *source->as<GeoJSONSource>(); + if (geojsonSource.getURL()) { + queueResource(Resource::source(*geojsonSource.getURL())); + } + break; + } + + case SourceType::Image: { + const auto& imageSource = *source->as<ImageSource>(); + auto imageUrl = imageSource.getURL(); + if (imageUrl && !imageUrl->empty()) { + queueResource(Resource::image(*imageUrl)); + } + break; + } + + case SourceType::Video: + case SourceType::Annotations: + case SourceType::CustomVector: + break; + } + } + + if (!parser.glyphURL.empty()) { + for (const auto& fontStack : parser.fontStacks()) { + for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) { + queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE))); + } + } + } + + if (!parser.spriteURL.empty()) { + // Always request 1x and @2x sprite images for portability. + queueResource(Resource::spriteImage(parser.spriteURL, 1)); + queueResource(Resource::spriteImage(parser.spriteURL, 2)); + queueResource(Resource::spriteJSON(parser.spriteURL, 1)); + queueResource(Resource::spriteJSON(parser.spriteURL, 2)); + } + + continueDownload(); + }); +} + +/* + Fill up our own request queue by requesting the next few resources. This is called + when activating the download, or when a request completes successfully. + + Note "successfully"; it's not called when a requests receives an error. A request + that errors will be retried after some delay. So in that sense it's still "active" + and consuming resources, notably the request object, its timer, and network resources + when the timer fires. + + We could try to squeeze in subsequent requests while we wait for the errored request + to retry. But that risks overloading the upstream request queue -- defeating our own + metering -- if there are a lot of errored requests that all come up for retry at the + same time. And many times, the cause of a request error will apply to many requests + of the same type. For instance if a server is unreachable, all the requests to that + host are going to error. In that case, continuing to try subsequent resources after + the first few errors is fruitless anyway. +*/ +void OfflineDownload::continueDownload() { + if (resourcesRemaining.empty() && status.complete()) { + setState(OfflineRegionDownloadState::Inactive); + return; + } + + while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) { + ensureResource(resourcesRemaining.front()); + resourcesRemaining.pop_front(); + } +} + +void OfflineDownload::deactivateDownload() { + requiredSourceURLs.clear(); + resourcesRemaining.clear(); + requests.clear(); +} + +void OfflineDownload::queueResource(Resource resource) { + resource.setPriority(Resource::Priority::Low); + status.requiredResourceCount++; + resourcesRemaining.push_front(std::move(resource)); +} + +void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) { + tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) { + status.requiredResourceCount++; + resourcesRemaining.push_back(Resource::tile( + tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x, + tile.y, tile.z, tileset.scheme, Resource::Priority::Low)); + }); +} + +void OfflineDownload::ensureResource(const Resource& resource, + std::function<void(Response)> callback) { + assert(resource.priority == Resource::Priority::Low); + + auto workRequestsIt = requests.insert(requests.begin(), nullptr); + *workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() { + requests.erase(workRequestsIt); + + auto getResourceSizeInDatabase = [&] () -> optional<int64_t> { + if (!callback) { + return offlineDatabase.hasRegionResource(id, resource); + } + optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource); + if (!response) { + return {}; + } + callback(response->first); + return response->second; + }; + + optional<int64_t> offlineResponse = getResourceSizeInDatabase(); + if (offlineResponse) { + status.completedResourceCount++; + status.completedResourceSize += *offlineResponse; + if (resource.kind == Resource::Kind::Tile) { + status.completedTileCount += 1; + status.completedTileSize += *offlineResponse; + } + + observer->statusChanged(status); + continueDownload(); + return; + } + + if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) { + onMapboxTileCountLimitExceeded(); + return; + } + + auto fileRequestsIt = requests.insert(requests.begin(), nullptr); + *fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) { + if (onlineResponse.error) { + observer->responseError(*onlineResponse.error); + return; + } + + requests.erase(fileRequestsIt); + + if (callback) { + callback(onlineResponse); + } + + // Queue up for batched insertion + buffer.emplace_back(resource, onlineResponse); + + // Flush buffer periodically + if (buffer.size() == 64 || resourcesRemaining.size() == 0) { + try { + offlineDatabase.putRegionResources(id, buffer, status); + } catch (const MapboxTileLimitExceededException&) { + onMapboxTileCountLimitExceeded(); + return; + } + + buffer.clear(); + observer->statusChanged(status); + } + + if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) { + onMapboxTileCountLimitExceeded(); + return; + } + + continueDownload(); + }); + }); +} + +void OfflineDownload::onMapboxTileCountLimitExceeded() { + observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit()); + setState(OfflineRegionDownloadState::Inactive); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/online_file_source.cpp b/platform/default/src/mbgl/storage/online_file_source.cpp new file mode 100644 index 0000000000..fce1c3e2b6 --- /dev/null +++ b/platform/default/src/mbgl/storage/online_file_source.cpp @@ -0,0 +1,488 @@ +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/http_file_source.hpp> +#include <mbgl/storage/network_status.hpp> + +#include <mbgl/storage/resource_transform.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/logging.hpp> + +#include <mbgl/actor/mailbox.hpp> +#include <mbgl/util/constants.hpp> +#include <mbgl/util/mapbox.hpp> +#include <mbgl/util/exception.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/async_task.hpp> +#include <mbgl/util/noncopyable.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/timer.hpp> +#include <mbgl/util/http_timeout.hpp> + +#include <algorithm> +#include <cassert> +#include <list> +#include <unordered_set> +#include <unordered_map> + +namespace mbgl { + +static uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; + +class OnlineFileRequest : public AsyncRequest { +public: + using Callback = std::function<void (Response)>; + + OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&); + ~OnlineFileRequest() override; + + void networkIsReachableAgain(); + void schedule(); + void schedule(optional<Timestamp> expires); + void completed(Response); + + void setTransformedURL(const std::string&& url); + ActorRef<OnlineFileRequest> actor(); + + OnlineFileSource::Impl& impl; + Resource resource; + std::unique_ptr<AsyncRequest> request; + util::Timer timer; + Callback callback; + + std::shared_ptr<Mailbox> mailbox; + + // Counts the number of times a response was already expired when received. We're using + // this to add a delay when making a new request so we don't keep retrying immediately + // in case of a server serving expired tiles. + uint32_t expiredRequests = 0; + + // Counts the number of subsequent failed requests. We're using this value for exponential + // backoff when retrying requests. + uint32_t failedRequests = 0; + Response::Error::Reason failedRequestReason = Response::Error::Reason::Success; + optional<Timestamp> retryAfter; +}; + +class OnlineFileSource::Impl { +public: + Impl() { + NetworkStatus::Subscribe(&reachability); + setMaximumConcurrentRequests(DEFAULT_MAXIMUM_CONCURRENT_REQUESTS); + } + + ~Impl() { + NetworkStatus::Unsubscribe(&reachability); + } + + void add(OnlineFileRequest* request) { + allRequests.insert(request); + if (resourceTransform) { + // Request the ResourceTransform actor a new url and replace the resource url with the + // transformed one before proceeding to schedule the request. + resourceTransform->invoke(&ResourceTransform::transform, request->resource.kind, + std::move(request->resource.url), [ref = request->actor()](const std::string&& url) mutable { + ref.invoke(&OnlineFileRequest::setTransformedURL, std::move(url)); + }); + } else { + request->schedule(); + } + } + + void remove(OnlineFileRequest* request) { + allRequests.erase(request); + if (activeRequests.erase(request)) { + activatePendingRequest(); + } else { + pendingRequests.remove(request); + } + } + + void activateOrQueueRequest(OnlineFileRequest* request) { + assert(allRequests.find(request) != allRequests.end()); + assert(activeRequests.find(request) == activeRequests.end()); + assert(!request->request); + + if (activeRequests.size() >= getMaximumConcurrentRequests()) { + queueRequest(request); + } else { + activateRequest(request); + } + } + + void queueRequest(OnlineFileRequest* request) { + pendingRequests.insert(request); + } + + void activateRequest(OnlineFileRequest* request) { + auto callback = [=](Response response) { + activeRequests.erase(request); + request->request.reset(); + request->completed(response); + activatePendingRequest(); + }; + + activeRequests.insert(request); + + if (online) { + request->request = httpFileSource.request(request->resource, callback); + } else { + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection, + "Online connectivity is disabled."); + callback(response); + } + + } + + void activatePendingRequest() { + + auto request = pendingRequests.pop(); + + if (request) { + activateRequest(*request); + } + } + + bool isPending(OnlineFileRequest* request) { + return pendingRequests.contains(request); + } + + bool isActive(OnlineFileRequest* request) { + return activeRequests.find(request) != activeRequests.end(); + } + + void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + resourceTransform = std::move(transform); + } + + void setOnlineStatus(const bool status) { + online = status; + networkIsReachableAgain(); + } + + uint32_t getMaximumConcurrentRequests() const { + return maximumConcurrentRequests; + } + + void setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + maximumConcurrentRequests = maximumConcurrentRequests_; + } + +private: + + void networkIsReachableAgain() { + for (auto& request : allRequests) { + request->networkIsReachableAgain(); + } + } + + // Using Pending Requests as an priority queue which processes + // file requests in a FIFO manner but prefers regular requests + // over offline requests with a low priority such that low priority + // requests do not throttle regular requests. + // + // The order of a queue is therefore: + // + // hi0 -- hi1 -- hi2 -- hi3 -- lo0 -- lo1 --lo2 + // ^ + // firstLowPriorityRequest + + struct PendingRequests { + PendingRequests() : queue(), firstLowPriorityRequest(queue.begin()) {} + + std::list<OnlineFileRequest*> queue; + std::list<OnlineFileRequest*>::iterator firstLowPriorityRequest; + + void remove(const OnlineFileRequest* request) { + auto it = std::find(queue.begin(), queue.end(), request); + if (it != queue.end()) { + if (it == firstLowPriorityRequest) { + firstLowPriorityRequest++; + } + queue.erase(it); + } + } + + void insert(OnlineFileRequest* request) { + if (request->resource.priority == Resource::Priority::Regular) { + firstLowPriorityRequest = queue.insert(firstLowPriorityRequest, request); + firstLowPriorityRequest++; + } + else { + if (firstLowPriorityRequest == queue.end()) { + firstLowPriorityRequest = queue.insert(queue.end(), request); + } + else { + queue.insert(queue.end(), request); + } + } + } + + + optional<OnlineFileRequest*> pop() { + if (queue.empty()) { + return optional<OnlineFileRequest*>(); + } + + if (queue.begin() == firstLowPriorityRequest) { + firstLowPriorityRequest++; + } + + OnlineFileRequest* next = queue.front(); + queue.pop_front(); + return optional<OnlineFileRequest*>(next); + } + + bool contains(OnlineFileRequest* request) const { + return (std::find(queue.begin(), queue.end(), request) != queue.end()); + } + + }; + + optional<ActorRef<ResourceTransform>> resourceTransform; + + /** + * The lifetime of a request is: + * + * 1. Waiting for timeout (revalidation or retry) + * 2. Pending (waiting for room in the active set) + * 3. Active (open network connection) + * 4. Back to #1 + * + * Requests in any state are in `allRequests`. Requests in the pending state are in + * `pendingRequests`. Requests in the active state are in `activeRequests`. + */ + std::unordered_set<OnlineFileRequest*> allRequests; + + PendingRequests pendingRequests; + + std::unordered_set<OnlineFileRequest*> activeRequests; + + bool online = true; + uint32_t maximumConcurrentRequests; + HTTPFileSource httpFileSource; + util::AsyncTask reachability { std::bind(&Impl::networkIsReachableAgain, this) }; +}; + +OnlineFileSource::OnlineFileSource() + : impl(std::make_unique<Impl>()) { +} + +OnlineFileSource::~OnlineFileSource() = default; + +std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) { + Resource res = resource; + + switch (resource.kind) { + case Resource::Kind::Unknown: + case Resource::Kind::Image: + break; + + case Resource::Kind::Style: + res.url = mbgl::util::mapbox::normalizeStyleURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::Source: + res.url = util::mapbox::normalizeSourceURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::Glyphs: + res.url = util::mapbox::normalizeGlyphsURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::SpriteImage: + case Resource::Kind::SpriteJSON: + res.url = util::mapbox::normalizeSpriteURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::Tile: + res.url = util::mapbox::normalizeTileURL(apiBaseURL, resource.url, accessToken); + break; + } + + return std::make_unique<OnlineFileRequest>(std::move(res), std::move(callback), *impl); +} + +void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + impl->setResourceTransform(std::move(transform)); +} + +OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_) + : impl(impl_), + resource(std::move(resource_)), + callback(std::move(callback_)) { + impl.add(this); +} + +void OnlineFileRequest::schedule() { + // Force an immediate first request if we don't have an expiration time. + if (resource.priorExpires) { + schedule(resource.priorExpires); + } else { + schedule(util::now()); + } +} + +OnlineFileRequest::~OnlineFileRequest() { + impl.remove(this); +} + +Timestamp interpolateExpiration(const Timestamp& current, + optional<Timestamp> prior, + bool& expired) { + auto now = util::now(); + if (current > now) { + return current; + } + + if (!bool(prior)) { + expired = true; + return current; + } + + // Expiring date is going backwards, + // fallback to exponential backoff. + if (current < *prior) { + expired = true; + return current; + } + + auto delta = current - *prior; + + // Server is serving the same expired resource + // over and over, fallback to exponential backoff. + if (delta == Duration::zero()) { + expired = true; + return current; + } + + // Assume that either the client or server clock is wrong and + // try to interpolate a valid expiration date (from the client POV) + // observing a minimum timeout. + return now + std::max<Seconds>(delta, util::CLOCK_SKEW_RETRY_TIMEOUT); +} + +void OnlineFileRequest::schedule(optional<Timestamp> expires) { + if (impl.isPending(this) || impl.isActive(this)) { + // There's already a request in progress; don't start another one. + return; + } + + // If we're not being asked for a forced refresh, calculate a timeout that depends on how many + // consecutive errors we've encountered, and on the expiration time, if present. + Duration timeout = std::min( + http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), + http::expirationTimeout(expires, expiredRequests)); + + if (timeout == Duration::max()) { + return; + } + + // Emulate a Connection error when the Offline mode is forced with + // a really long timeout. The request will get re-triggered when + // the NetworkStatus is set back to Online. + if (NetworkStatus::Get() == NetworkStatus::Status::Offline) { + failedRequestReason = Response::Error::Reason::Connection; + failedRequests = 1; + timeout = Duration::max(); + } + + timer.start(timeout, Duration::zero(), [&] { + impl.activateOrQueueRequest(this); + }); +} + +void OnlineFileRequest::completed(Response response) { + // If we didn't get various caching headers in the response, continue using the + // previous values. Otherwise, update the previous values to the new values. + + if (!response.modified) { + response.modified = resource.priorModified; + } else { + resource.priorModified = response.modified; + } + + if (response.notModified && resource.priorData) { + // When the priorData field is set, it indicates that we had to revalidate the request and + // that the requestor hasn't gotten data yet. If we get a 304 response, this means that we + // have send the cached data to give the requestor a chance to actually obtain the data. + response.data = std::move(resource.priorData); + response.notModified = false; + } + + bool isExpired = false; + + if (response.expires) { + auto prior = resource.priorExpires; + resource.priorExpires = response.expires; + response.expires = interpolateExpiration(*response.expires, prior, isExpired); + } + + if (isExpired) { + expiredRequests++; + } else { + expiredRequests = 0; + } + + if (!response.etag) { + response.etag = resource.priorEtag; + } else { + resource.priorEtag = response.etag; + } + + if (response.error) { + failedRequests++; + failedRequestReason = response.error->reason; + retryAfter = response.error->retryAfter; + } else { + failedRequests = 0; + failedRequestReason = Response::Error::Reason::Success; + } + + schedule(response.expires); + + // Calling the callback may result in `this` being deleted. It needs to be done last, + // and needs to make a local copy of the callback to ensure that it remains valid for + // the duration of the call. + auto callback_ = callback; + callback_(response); +} + +void OnlineFileRequest::networkIsReachableAgain() { + // We need all requests to fail at least once before we are going to start retrying + // them, and we only immediately restart request that failed due to connection issues. + if (failedRequestReason == Response::Error::Reason::Connection) { + schedule(util::now()); + } +} + +void OnlineFileRequest::setTransformedURL(const std::string&& url) { + resource.url = std::move(url); + schedule(); +} + +ActorRef<OnlineFileRequest> OnlineFileRequest::actor() { + if (!mailbox) { + // Lazy constructed because this can be costly and + // the ResourceTransform is not used by many apps. + mailbox = std::make_shared<Mailbox>(*Scheduler::GetCurrent()); + } + + return ActorRef<OnlineFileRequest>(*this, mailbox); +} + +void OnlineFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + impl->setMaximumConcurrentRequests(maximumConcurrentRequests_); +} + +uint32_t OnlineFileSource::getMaximumConcurrentRequests() const { + return impl->getMaximumConcurrentRequests(); +} + + +// For testing only: + +void OnlineFileSource::setOnlineStatus(const bool status) { + impl->setOnlineStatus(status); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/sqlite3.cpp b/platform/default/src/mbgl/storage/sqlite3.cpp new file mode 100644 index 0000000000..0017dc45db --- /dev/null +++ b/platform/default/src/mbgl/storage/sqlite3.cpp @@ -0,0 +1,494 @@ +#include <mbgl/storage/sqlite3.hpp> +#include <sqlite3.h> + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <cstdio> +#include <chrono> +#include <experimental/optional> + +#include <mbgl/util/traits.hpp> +#include <mbgl/util/logging.hpp> + +namespace mapbox { +namespace sqlite { + +static_assert(mbgl::underlying_type(ResultCode::OK) == SQLITE_OK, "error"); +static_assert(mbgl::underlying_type(ResultCode::Error) == SQLITE_ERROR, "error"); +static_assert(mbgl::underlying_type(ResultCode::Internal) == SQLITE_INTERNAL, "error"); +static_assert(mbgl::underlying_type(ResultCode::Perm) == SQLITE_PERM, "error"); +static_assert(mbgl::underlying_type(ResultCode::Abort) == SQLITE_ABORT, "error"); +static_assert(mbgl::underlying_type(ResultCode::Busy) == SQLITE_BUSY, "error"); +static_assert(mbgl::underlying_type(ResultCode::Locked) == SQLITE_LOCKED, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoMem) == SQLITE_NOMEM, "error"); +static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == SQLITE_READONLY, "error"); +static_assert(mbgl::underlying_type(ResultCode::Interrupt) == SQLITE_INTERRUPT, "error"); +static_assert(mbgl::underlying_type(ResultCode::IOErr) == SQLITE_IOERR, "error"); +static_assert(mbgl::underlying_type(ResultCode::Corrupt) == SQLITE_CORRUPT, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotFound) == SQLITE_NOTFOUND, "error"); +static_assert(mbgl::underlying_type(ResultCode::Full) == SQLITE_FULL, "error"); +static_assert(mbgl::underlying_type(ResultCode::CantOpen) == SQLITE_CANTOPEN, "error"); +static_assert(mbgl::underlying_type(ResultCode::Protocol) == SQLITE_PROTOCOL, "error"); +static_assert(mbgl::underlying_type(ResultCode::Schema) == SQLITE_SCHEMA, "error"); +static_assert(mbgl::underlying_type(ResultCode::TooBig) == SQLITE_TOOBIG, "error"); +static_assert(mbgl::underlying_type(ResultCode::Constraint) == SQLITE_CONSTRAINT, "error"); +static_assert(mbgl::underlying_type(ResultCode::Mismatch) == SQLITE_MISMATCH, "error"); +static_assert(mbgl::underlying_type(ResultCode::Misuse) == SQLITE_MISUSE, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoLFS) == SQLITE_NOLFS, "error"); +static_assert(mbgl::underlying_type(ResultCode::Auth) == SQLITE_AUTH, "error"); +static_assert(mbgl::underlying_type(ResultCode::Range) == SQLITE_RANGE, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotADB) == SQLITE_NOTADB, "error"); + +void setTempPath(const std::string& path) { + sqlite3_temp_directory = sqlite3_mprintf("%s", path.c_str()); +} + +class DatabaseImpl { +public: + DatabaseImpl(sqlite3* db_) + : db(db_) + { + const int error = sqlite3_extended_result_codes(db, true); + if (error != SQLITE_OK) { + mbgl::Log::Warning(mbgl::Event::Database, error, "Failed to enable extended result codes: %s", sqlite3_errmsg(db)); + } + } + + ~DatabaseImpl() + { + const int error = sqlite3_close(db); + if (error != SQLITE_OK) { + mbgl::Log::Error(mbgl::Event::Database, error, "Failed to close database: %s", sqlite3_errmsg(db)); + } + } + + void setBusyTimeout(std::chrono::milliseconds timeout); + void exec(const std::string& sql); + + sqlite3* db; +}; + +class StatementImpl { +public: + StatementImpl(sqlite3* db, const char* sql) + { + const int error = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr); + if (error != SQLITE_OK) { + stmt = nullptr; + throw Exception { error, sqlite3_errmsg(db) }; + } + } + + ~StatementImpl() + { + if (!stmt) return; + + sqlite3_finalize(stmt); + } + + void check(int err) { + if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt)) }; + } + } + + sqlite3_stmt* stmt = nullptr; + int64_t lastInsertRowId = 0; + int64_t changes = 0; +}; + +template <typename T> +using optional = std::experimental::optional<T>; + + +#ifndef NDEBUG +void logSqlMessage(void *, const int err, const char *msg) { + mbgl::Log::Record(mbgl::EventSeverity::Debug, mbgl::Event::Database, err, "%s", msg); +} +#endif + +__attribute__((constructor)) +static void initalize() { + if (sqlite3_libversion_number() / 1000000 != SQLITE_VERSION_NUMBER / 1000000) { + char message[96]; + snprintf(message, 96, + "sqlite3 libversion mismatch: headers report %d, but library reports %d", + SQLITE_VERSION_NUMBER, sqlite3_libversion_number()); + throw std::runtime_error(message); + } + +#ifndef NDEBUG + // Enable SQLite logging before initializing the database. + sqlite3_config(SQLITE_CONFIG_LOG, &logSqlMessage, nullptr); +#endif +} + +mapbox::util::variant<Database, Exception> Database::tryOpen(const std::string &filename, int flags) { + sqlite3* db = nullptr; + const int error = sqlite3_open_v2(filename.c_str(), &db, flags | SQLITE_OPEN_URI, nullptr); + if (error != SQLITE_OK) { + const auto message = sqlite3_errmsg(db); + return Exception { error, message }; + } + return Database(std::make_unique<DatabaseImpl>(db)); +} + +Database Database::open(const std::string &filename, int flags) { + auto result = tryOpen(filename, flags); + if (result.is<Exception>()) { + throw result.get<Exception>(); + } else { + return std::move(result.get<Database>()); + } +} + +Database::Database(std::unique_ptr<DatabaseImpl> impl_) + : impl(std::move(impl_)) +{} + +Database::Database(Database &&other) + : impl(std::move(other.impl)) {} + +Database &Database::operator=(Database &&other) { + std::swap(impl, other.impl); + return *this; +} + +Database::~Database() = default; + +void Database::setBusyTimeout(std::chrono::milliseconds timeout) { + assert(impl); + impl->setBusyTimeout(timeout); +} + +void DatabaseImpl::setBusyTimeout(std::chrono::milliseconds timeout) { + const int err = sqlite3_busy_timeout(db, + int(std::min<std::chrono::milliseconds::rep>(timeout.count(), std::numeric_limits<int>::max()))); + if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(db) }; + } +} + +void Database::exec(const std::string &sql) { + assert(impl); + impl->exec(sql); +} + +void DatabaseImpl::exec(const std::string& sql) { + char *msg = nullptr; + const int err = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &msg); + if (msg) { + const std::string message = msg; + sqlite3_free(msg); + throw Exception { err, message }; + } else if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(db) }; + } +} + +Statement::Statement(Database& db, const char* sql) + : impl(std::make_unique<StatementImpl>(db.impl->db, sql)) { +} + +Statement::~Statement() { +#ifndef NDEBUG + // Crash if we're destructing this object while we know a Query object references this. + assert(!used); +#endif +} + +Query::Query(Statement& stmt_) : stmt(stmt_) { + assert(stmt.impl); + +#ifndef NDEBUG + assert(!stmt.used); + stmt.used = true; +#endif +} + +Query::~Query() { + reset(); + clearBindings(); + +#ifndef NDEBUG + stmt.used = false; +#endif +} + +template <> void Query::bind(int offset, std::nullptr_t) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_null(stmt.impl->stmt, offset)); +} + +template <> void Query::bind(int offset, int8_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, int16_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, int32_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, int64_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, uint8_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, uint16_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, uint32_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, float value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, double value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, bool value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, const char *value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, -1, SQLITE_STATIC)); +} + +// We currently cannot use sqlite3_bind_blob64 / sqlite3_bind_text64 because they +// were introduced in SQLite 3.8.7, and we need to support earlier versions: +// Android 11: 3.7 +// Android 21: 3.8 +// Android 24: 3.9 +// Per https://developer.android.com/reference/android/database/sqlite/package-summary. +// The first iOS version with 3.8.7+ was 9.0, with 3.8.8. + +void Query::bind(int offset, const char * value, std::size_t length, bool retain) { + assert(stmt.impl); + if (length > std::numeric_limits<int>::max()) { + throw std::range_error("value too long for sqlite3_bind_text"); + } + stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, int(length), + retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); +} + +void Query::bind(int offset, const std::string& value, bool retain) { + bind(offset, value.data(), value.size(), retain); +} + +void Query::bindBlob(int offset, const void * value, std::size_t length, bool retain) { + assert(stmt.impl); + if (length > std::numeric_limits<int>::max()) { + throw std::range_error("value too long for sqlite3_bind_text"); + } + stmt.impl->check(sqlite3_bind_blob(stmt.impl->stmt, offset, value, int(length), + retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); +} + +void Query::bindBlob(int offset, const std::vector<uint8_t>& value, bool retain) { + bindBlob(offset, value.data(), value.size(), retain); +} + +template <> +void Query::bind( + int offset, std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds> value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, std::chrono::system_clock::to_time_t(value))); +} + +template <> void Query::bind(int offset, optional<std::string> value) { + if (!value) { + bind(offset, nullptr); + } else { + bind(offset, *value); + } +} + +template <> +void Query::bind( + int offset, + optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>> value) { + if (!value) { + bind(offset, nullptr); + } else { + bind(offset, *value); + } +} + +bool Query::run() { + assert(stmt.impl); + const int err = sqlite3_step(stmt.impl->stmt); + stmt.impl->lastInsertRowId = sqlite3_last_insert_rowid(sqlite3_db_handle(stmt.impl->stmt)); + stmt.impl->changes = sqlite3_changes(sqlite3_db_handle(stmt.impl->stmt)); + if (err == SQLITE_DONE) { + return false; + } else if (err == SQLITE_ROW) { + return true; + } else if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt.impl->stmt)) }; + } else { + return false; + } +} + +template <> bool Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_int(stmt.impl->stmt, offset); +} + +template <> int Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_int(stmt.impl->stmt, offset); +} + +template <> int64_t Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_int64(stmt.impl->stmt, offset); +} + +template <> double Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_double(stmt.impl->stmt, offset); +} + +template <> std::string Query::get(int offset) { + assert(stmt.impl); + return { + reinterpret_cast<const char *>(sqlite3_column_blob(stmt.impl->stmt, offset)), + size_t(sqlite3_column_bytes(stmt.impl->stmt, offset)) + }; +} + +template <> std::vector<uint8_t> Query::get(int offset) { + assert(stmt.impl); + const auto* begin = reinterpret_cast<const uint8_t*>(sqlite3_column_blob(stmt.impl->stmt, offset)); + const uint8_t* end = begin + sqlite3_column_bytes(stmt.impl->stmt, offset); + return { begin, end }; +} + +template <> +std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds> +Query::get(int offset) { + assert(stmt.impl); + return std::chrono::time_point_cast<std::chrono::seconds>( + std::chrono::system_clock::from_time_t(sqlite3_column_int64(stmt.impl->stmt, offset))); +} + +template <> optional<int64_t> Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return optional<int64_t>(); + } else { + return get<int64_t>(offset); + } +} + +template <> optional<double> Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return optional<double>(); + } else { + return get<double>(offset); + } +} + +template <> optional<std::string> Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return optional<std::string>(); + } else { + return get<std::string>(offset); + } +} + +template <> +optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>> +Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return {}; + } else { + return get<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>>( + offset); + } +} + +void Query::reset() { + assert(stmt.impl); + sqlite3_reset(stmt.impl->stmt); +} + +void Query::clearBindings() { + assert(stmt.impl); + sqlite3_clear_bindings(stmt.impl->stmt); +} + +int64_t Query::lastInsertRowId() const { + assert(stmt.impl); + return stmt.impl->lastInsertRowId; +} + +uint64_t Query::changes() const { + assert(stmt.impl); + auto changes_ = stmt.impl->changes; + return (changes_ < 0 ? 0 : changes_); +} + +Transaction::Transaction(Database& db_, Mode mode) + : dbImpl(*db_.impl) { + switch (mode) { + case Deferred: + dbImpl.exec("BEGIN DEFERRED TRANSACTION"); + break; + case Immediate: + dbImpl.exec("BEGIN IMMEDIATE TRANSACTION"); + break; + case Exclusive: + dbImpl.exec("BEGIN EXCLUSIVE TRANSACTION"); + break; + } +} + +Transaction::~Transaction() { + if (needRollback) { + try { + rollback(); + } catch (...) { + // Ignore failed rollbacks in destructor. + } + } +} + +void Transaction::commit() { + needRollback = false; + dbImpl.exec("COMMIT TRANSACTION"); +} + +void Transaction::rollback() { + needRollback = false; + dbImpl.exec("ROLLBACK TRANSACTION"); +} + +} // namespace sqlite +} // namespace mapbox diff --git a/platform/default/src/mbgl/test/main.cpp b/platform/default/src/mbgl/test/main.cpp new file mode 100644 index 0000000000..d01cf75ffc --- /dev/null +++ b/platform/default/src/mbgl/test/main.cpp @@ -0,0 +1,20 @@ +#include <mbgl/test.hpp> +#include <unistd.h> +#include <cstring> +#include <cerrno> +#include <cstdio> + +#define xstr(s) str(s) +#define str(s) #s + +int main(int argc, char *argv[]) { +#ifdef WORK_DIRECTORY + const int result = chdir(xstr(WORK_DIRECTORY)); + if (result != 0) { + fprintf(stderr, "failed to change directory: %s\n", strerror(errno)); + return errno; + } +#endif + + return mbgl::runTests(argc, argv); +} diff --git a/platform/default/src/mbgl/text/bidi.cpp b/platform/default/src/mbgl/text/bidi.cpp new file mode 100644 index 0000000000..32a3dc23ef --- /dev/null +++ b/platform/default/src/mbgl/text/bidi.cpp @@ -0,0 +1,239 @@ +#include <mbgl/text/bidi.hpp> +#include <mbgl/util/traits.hpp> + +#include <unicode/ubidi.h> +#include <unicode/ushape.h> + +#include <memory> + +namespace mbgl { + +class BiDiImpl { +public: + BiDiImpl() : bidiText(ubidi_open()), bidiLine(ubidi_open()) { + } + ~BiDiImpl() { + ubidi_close(bidiText); + ubidi_close(bidiLine); + } + + UBiDi* bidiText = nullptr; + UBiDi* bidiLine = nullptr; +}; + +BiDi::BiDi() : impl(std::make_unique<BiDiImpl>()) {} +BiDi::~BiDi() = default; + +// Takes UTF16 input in logical order and applies Arabic shaping to the input while maintaining +// logical order. Output won't be intelligible until the bidirectional algorithm is applied +std::u16string applyArabicShaping(const std::u16string& input) { + UErrorCode errorCode = U_ZERO_ERROR; + + const int32_t outputLength = + u_shapeArabic(mbgl::utf16char_cast<const UChar*>(input.c_str()), static_cast<int32_t>(input.size()), nullptr, 0, + (U_SHAPE_LETTERS_SHAPE & U_SHAPE_LETTERS_MASK) | + (U_SHAPE_TEXT_DIRECTION_LOGICAL & U_SHAPE_TEXT_DIRECTION_MASK), + &errorCode); + + // Pre-flighting will always set U_BUFFER_OVERFLOW_ERROR + errorCode = U_ZERO_ERROR; + + std::u16string outputText(outputLength, 0); + + u_shapeArabic(mbgl::utf16char_cast<const UChar*>(input.c_str()), static_cast<int32_t>(input.size()), mbgl::utf16char_cast<UChar*>(&outputText[0]), outputLength, + (U_SHAPE_LETTERS_SHAPE & U_SHAPE_LETTERS_MASK) | + (U_SHAPE_TEXT_DIRECTION_LOGICAL & U_SHAPE_TEXT_DIRECTION_MASK), + &errorCode); + + // If the algorithm fails for any reason, fall back to non-transformed text + if (U_FAILURE(errorCode)) + return input; + + return outputText; +} + +void BiDi::mergeParagraphLineBreaks(std::set<size_t>& lineBreakPoints) { + int32_t paragraphCount = ubidi_countParagraphs(impl->bidiText); + for (int32_t i = 0; i < paragraphCount; i++) { + UErrorCode errorCode = U_ZERO_ERROR; + int32_t paragraphEndIndex; + ubidi_getParagraphByIndex(impl->bidiText, i, nullptr, ¶graphEndIndex, nullptr, &errorCode); + + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("ProcessedBiDiText::mergeParagraphLineBreaks: ") + + u_errorName(errorCode)); + } + + lineBreakPoints.insert(static_cast<std::size_t>(paragraphEndIndex)); + } +} + +std::vector<std::u16string> BiDi::applyLineBreaking(std::set<std::size_t> lineBreakPoints) { + // BiDi::getLine will error if called across a paragraph boundary, so we need to ensure that all + // paragraph boundaries are included in the set of line break points. The calling code might not + // include the line break because it didn't need to wrap at that point, or because the text was + // separated with a more exotic code point such as (U+001C) + mergeParagraphLineBreaks(lineBreakPoints); + + std::vector<std::u16string> transformedLines; + transformedLines.reserve(lineBreakPoints.size()); + + std::size_t start = 0; + for (std::size_t lineBreakPoint : lineBreakPoints) { + transformedLines.push_back(getLine(start, lineBreakPoint)); + start = lineBreakPoint; + } + + return transformedLines; +} + +std::vector<std::u16string> BiDi::processText(const std::u16string& input, + std::set<std::size_t> lineBreakPoints) { + UErrorCode errorCode = U_ZERO_ERROR; + + ubidi_setPara(impl->bidiText, mbgl::utf16char_cast<const UChar*>(input.c_str()), static_cast<int32_t>(input.size()), + UBIDI_DEFAULT_LTR, nullptr, &errorCode); + + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::processText: ") + u_errorName(errorCode)); + } + + return applyLineBreaking(lineBreakPoints); +} + +std::vector<StyledText> BiDi::processStyledText(const StyledText& input, std::set<std::size_t> lineBreakPoints) { + std::vector<StyledText> lines; + const auto& inputText = input.first; + const auto& styleIndices = input.second; + + UErrorCode errorCode = U_ZERO_ERROR; + + ubidi_setPara(impl->bidiText, mbgl::utf16char_cast<const UChar*>(inputText.c_str()), static_cast<int32_t>(inputText.size()), + UBIDI_DEFAULT_LTR, nullptr, &errorCode); + + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::processStyledText: ") + u_errorName(errorCode)); + } + + mergeParagraphLineBreaks(lineBreakPoints); + + std::size_t lineStartIndex = 0; + + for (std::size_t lineBreakPoint : lineBreakPoints) { + StyledText line; + line.second.reserve(lineBreakPoint - lineStartIndex); + + errorCode = U_ZERO_ERROR; + ubidi_setLine(impl->bidiText, static_cast<int32_t>(lineStartIndex), static_cast<int32_t>(lineBreakPoint), impl->bidiLine, &errorCode); + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::processStyledText (setLine): ") + u_errorName(errorCode)); + } + + errorCode = U_ZERO_ERROR; + uint32_t runCount = ubidi_countRuns(impl->bidiLine, &errorCode); + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::processStyledText (countRuns): ") + u_errorName(errorCode)); + } + + for (uint32_t runIndex = 0; runIndex < runCount; runIndex++) { + int32_t runLogicalStart; + int32_t runLength; + UBiDiDirection direction = ubidi_getVisualRun(impl->bidiLine, runIndex, &runLogicalStart, &runLength); + const bool isReversed = direction == UBIDI_RTL; + + std::size_t logicalStart = lineStartIndex + runLogicalStart; + std::size_t logicalEnd = logicalStart + runLength; + if (isReversed) { + // Within this reversed section, iterate logically backwards + // Each time we see a change in style, render a reversed chunk + // of everything since the last change + std::size_t styleRunStart = logicalEnd; + uint8_t currentStyleIndex = styleIndices.at(styleRunStart - 1); + for (std::size_t i = logicalEnd - 1; i >= logicalStart; i--) { + if (currentStyleIndex != styleIndices.at(i) || i == logicalStart) { + std::size_t styleRunEnd = i == logicalStart ? i : i + 1; + std::u16string reversed = writeReverse(inputText, styleRunEnd, styleRunStart); + line.first += reversed; + for (std::size_t j = 0; j < reversed.size(); j++) { + line.second.push_back(currentStyleIndex); + } + currentStyleIndex = styleIndices.at(i); + styleRunStart = styleRunEnd; + } + if (i == 0) { + break; + } + } + + } else { + line.first += input.first.substr(logicalStart, runLength); + line.second.insert(line.second.end(), styleIndices.begin() + logicalStart, styleIndices.begin() + logicalStart + runLength); + } + } + + lines.push_back(line); + lineStartIndex = lineBreakPoint; + } + + return lines; +} + +std::u16string BiDi::writeReverse(const std::u16string& input, std::size_t logicalStart, std::size_t logicalEnd) { + UErrorCode errorCode = U_ZERO_ERROR; + int32_t logicalLength = static_cast<int32_t>(logicalEnd - logicalStart); + std::u16string outputText(logicalLength + 1, 0); + + // UBIDI_DO_MIRRORING: Apply unicode mirroring of characters like parentheses + // UBIDI_REMOVE_BIDI_CONTROLS: Now that all the lines are set, remove control characters so that + // they don't show up on screen (some fonts have glyphs representing them) + int32_t outputLength = + ubidi_writeReverse(mbgl::utf16char_cast<const UChar*>(&input[logicalStart]), + logicalLength, + mbgl::utf16char_cast<UChar*>(&outputText[0]), + logicalLength + 1, // Extra room for null terminator, although we don't really need to have ICU write it for us + UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS, + &errorCode); + + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::writeReverse: ") + u_errorName(errorCode)); + } + + outputText.resize(outputLength); // REMOVE_BIDI_CONTROLS may have shrunk the string + + return outputText; +} + +std::u16string BiDi::getLine(std::size_t start, std::size_t end) { + UErrorCode errorCode = U_ZERO_ERROR; + ubidi_setLine(impl->bidiText, static_cast<int32_t>(start), static_cast<int32_t>(end), impl->bidiLine, &errorCode); + + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::getLine (setLine): ") + u_errorName(errorCode)); + } + + // Because we set UBIDI_REMOVE_BIDI_CONTROLS, the output may be smaller than what we reserve + // Setting UBIDI_INSERT_LRM_FOR_NUMERIC would require + // ubidi_getLength(pBiDi)+2*ubidi_countRuns(pBiDi) + const int32_t outputLength = ubidi_getProcessedLength(impl->bidiLine); + std::u16string outputText(outputLength, 0); + + // UBIDI_DO_MIRRORING: Apply unicode mirroring of characters like parentheses + // UBIDI_REMOVE_BIDI_CONTROLS: Now that all the lines are set, remove control characters so that + // they don't show up on screen (some fonts have glyphs representing them) + int32_t finalLength = ubidi_writeReordered(impl->bidiLine, + mbgl::utf16char_cast<UChar*>(&outputText[0]), + outputLength, + UBIDI_DO_MIRRORING | UBIDI_REMOVE_BIDI_CONTROLS, + &errorCode); + + outputText.resize(finalLength); // REMOVE_BIDI_CONTROLS may have shrunk the string + + if (U_FAILURE(errorCode)) { + throw std::runtime_error(std::string("BiDi::getLine (writeReordered): ") + + u_errorName(errorCode)); + } + + return outputText; +} + +} // end namespace mbgl diff --git a/platform/default/src/mbgl/text/collator.cpp b/platform/default/src/mbgl/text/collator.cpp new file mode 100644 index 0000000000..400fa4d94d --- /dev/null +++ b/platform/default/src/mbgl/text/collator.cpp @@ -0,0 +1,79 @@ +#include <mbgl/style/expression/collator.hpp> +#include <mbgl/util/platform.hpp> +#include <libnu/strcoll.h> +#include <mbgl/text/unaccent.hpp> + +/* + The default implementation of Collator ignores locale. + Case sensitivity and collation order are based on + Default Unicode Collation Element Table (DUCET). + + Diacritic-insensitivity is implemented with nunicode's + non-standard "unaccent" functionality, which is tailored + to European languages. + + It would be possible to implement locale awareness using ICU, + but would require bundling locale data. +*/ + +namespace mbgl { +namespace style { +namespace expression { + +class Collator::Impl { +public: + Impl(bool caseSensitive_, bool diacriticSensitive_, optional<std::string>) + : caseSensitive(caseSensitive_) + , diacriticSensitive(diacriticSensitive_) + {} + + bool operator==(const Impl& other) const { + return caseSensitive == other.caseSensitive && + diacriticSensitive == other.diacriticSensitive; + } + + int compare(const std::string& lhs, const std::string& rhs) const { + if (caseSensitive && diacriticSensitive) { + return nu_strcoll(lhs.c_str(), rhs.c_str(), + nu_utf8_read, nu_utf8_read); + } else if (!caseSensitive && diacriticSensitive) { + return nu_strcasecoll(lhs.c_str(), rhs.c_str(), + nu_utf8_read, nu_utf8_read); + } else if (caseSensitive && !diacriticSensitive) { + return nu_strcoll(platform::unaccent(lhs).c_str(), platform::unaccent(rhs).c_str(), + nu_utf8_read, nu_utf8_read); + } else { + return nu_strcasecoll(platform::unaccent(lhs).c_str(), platform::unaccent(rhs).c_str(), + nu_utf8_read, nu_utf8_read); + } + } + + std::string resolvedLocale() const { + return ""; + } +private: + bool caseSensitive; + bool diacriticSensitive; +}; + + +Collator::Collator(bool caseSensitive, bool diacriticSensitive, optional<std::string> locale_) + : impl(std::make_shared<Impl>(caseSensitive, diacriticSensitive, std::move(locale_))) +{} + +bool Collator::operator==(const Collator& other) const { + return *impl == *(other.impl); +} + +int Collator::compare(const std::string& lhs, const std::string& rhs) const { + return impl->compare(lhs, rhs); +} + +std::string Collator::resolvedLocale() const { + return impl->resolvedLocale(); +} + + +} // namespace expression +} // namespace style +} // namespace mbgl diff --git a/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp b/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp new file mode 100644 index 0000000000..7866f29420 --- /dev/null +++ b/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp @@ -0,0 +1,22 @@ +#include <mbgl/text/local_glyph_rasterizer.hpp> + +namespace mbgl { + +class LocalGlyphRasterizer::Impl { +}; + +LocalGlyphRasterizer::LocalGlyphRasterizer(const optional<std::string>) +{} + +LocalGlyphRasterizer::~LocalGlyphRasterizer() +{} + +bool LocalGlyphRasterizer::canRasterizeGlyph(const FontStack&, GlyphID) { + return false; +} + +Glyph LocalGlyphRasterizer::rasterizeGlyph(const FontStack&, GlyphID) { + return Glyph(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/text/unaccent.cpp b/platform/default/src/mbgl/text/unaccent.cpp new file mode 100644 index 0000000000..37b9a0d9ca --- /dev/null +++ b/platform/default/src/mbgl/text/unaccent.cpp @@ -0,0 +1,43 @@ +#include <mbgl/util/platform.hpp> +#include <libnu/unaccent.h> +#include <mbgl/text/unaccent.hpp> + +#include <cstring> +#include <sstream> + +namespace mbgl { namespace platform { + +std::string unaccent(const std::string& str) +{ + std::stringstream output; + char const *itr = str.c_str(), *nitr; + char const *end = itr + str.length(); + char lo[5] = { 0 }; + + for (; itr < end; itr = nitr) + { + uint32_t code_point = 0; + char const* buf = nullptr; + + nitr = _nu_tounaccent(itr, end, nu_utf8_read, &code_point, &buf, nullptr); + if (buf != nullptr) + { + do + { + buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point); + if (code_point == 0) break; + output.write(lo, nu_utf8_write(code_point, lo) - lo); + } + while (code_point != 0); + } + else + { + output.write(itr, nitr - itr); + } + } + + return output.str(); +} + +} // namespace platform +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/async_task.cpp b/platform/default/src/mbgl/util/async_task.cpp new file mode 100644 index 0000000000..50891056d8 --- /dev/null +++ b/platform/default/src/mbgl/util/async_task.cpp @@ -0,0 +1,66 @@ +#include <mbgl/util/async_task.hpp> + +#include <mbgl/util/run_loop.hpp> + +#include <atomic> +#include <functional> + +#include <uv.h> + +namespace mbgl { +namespace util { + +class AsyncTask::Impl { +public: + Impl(std::function<void()>&& fn) + : async(new uv_async_t), + task(std::move(fn)) { + + auto* loop = reinterpret_cast<uv_loop_t*>(RunLoop::getLoopHandle()); + if (uv_async_init(loop, async, asyncCallback) != 0) { + throw std::runtime_error("Failed to initialize async."); + } + + handle()->data = this; + uv_unref(handle()); + } + + ~Impl() { + uv_close(handle(), [](uv_handle_t* h) { + delete reinterpret_cast<uv_async_t*>(h); + }); + } + + void maySend() { + // uv_async_send will do the call coalescing for us. + if (uv_async_send(async) != 0) { + throw std::runtime_error("Failed to async send."); + } + } + +private: + static void asyncCallback(uv_async_t* handle) { + reinterpret_cast<Impl*>(handle->data)->task(); + } + + uv_handle_t* handle() { + return reinterpret_cast<uv_handle_t*>(async); + } + + uv_async_t* async; + + std::function<void()> task; +}; + +AsyncTask::AsyncTask(std::function<void()>&& fn) + : impl(std::make_unique<Impl>(std::move(fn))) { +} + +AsyncTask::~AsyncTask() = default; + +void AsyncTask::send() { + impl->maySend(); +} + +} // namespace util +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/default_thread_pool.cpp b/platform/default/src/mbgl/util/default_thread_pool.cpp new file mode 100644 index 0000000000..d3950bb8aa --- /dev/null +++ b/platform/default/src/mbgl/util/default_thread_pool.cpp @@ -0,0 +1,57 @@ +#include <mbgl/util/default_thread_pool.hpp> +#include <mbgl/actor/mailbox.hpp> +#include <mbgl/util/platform.hpp> +#include <mbgl/util/string.hpp> + +namespace mbgl { + +ThreadPool::ThreadPool(std::size_t count) { + threads.reserve(count); + for (std::size_t i = 0; i < count; ++i) { + threads.emplace_back([this, i]() { + platform::setCurrentThreadName(std::string{ "Worker " } + util::toString(i + 1)); + + while (true) { + std::unique_lock<std::mutex> lock(mutex); + + cv.wait(lock, [this] { + return !queue.empty() || terminate; + }); + + if (terminate) { + return; + } + + auto mailbox = queue.front(); + queue.pop(); + lock.unlock(); + + Mailbox::maybeReceive(mailbox); + } + }); + } +} + +ThreadPool::~ThreadPool() { + { + std::lock_guard<std::mutex> lock(mutex); + terminate = true; + } + + cv.notify_all(); + + for (auto& thread : threads) { + thread.join(); + } +} + +void ThreadPool::schedule(std::weak_ptr<Mailbox> mailbox) { + { + std::lock_guard<std::mutex> lock(mutex); + queue.push(mailbox); + } + + cv.notify_one(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/image.cpp b/platform/default/src/mbgl/util/image.cpp new file mode 100644 index 0000000000..25063892b7 --- /dev/null +++ b/platform/default/src/mbgl/util/image.cpp @@ -0,0 +1,31 @@ +#include <mbgl/util/image.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/premultiply.hpp> + +namespace mbgl { + +PremultipliedImage decodePNG(const uint8_t*, size_t); +PremultipliedImage decodeJPEG(const uint8_t*, size_t); + +PremultipliedImage decodeImage(const std::string& string) { + const auto* data = reinterpret_cast<const uint8_t*>(string.data()); + const size_t size = string.size(); + + if (size >= 4) { + uint32_t magic = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; + if (magic == 0x89504E47U) { + return decodePNG(data, size); + } + } + + if (size >= 2) { + uint16_t magic = ((data[0] << 8) | data[1]) & 0xffff; + if (magic == 0xFFD8) { + return decodeJPEG(data, size); + } + } + + throw std::runtime_error("unsupported image type"); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/jpeg_reader.cpp b/platform/default/src/mbgl/util/jpeg_reader.cpp new file mode 100644 index 0000000000..5f613f9423 --- /dev/null +++ b/platform/default/src/mbgl/util/jpeg_reader.cpp @@ -0,0 +1,151 @@ +#include <mbgl/util/image.hpp> +#include <mbgl/util/char_array_buffer.hpp> + +#include <istream> +#include <sstream> +#include <array> + +extern "C" +{ +#include <jpeglib.h> +} + +namespace mbgl { + +const static unsigned BUF_SIZE = 4096; + +struct jpeg_stream_wrapper { + jpeg_source_mgr manager; + std::istream* stream; + std::array<JOCTET, BUF_SIZE> buffer; +}; + +static void init_source(j_decompress_ptr cinfo) { + auto* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src); + wrap->stream->seekg(0, std::ios_base::beg); +} + +static boolean fill_input_buffer(j_decompress_ptr cinfo) { + auto* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src); + wrap->stream->read(reinterpret_cast<char*>(&wrap->buffer[0]), BUF_SIZE); + std::streamsize size = wrap->stream->gcount(); + wrap->manager.next_input_byte = wrap->buffer.data(); + wrap->manager.bytes_in_buffer = BUF_SIZE; + return (size > 0) ? TRUE : FALSE; +} + +static void skip(j_decompress_ptr cinfo, long count) { + if (count <= 0) return; // A zero or negative skip count should be treated as a no-op. + auto* wrap = reinterpret_cast<jpeg_stream_wrapper*>(cinfo->src); + + if (wrap->manager.bytes_in_buffer > 0 && count < static_cast<long>(wrap->manager.bytes_in_buffer)) + { + wrap->manager.bytes_in_buffer -= count; + wrap->manager.next_input_byte = &wrap->buffer[BUF_SIZE - wrap->manager.bytes_in_buffer]; + } + else + { + wrap->stream->seekg(count - wrap->manager.bytes_in_buffer, std::ios_base::cur); + // trigger buffer fill + wrap->manager.next_input_byte = nullptr; + wrap->manager.bytes_in_buffer = 0; // bytes_in_buffer may be zero on return. + } +} + +static void term(j_decompress_ptr) {} + +static void attach_stream(j_decompress_ptr cinfo, std::istream* in) { + if (cinfo->src == nullptr) { + cinfo->src = (struct jpeg_source_mgr *) + (*cinfo->mem->alloc_small) ((j_common_ptr) cinfo, JPOOL_PERMANENT, sizeof(jpeg_stream_wrapper)); + } + auto * src = reinterpret_cast<jpeg_stream_wrapper*> (cinfo->src); + src->manager.init_source = init_source; + src->manager.fill_input_buffer = fill_input_buffer; + src->manager.skip_input_data = skip; + src->manager.resync_to_restart = jpeg_resync_to_restart; + src->manager.term_source = term; + src->manager.bytes_in_buffer = 0; + src->manager.next_input_byte = nullptr; + src->stream = in; +} + +static void on_error(j_common_ptr) {} + +static void on_error_message(j_common_ptr cinfo) { + char buffer[JMSG_LENGTH_MAX]; + (*cinfo->err->format_message)(cinfo, buffer); + throw std::runtime_error(std::string("JPEG Reader: libjpeg could not read image: ") + buffer); +} + +struct jpeg_info_guard { + jpeg_info_guard(jpeg_decompress_struct* cinfo) + : i_(cinfo) {} + + ~jpeg_info_guard() { + jpeg_destroy_decompress(i_); + } + + jpeg_decompress_struct* i_; +}; + +PremultipliedImage decodeJPEG(const uint8_t* data, size_t size) { + util::CharArrayBuffer dataBuffer { reinterpret_cast<const char*>(data), size }; + std::istream stream(&dataBuffer); + + jpeg_decompress_struct cinfo; + jpeg_info_guard iguard(&cinfo); + jpeg_error_mgr jerr; + cinfo.err = jpeg_std_error(&jerr); + jerr.error_exit = on_error; + jerr.output_message = on_error_message; + jpeg_create_decompress(&cinfo); + attach_stream(&cinfo, &stream); + + int ret = jpeg_read_header(&cinfo, TRUE); + if (ret != JPEG_HEADER_OK) + throw std::runtime_error("JPEG Reader: failed to read header"); + + jpeg_start_decompress(&cinfo); + + if (cinfo.out_color_space == JCS_UNKNOWN) + throw std::runtime_error("JPEG Reader: failed to read unknown color space"); + + if (cinfo.output_width == 0 || cinfo.output_height == 0) + throw std::runtime_error("JPEG Reader: failed to read image size"); + + size_t width = cinfo.output_width; + size_t height = cinfo.output_height; + size_t components = cinfo.output_components; + size_t rowStride = components * width; + + PremultipliedImage image({ static_cast<uint32_t>(width), static_cast<uint32_t>(height) }); + uint8_t* dst = image.data.get(); + + JSAMPARRAY buffer = (*cinfo.mem->alloc_sarray)((j_common_ptr) &cinfo, JPOOL_IMAGE, rowStride, 1); + + while (cinfo.output_scanline < cinfo.output_height) { + jpeg_read_scanlines(&cinfo, buffer, 1); + + for (size_t i = 0; i < width; ++i) { + dst[0] = buffer[0][components * i]; + dst[3] = 0xFF; + + if (components > 2) { + dst[1] = buffer[0][components * i + 1]; + dst[2] = buffer[0][components * i + 2]; + } else { + dst[1] = dst[0]; + dst[2] = dst[0]; + } + + dst += 4; + } + } + + jpeg_finish_decompress(&cinfo); + + return image; +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/logging_stderr.cpp b/platform/default/src/mbgl/util/logging_stderr.cpp new file mode 100644 index 0000000000..41585fb7bb --- /dev/null +++ b/platform/default/src/mbgl/util/logging_stderr.cpp @@ -0,0 +1,12 @@ +#include <mbgl/util/logging.hpp> +#include <mbgl/util/enum.hpp> + +#include <iostream> + +namespace mbgl { + +void Log::platformRecord(EventSeverity severity, const std::string &msg) { + std::cerr << "[" << Enum<EventSeverity>::toString(severity) << "] " << msg << std::endl; +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/png_reader.cpp b/platform/default/src/mbgl/util/png_reader.cpp new file mode 100644 index 0000000000..4d4ee29d1f --- /dev/null +++ b/platform/default/src/mbgl/util/png_reader.cpp @@ -0,0 +1,142 @@ +#include <mbgl/util/image.hpp> +#include <mbgl/util/premultiply.hpp> +#include <mbgl/util/char_array_buffer.hpp> +#include <mbgl/util/logging.hpp> + +#include <istream> +#include <sstream> + +extern "C" +{ +#include <png.h> +} + +template<size_t max, typename... Args> +static std::string sprintf(const char *msg, Args... args) { + char res[max]; + int len = snprintf(res, sizeof(res), msg, args...); + return std::string(res, len); +} + +const static bool png_version_check __attribute__((unused)) = []() { + const png_uint_32 version = png_access_version_number(); + if (version != PNG_LIBPNG_VER) { + throw std::runtime_error(sprintf<96>( + "libpng version mismatch: headers report %d.%d.%d, but library reports %d.%d.%d", + PNG_LIBPNG_VER / 10000, (PNG_LIBPNG_VER / 100) % 100, PNG_LIBPNG_VER % 100, + version / 10000, (version / 100) % 100, version % 100)); + } + return true; +}(); + +namespace mbgl { + +static void user_error_fn(png_structp, png_const_charp error_msg) { + throw std::runtime_error(std::string("failed to read invalid png: '") + error_msg + "'"); +} + +static void user_warning_fn(png_structp, png_const_charp warning_msg) { + Log::Warning(Event::Image, "ImageReader (PNG): %s", warning_msg); +} + +static void png_read_data(png_structp png_ptr, png_bytep data, png_size_t length) { + auto* fin = reinterpret_cast<std::istream*>(png_get_io_ptr(png_ptr)); + fin->read(reinterpret_cast<char*>(data), length); + std::streamsize read_count = fin->gcount(); + if (read_count < 0 || static_cast<png_size_t>(read_count) != length) + { + png_error(png_ptr, "Read Error"); + } +} + +struct png_struct_guard { + png_struct_guard(png_structpp png_ptr_ptr, png_infopp info_ptr_ptr) + : p_(png_ptr_ptr), + i_(info_ptr_ptr) {} + + ~png_struct_guard() { + png_destroy_read_struct(p_,i_,nullptr); + } + + png_structpp p_; + png_infopp i_; +}; + +PremultipliedImage decodePNG(const uint8_t* data, size_t size) { + util::CharArrayBuffer dataBuffer { reinterpret_cast<const char*>(data), size }; + std::istream stream(&dataBuffer); + + png_byte header[8] = { 0 }; + stream.read(reinterpret_cast<char*>(header), 8); + if (stream.gcount() != 8) + throw std::runtime_error("PNG reader: Could not read image"); + + int is_png = !png_sig_cmp(header, 0, 8); + if (!is_png) + throw std::runtime_error("File or stream is not a png"); + + png_structp png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); + if (!png_ptr) + throw std::runtime_error("failed to allocate png_ptr"); + + // catch errors in a custom way to avoid the need for setjmp + png_set_error_fn(png_ptr, png_get_error_ptr(png_ptr), user_error_fn, user_warning_fn); + + png_infop info_ptr; + png_struct_guard sguard(&png_ptr, &info_ptr); + info_ptr = png_create_info_struct(png_ptr); + if (!info_ptr) + throw std::runtime_error("failed to create info_ptr"); + + png_set_read_fn(png_ptr, &stream, png_read_data); + png_set_sig_bytes(png_ptr, 8); + png_read_info(png_ptr, info_ptr); + + png_uint_32 width = 0; + png_uint_32 height = 0; + int bit_depth = 0; + int color_type = 0; + png_get_IHDR(png_ptr, info_ptr, &width, &height, &bit_depth, &color_type, nullptr, nullptr, nullptr); + + UnassociatedImage image({ static_cast<uint32_t>(width), static_cast<uint32_t>(height) }); + + if (color_type == PNG_COLOR_TYPE_PALETTE) + png_set_expand(png_ptr); + + if (color_type == PNG_COLOR_TYPE_GRAY && bit_depth < 8) + png_set_expand(png_ptr); + + if (png_get_valid(png_ptr, info_ptr, PNG_INFO_tRNS)) + png_set_expand(png_ptr); + + if (bit_depth == 16) + png_set_strip_16(png_ptr); + + if (color_type == PNG_COLOR_TYPE_GRAY || + color_type == PNG_COLOR_TYPE_GRAY_ALPHA) + png_set_gray_to_rgb(png_ptr); + + png_set_add_alpha(png_ptr, 0xff, PNG_FILLER_AFTER); + + if (png_get_interlace_type(png_ptr,info_ptr) == PNG_INTERLACE_ADAM7) { + png_set_interlace_handling(png_ptr); // FIXME: libpng bug? + // according to docs png_read_image + // "..automatically handles interlacing, + // so you don't need to call png_set_interlace_handling()" + } + + png_read_update_info(png_ptr, info_ptr); + + // we can read whole image at once + // alloc row pointers + const std::unique_ptr<png_bytep[]> rows(new png_bytep[height]); + for (unsigned row = 0; row < height; ++row) + rows[row] = image.data.get() + row * width * 4; + png_read_image(png_ptr, rows.get()); + + png_read_end(png_ptr, nullptr); + + return util::premultiply(std::move(image)); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/png_writer.cpp b/platform/default/src/mbgl/util/png_writer.cpp new file mode 100644 index 0000000000..b89e253f85 --- /dev/null +++ b/platform/default/src/mbgl/util/png_writer.cpp @@ -0,0 +1,77 @@ +#include <mbgl/util/compression.hpp> +#include <mbgl/util/image.hpp> +#include <mbgl/util/premultiply.hpp> + +#include <boost/crc.hpp> + +#include <cassert> +#include <cstring> + +#define NETWORK_BYTE_UINT32(value) \ + char(value >> 24), char(value >> 16), char(value >> 8), char(value >> 0) + +namespace { + +void addChunk(std::string& png, const char* type, const char* data = "", const uint32_t size = 0) { + assert(strlen(type) == 4); + + // Checksum encompasses type + data + boost::crc_32_type checksum; + checksum.process_bytes(type, 4); + checksum.process_bytes(data, size); + + const char length[4] = { NETWORK_BYTE_UINT32(size) }; + const char crc[4] = { NETWORK_BYTE_UINT32(checksum.checksum()) }; + + png.reserve(png.size() + 4 /* length */ + 4 /* type */ + size + 4 /* CRC */); + png.append(length, 4); + png.append(type, 4); + png.append(data, size); + png.append(crc, 4); +} + +} // namespace + +namespace mbgl { + +// Encode PNGs without libpng. +std::string encodePNG(const PremultipliedImage& pre) { + // Make copy of the image so that we can unpremultiply it. + const auto src = util::unpremultiply(pre.clone()); + + // PNG magic bytes + const char preamble[8] = { char(0x89), 'P', 'N', 'G', '\r', '\n', 0x1a, '\n' }; + + // IHDR chunk for our RGBA image. + const char ihdr[13] = { + NETWORK_BYTE_UINT32(src.size.width), // width + NETWORK_BYTE_UINT32(src.size.height), // height + 8, // bit depth == 8 bits + 6, // color type == RGBA + 0, // compression method == deflate + 0, // filter method == default + 0, // interlace method == none + }; + + // Prepare the (compressed) data chunk. + const auto stride = src.stride(); + std::string idat; + for (uint32_t y = 0; y < src.size.height; y++) { + // Every scanline needs to be prefixed with one byte that indicates the filter type. + idat.append(1, 0); // filter type 0 + idat.append((const char*)(src.data.get() + y * stride), stride); + } + idat = util::compress(idat); + + // Assemble the PNG. + std::string png; + png.reserve((8 /* preamble */) + (12 + 13 /* IHDR */) + + (12 + idat.size() /* IDAT */) + (12 /* IEND */)); + png.append(preamble, 8); + addChunk(png, "IHDR", ihdr, 13); + addChunk(png, "IDAT", idat.data(), static_cast<uint32_t>(idat.size())); + addChunk(png, "IEND"); + return png; +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/run_loop.cpp b/platform/default/src/mbgl/util/run_loop.cpp new file mode 100644 index 0000000000..868ee72114 --- /dev/null +++ b/platform/default/src/mbgl/util/run_loop.cpp @@ -0,0 +1,219 @@ +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/async_task.hpp> +#include <mbgl/util/thread_local.hpp> +#include <mbgl/actor/scheduler.hpp> + +#include <uv.h> + +#include <cassert> +#include <functional> +#include <unordered_map> + +namespace { + +void dummyCallback(uv_async_t*) {} + +} // namespace + +namespace mbgl { +namespace util { + +struct Watch { + static void onEvent(uv_poll_t* poll, int, int event) { + auto watch = reinterpret_cast<Watch*>(poll->data); + + RunLoop::Event watchEvent = RunLoop::Event::None; + switch (event) { + case UV_READABLE: + watchEvent = RunLoop::Event::Read; + break; + case UV_WRITABLE: + watchEvent = RunLoop::Event::Write; + break; + case UV_READABLE | UV_WRITABLE: + watchEvent = RunLoop::Event::ReadWrite; + break; + } + + watch->eventCallback(watch->fd, watchEvent); + }; + + static void onClose(uv_handle_t *poll) { + auto watch = reinterpret_cast<Watch*>(poll->data); + watch->closeCallback(); + }; + + uv_poll_t poll; + int fd; + + std::function<void(int, RunLoop::Event)> eventCallback; + std::function<void()> closeCallback; +}; + +RunLoop* RunLoop::Get() { + assert(static_cast<RunLoop*>(Scheduler::GetCurrent())); + return static_cast<RunLoop*>(Scheduler::GetCurrent()); +} + +class RunLoop::Impl { +public: + void closeHolder() { + uv_close(holderHandle(), [](uv_handle_t* h) { + delete reinterpret_cast<uv_async_t*>(h); + }); + } + + uv_handle_t* holderHandle() { + return reinterpret_cast<uv_handle_t*>(holder); + } + + uv_loop_t *loop = nullptr; + uv_async_t* holder = new uv_async_t; + + RunLoop::Type type; + std::unique_ptr<AsyncTask> async; + + std::unordered_map<int, std::unique_ptr<Watch>> watchPoll; +}; + +RunLoop::RunLoop(Type type) : impl(std::make_unique<Impl>()) { + switch (type) { + case Type::New: + impl->loop = new uv_loop_t; + if (uv_loop_init(impl->loop) != 0) { + throw std::runtime_error("Failed to initialize loop."); + } + break; + case Type::Default: + impl->loop = uv_default_loop(); + break; + } + + // Just for holding a ref to the main loop and keep + // it alive as required by libuv. + if (uv_async_init(impl->loop, impl->holder, dummyCallback) != 0) { + throw std::runtime_error("Failed to initialize async."); + } + + impl->type = type; + + Scheduler::SetCurrent(this); + impl->async = std::make_unique<AsyncTask>(std::bind(&RunLoop::process, this)); +} + +RunLoop::~RunLoop() { + Scheduler::SetCurrent(nullptr); + + // Close the dummy handle that we have + // just to keep the main loop alive. + impl->closeHolder(); + + if (impl->type == Type::Default) { + return; + } + + // Run the loop again to ensure that async + // close callbacks have been called. Not needed + // for the default main loop because it is only + // closed when the application exits. + impl->async.reset(); + runOnce(); + + if (uv_loop_close(impl->loop) == UV_EBUSY) { + assert(false && "Failed to close loop."); + } + delete impl->loop; +} + +LOOP_HANDLE RunLoop::getLoopHandle() { + return Get()->impl->loop; +} + +void RunLoop::wake() { + impl->async->send(); +} + +void RunLoop::run() { + MBGL_VERIFY_THREAD(tid); + + uv_ref(impl->holderHandle()); + uv_run(impl->loop, UV_RUN_DEFAULT); +} + +void RunLoop::runOnce() { + MBGL_VERIFY_THREAD(tid); + + uv_run(impl->loop, UV_RUN_NOWAIT); +} + +void RunLoop::stop() { + invoke([&] { uv_unref(impl->holderHandle()); }); +} + +void RunLoop::addWatch(int fd, Event event, std::function<void(int, Event)>&& callback) { + MBGL_VERIFY_THREAD(tid); + + Watch *watch = nullptr; + auto watchPollIter = impl->watchPoll.find(fd); + + if (watchPollIter == impl->watchPoll.end()) { + std::unique_ptr<Watch> watchPtr = std::make_unique<Watch>(); + + watch = watchPtr.get(); + impl->watchPoll[fd] = std::move(watchPtr); + + if (uv_poll_init(impl->loop, &watch->poll, fd)) { + throw std::runtime_error("Failed to init poll on file descriptor."); + } + } else { + watch = watchPollIter->second.get(); + } + + watch->poll.data = watch; + watch->fd = fd; + watch->eventCallback = std::move(callback); + + int pollEvent = 0; + switch (event) { + case Event::Read: + pollEvent = UV_READABLE; + break; + case Event::Write: + pollEvent = UV_WRITABLE; + break; + case Event::ReadWrite: + pollEvent = UV_READABLE | UV_WRITABLE; + break; + default: + throw std::runtime_error("Unhandled event."); + } + + if (uv_poll_start(&watch->poll, pollEvent, &Watch::onEvent)) { + throw std::runtime_error("Failed to start poll on file descriptor."); + } +} + +void RunLoop::removeWatch(int fd) { + MBGL_VERIFY_THREAD(tid); + + auto watchPollIter = impl->watchPoll.find(fd); + if (watchPollIter == impl->watchPoll.end()) { + return; + } + + Watch* watch = watchPollIter->second.release(); + impl->watchPoll.erase(watchPollIter); + + watch->closeCallback = [watch] { + delete watch; + }; + + if (uv_poll_stop(&watch->poll)) { + throw std::runtime_error("Failed to stop poll on file descriptor."); + } + + uv_close(reinterpret_cast<uv_handle_t*>(&watch->poll), &Watch::onClose); +} + +} // namespace util +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/shared_thread_pool.cpp b/platform/default/src/mbgl/util/shared_thread_pool.cpp new file mode 100644 index 0000000000..d7facbab94 --- /dev/null +++ b/platform/default/src/mbgl/util/shared_thread_pool.cpp @@ -0,0 +1,14 @@ +#include <mbgl/util/shared_thread_pool.hpp> + +namespace mbgl { + +std::shared_ptr<ThreadPool> sharedThreadPool() { + static std::weak_ptr<ThreadPool> weak; + auto pool = weak.lock(); + if (!pool) { + weak = pool = std::make_shared<ThreadPool>(4); + } + return pool; +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/string_stdlib.cpp b/platform/default/src/mbgl/util/string_stdlib.cpp new file mode 100644 index 0000000000..103444df1c --- /dev/null +++ b/platform/default/src/mbgl/util/string_stdlib.cpp @@ -0,0 +1,74 @@ +#include <mbgl/util/platform.hpp> +#include <libnu/casemap.h> +#include <cstring> +#include <sstream> + +namespace mbgl { namespace platform { + +std::string uppercase(const std::string& str) +{ + std::stringstream output; + char const *itr = str.c_str(), *nitr; + char const *end = itr + str.length(); + char lo[5] = { 0 }; + + for (; itr < end; itr = nitr) + { + uint32_t code_point = 0; + char const* buf = nullptr; + + nitr = _nu_toupper(itr, end, nu_utf8_read, &code_point, &buf, nullptr); + if (buf != nullptr) + { + do + { + buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point); + if (code_point == 0) break; + output.write(lo, nu_utf8_write(code_point, lo) - lo); + } + while (code_point != 0); + } + else + { + output.write(itr, nitr - itr); + } + } + + return output.str(); + +} + +std::string lowercase(const std::string& str) +{ + std::stringstream output; + char const *itr = str.c_str(), *nitr; + char const *end = itr + str.length(); + char lo[5] = { 0 }; + + for (; itr < end; itr = nitr) + { + uint32_t code_point = 0; + char const* buf = nullptr; + + nitr = _nu_tolower(itr, end, nu_utf8_read, &code_point, &buf, nullptr); + if (buf != nullptr) + { + do + { + buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point); + if (code_point == 0) break; + output.write(lo, nu_utf8_write(code_point, lo) - lo); + } + while (code_point != 0); + } + else + { + output.write(itr, nitr - itr); + } + } + + return output.str(); +} + +} // namespace platform +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/thread.cpp b/platform/default/src/mbgl/util/thread.cpp new file mode 100644 index 0000000000..c7c79b4fb0 --- /dev/null +++ b/platform/default/src/mbgl/util/thread.cpp @@ -0,0 +1,37 @@ +#include <mbgl/util/platform.hpp> +#include <mbgl/util/logging.hpp> + +#include <string> + +#include <pthread.h> +#include <sched.h> + +namespace mbgl { +namespace platform { + +std::string getCurrentThreadName() { + char name[32] = "unknown"; + pthread_getname_np(pthread_self(), name, sizeof(name)); + + return name; +} + +void setCurrentThreadName(const std::string& name) { + if (name.size() > 15) { // Linux hard limit (see manpages). + pthread_setname_np(pthread_self(), name.substr(0, 15).c_str()); + } else { + pthread_setname_np(pthread_self(), name.c_str()); + } +} + +void makeThreadLowPriority() { + struct sched_param param; + param.sched_priority = 0; + + if (sched_setscheduler(0, SCHED_IDLE, ¶m) != 0) { + Log::Warning(Event::General, "Couldn't set thread scheduling policy"); + } +} + +} // namespace platform +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/thread_local.cpp b/platform/default/src/mbgl/util/thread_local.cpp new file mode 100644 index 0000000000..db70773c12 --- /dev/null +++ b/platform/default/src/mbgl/util/thread_local.cpp @@ -0,0 +1,66 @@ +#include <mbgl/util/thread_local.hpp> + +#include <mbgl/renderer/backend_scope.hpp> +#include <mbgl/util/logging.hpp> +#include <mbgl/util/run_loop.hpp> + +#include <stdexcept> +#include <cassert> + +#include <pthread.h> + +namespace mbgl { +namespace util { + +template <class T> +class ThreadLocal<T>::Impl { +public: + pthread_key_t key; +}; + +template <class T> +ThreadLocal<T>::ThreadLocal() : impl(std::make_unique<Impl>()) { + int ret = pthread_key_create(&impl->key, [](void *) {}); + + if (ret) { + throw std::runtime_error("Failed to init local storage key."); + } +} + +template <class T> +ThreadLocal<T>::~ThreadLocal() { + // ThreadLocal will not take ownership + // of the pointer it is managing. The pointer + // needs to be explicitly cleared before we + // destroy this object. + assert(!get()); + + if (pthread_key_delete(impl->key)) { + Log::Error(Event::General, "Failed to delete local storage key."); + assert(false); + } +} + +template <class T> +T* ThreadLocal<T>::get() { + auto* ret = reinterpret_cast<T*>(pthread_getspecific(impl->key)); + if (!ret) { + return nullptr; + } + + return ret; +} + +template <class T> +void ThreadLocal<T>::set(T* ptr) { + if (pthread_setspecific(impl->key, ptr)) { + throw std::runtime_error("Failed to set local storage."); + } +} + +template class ThreadLocal<BackendScope>; +template class ThreadLocal<Scheduler>; +template class ThreadLocal<int>; // For unit tests + +} // namespace util +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/timer.cpp b/platform/default/src/mbgl/util/timer.cpp new file mode 100644 index 0000000000..90a85bfc1f --- /dev/null +++ b/platform/default/src/mbgl/util/timer.cpp @@ -0,0 +1,73 @@ +#include <mbgl/util/timer.hpp> + +#include <mbgl/util/run_loop.hpp> + +#include <uv.h> + +namespace mbgl { +namespace util { + +class Timer::Impl { +public: + Impl() : timer(new uv_timer_t) { + auto* loop = reinterpret_cast<uv_loop_t*>(RunLoop::getLoopHandle()); + if (uv_timer_init(loop, timer) != 0) { + throw std::runtime_error("Failed to initialize timer."); + } + + handle()->data = this; + uv_unref(handle()); + } + + ~Impl() { + uv_close(handle(), [](uv_handle_t* h) { + delete reinterpret_cast<uv_timer_t*>(h); + }); + } + + void start(uint64_t timeout, uint64_t repeat, std::function<void ()>&& cb_) { + cb = std::move(cb_); + if (uv_timer_start(timer, timerCallback, timeout, repeat) != 0) { + throw std::runtime_error("Failed to start timer."); + } + } + + void stop() { + cb = nullptr; + if (uv_timer_stop(timer) != 0) { + throw std::runtime_error("Failed to stop timer."); + } + } + +private: + static void timerCallback(uv_timer_t* handle) { + reinterpret_cast<Impl*>(handle->data)->cb(); + } + + uv_handle_t* handle() { + return reinterpret_cast<uv_handle_t*>(timer); + } + + uv_timer_t* timer; + + std::function<void()> cb; +}; + +Timer::Timer() + : impl(std::make_unique<Impl>()) { +} + +Timer::~Timer() = default; + +void Timer::start(Duration timeout, Duration repeat, std::function<void()>&& cb) { + impl->start(std::chrono::duration_cast<Milliseconds>(timeout).count(), + std::chrono::duration_cast<Milliseconds>(repeat).count(), + std::move(cb)); +} + +void Timer::stop() { + impl->stop(); +} + +} // namespace util +} // namespace mbgl diff --git a/platform/default/src/mbgl/util/utf.cpp b/platform/default/src/mbgl/util/utf.cpp new file mode 100644 index 0000000000..f0f9d3e67a --- /dev/null +++ b/platform/default/src/mbgl/util/utf.cpp @@ -0,0 +1,17 @@ +#include <mbgl/util/utf.hpp> + +#include <boost/locale/encoding_utf.hpp> + +namespace mbgl { +namespace util { + +std::u16string convertUTF8ToUTF16(const std::string& str) { + return boost::locale::conv::utf_to_utf<char16_t>(str); +} + +std::string convertUTF16ToUTF8(const std::u16string& str) { + return boost::locale::conv::utf_to_utf<char>(str); +} + +} // namespace util +} // namespace mbgl |