OSM Geoservice plugin improvements, including offline data support

This patch contains improvements for the OSM geoservice plugin, in form
of property-controllable disk cache, and the support for a directory
containing an offline dataset.

The offline directory can contain subdirectories also in form of
symlinks. If duplicate tiles are present, the newest is used.

The population of the offline tiles registry is offloaded to a
thread to prevent blocking the startup of the application.
No synchronization is used (except on cache destruction),
so that, as the tiles are being indexed, they can be used in
the application.

For the time being, no support is given to populate such directory
(mostly due to current API limitations), but tiles can be placed
manually, for example copying them from the disk cache directory.

Task-number: QTBUG-45284
Change-Id: I518980669a3ee474545025adf05adc69cdd29781
Reviewed-by: Paolo Angelelli <paolo.angelelli@theqtcompany.com>
Reviewed-by: Alex Blasche <alexander.blasche@theqtcompany.com>

5 files changed, 243 insertions, 3 deletions

diff --git a/src/plugins/geoservices/osm/osm.pro b/src/plugins/geoservices/osm/osm.pro
index e73c16d7..3de72cea 100644
--- a/src/plugins/geoservices/osm/osm.pro
+++ b/src/plugins/geoservices/osm/osm.pro
@@ -1,6 +1,6 @@
 TARGET = qtgeoservices_osm
 
-QT += location-private positioning-private network
+QT += location-private positioning-private network concurrent
 
 HEADERS += \
     qgeoserviceproviderpluginosm.h \
@@ -14,7 +14,8 @@ HEADERS += \
     qplacemanagerengineosm.h \
     qplacesearchreplyosm.h \
     qplacecategoriesreplyosm.h \
-    qgeotiledmaposm.h
+    qgeotiledmaposm.h \
+    qgeofiletilecacheosm.h
 
 SOURCES += \
     qgeoserviceproviderpluginosm.cpp \
@@ -28,7 +29,8 @@ SOURCES += \
     qplacemanagerengineosm.cpp \
     qplacesearchreplyosm.cpp \
     qplacecategoriesreplyosm.cpp \
-    qgeotiledmaposm.cpp
+    qgeotiledmaposm.cpp \
+    qgeofiletilecacheosm.cpp
 
 
 OTHER_FILES += \
diff --git a/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp b/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
new file mode 100644
index 00000000..a780faa9
--- /dev/null
+++ b/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
@@ -0,0 +1,131 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the QtLocation module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qgeofiletilecacheosm.h"
+#include <QtLocation/private/qgeotilespec_p.h>
+#include <QDir>
+#include <QDirIterator>
+#include <QPair>
+#include <QDateTime>
+#include <QtConcurrent>
+#include <QThread>
+
+QT_BEGIN_NAMESPACE
+
+QGeoFileTileCacheOsm::QGeoFileTileCacheOsm(const QString &offlineDirectory, const QString &directory, QObject *parent)
+    :QGeoFileTileCache(directory, parent), m_offlineDirectory(offlineDirectory),
+     m_requestCancel(0)
+{
+}
+
+QGeoFileTileCacheOsm::~QGeoFileTileCacheOsm()
+{
+    m_requestCancel = 1;
+    m_future.waitForFinished();
+}
+
+QSharedPointer<QGeoTileTexture> QGeoFileTileCacheOsm::get(const QGeoTileSpec &spec)
+{
+    QSharedPointer<QGeoTileTexture> tt = getFromMemory(spec);
+    if (tt)
+        return tt;
+    if (tt = getFromOfflineStorage(spec))
+        return tt;
+    return getFromDisk(spec);
+}
+
+void QGeoFileTileCacheOsm::init()
+{
+    QGeoFileTileCache::init();
+    if (!m_offlineDirectory.isEmpty())
+        m_future = QtConcurrent::run(this, &QGeoFileTileCacheOsm::initOfflineRegistry);
+}
+
+QSharedPointer<QGeoTileTexture> QGeoFileTileCacheOsm::getFromOfflineStorage(const QGeoTileSpec &spec)
+{
+    QMutexLocker locker(&storageLock);
+    if (m_tilespecToOfflineFilepath.contains(spec)) {
+        QFile file(m_tilespecToOfflineFilepath[spec]);
+        file.open(QIODevice::ReadOnly);
+        QByteArray bytes = file.readAll();
+        file.close();
+        locker.unlock();
+
+        QImage image;
+        if (!image.loadFromData(bytes)) {
+            handleError(spec, QLatin1String("Problem with tile image"));
+            return QSharedPointer<QGeoTileTexture>(0);
+        }
+
+        addToMemoryCache(spec, bytes, QString());
+        QSharedPointer<QGeoTileTexture> tt = addToTextureCache(spec, image);
+        if (tt)
+            return tt;
+    }
+
+    return QSharedPointer<QGeoTileTexture>();
+}
+
+void QGeoFileTileCacheOsm::initOfflineRegistry()
+{
+    // Dealing with duplicates: picking the newest
+    QMap<QString, QPair<QString, QDateTime> > fileDates; // key is filename, value is <filepath, lastmodified>
+    QDirIterator it(m_offlineDirectory, QStringList() << "*.*", QDir::Files, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks );
+    while (it.hasNext()) {
+        const QString &path = it.next();
+        QFileInfo f(path);
+        if (!fileDates.contains(f.fileName()) || fileDates[f.fileName()].second < f.lastModified())
+            fileDates[f.fileName()] = QPair<QString, QDateTime>(f.filePath(), f.lastModified());
+        if (m_requestCancel)
+            return;
+    }
+
+    int count = 0;
+    for (auto i= fileDates.begin(); i != fileDates.end(); ++i) {
+        QGeoTileSpec spec = filenameToTileSpec(i.key());
+        if (spec.zoom() == -1)
+            continue;
+        count++;
+        storageLock.lock();
+        m_tilespecToOfflineFilepath[spec] = i.value().first;
+        storageLock.unlock();
+        if (m_requestCancel)
+            return;
+    }
+    qWarning() << "OSM Offline tiles: "<<count;
+}
+
+QT_END_NAMESPACE
diff --git a/src/plugins/geoservices/osm/qgeofiletilecacheosm.h b/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
new file mode 100644
index 00000000..f48fad7f
--- /dev/null
+++ b/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the QtLocation module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QGEOFILETILECACHEOSM_H
+#define QGEOFILETILECACHEOSM_H
+
+#include <QtLocation/private/qgeofiletilecache_p.h>
+#include <QHash>
+#include <QtConcurrent>
+#include <qatomic.h>
+
+QT_BEGIN_NAMESPACE
+
+class QGeoFileTileCacheOsm : public QGeoFileTileCache
+{
+    Q_OBJECT
+public:
+    QGeoFileTileCacheOsm(const QString &offlineDirectory = QString(), const QString &directory = QString(), QObject *parent = 0);
+    ~QGeoFileTileCacheOsm();
+
+    QSharedPointer<QGeoTileTexture> get(const QGeoTileSpec &spec) Q_DECL_OVERRIDE;
+
+protected:
+    void init() Q_DECL_OVERRIDE;
+    QSharedPointer<QGeoTileTexture> getFromOfflineStorage(const QGeoTileSpec &spec);
+
+    void initOfflineRegistry();
+
+    QString m_offlineDirectory;
+    QHash<QGeoTileSpec, QString> m_tilespecToOfflineFilepath;
+    QAtomicInt m_requestCancel;
+    QFuture<void> m_future;
+    QMutex storageLock;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGEOFILETILECACHEOSM_H
diff --git a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
index d5490559..043f3f29 100644
--- a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
+++ b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
@@ -40,6 +40,7 @@
 #include "qgeotiledmappingmanagerengineosm.h"
 #include "qgeotilefetcherosm.h"
 #include "qgeotiledmaposm.h"
+#include "qgeofiletilecacheosm.h"
 
 #include <QtLocation/private/qgeocameracapabilities_p.h>
 #include <QtLocation/private/qgeomaptype_p.h>
@@ -84,6 +85,39 @@ QGeoTiledMappingManagerEngineOsm::QGeoTiledMappingManagerEngineOsm(const QVarian
 
     setTileFetcher(tileFetcher);
 
+
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.directory"))) {
+        m_cacheDirectory = parameters.value(QStringLiteral("osm.mapping.cache.directory")).toString();
+    } else {
+        // managerName() is not yet set, we have to hardcode the plugin name below
+        m_cacheDirectory = QAbstractGeoTileCache::baseCacheDirectory() + QLatin1String("osm");
+    }
+    if (parameters.contains(QStringLiteral("osm.mapping.offline.directory"))) {
+        m_offlineDirectory = parameters.value(QStringLiteral("osm.mapping.offline.directory")).toString();
+    }
+    QAbstractGeoTileCache *tileCache = new QGeoFileTileCacheOsm(m_offlineDirectory, m_cacheDirectory);
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.disk.size"))) {
+        bool ok = false;
+        int cacheSize = parameters.value(QStringLiteral("osm.mapping.cache.disk.size")).toString().toInt(&ok);
+        if (ok)
+            tileCache->setMaxDiskUsage(cacheSize);
+    } else {
+        tileCache->setMaxDiskUsage(100 * 1024 * 1024);
+    }
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.memory.size"))) {
+        bool ok = false;
+        int cacheSize = parameters.value(QStringLiteral("osm.mapping.cache.memory.size")).toString().toInt(&ok);
+        if (ok)
+            tileCache->setMaxMemoryUsage(cacheSize);
+    }
+    if (parameters.contains(QStringLiteral("osm.mapping.cache.texture.size"))) {
+        bool ok = false;
+        int cacheSize = parameters.value(QStringLiteral("osm.mapping.cache.texture.size")).toString().toInt(&ok);
+        if (ok)
+            tileCache->setExtraTextureUsage(cacheSize);
+    }
+    setTileCache(tileCache);
+
     *error = QGeoServiceProvider::NoError;
     errorString->clear();
 }
diff --git a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h
index e0f7d873..4998caf9 100644
--- a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h
+++ b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.h
@@ -60,6 +60,8 @@ public:
 
 private:
     QString m_customCopyright;
+    QString m_cacheDirectory;
+    QString m_offlineDirectory;
 };
 
 QT_END_NAMESPACE