Fix cache mixing with high-dpi and low-dpi osm providers

This cache fixes the cache mixing problem that has been introduced
with the osm high dpi tiles support.

high dpi providers fall back to low dpi ones and ultimately
to hardcoded providers (also low dpi), and can also be enabled/disabled
via plugin parameter, thus leaving the cache for a given map id
dirty for the next run.

With this patch high dpi tiles are named differently from low dpi ones.
If high-dpi providers are selected, but become not available, the
cache can also change the tileset to load at runtime

Change-Id: I229692da07c1fc61c58fb0b6fae6ec5af16e43a7
Reviewed-by: Paolo Angelelli <paolo.angelelli@qt.io>

7 files changed, 227 insertions, 21 deletions

diff --git a/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp b/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
index a780faa9..a8429f3d 100644
--- a/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
+++ b/src/plugins/geoservices/osm/qgeofiletilecacheosm.cpp
@@ -45,10 +45,16 @@
 
 QT_BEGIN_NAMESPACE
 
-QGeoFileTileCacheOsm::QGeoFileTileCacheOsm(const QString &offlineDirectory, const QString &directory, QObject *parent)
-    :QGeoFileTileCache(directory, parent), m_offlineDirectory(offlineDirectory),
-     m_requestCancel(0)
+QGeoFileTileCacheOsm::QGeoFileTileCacheOsm(const QVector<QGeoTileProviderOsm *> &providers, const QString &offlineDirectory, const QString &directory, QObject *parent)
+:   QGeoFileTileCache(directory, parent), m_offlineDirectory(offlineDirectory),
+     m_requestCancel(0), m_providers(providers)
 {
+    m_highDpi.resize(providers.size());
+    for (int i = 0; i < providers.size(); i++) {
+        m_highDpi[i] = providers[i]->isHighDpi();
+        connect(providers[i], &QGeoTileProviderOsm::resolutionFinished, this, &QGeoFileTileCacheOsm::onProviderResolutionFinished);
+        connect(providers[i], &QGeoTileProviderOsm::resolutionError, this, &QGeoFileTileCacheOsm::onProviderResolutionFinished);
+    }
 }
 
 QGeoFileTileCacheOsm::~QGeoFileTileCacheOsm()
@@ -67,6 +73,24 @@ QSharedPointer<QGeoTileTexture> QGeoFileTileCacheOsm::get(const QGeoTileSpec &sp
     return getFromDisk(spec);
 }
 
+void QGeoFileTileCacheOsm::onProviderResolutionFinished(const QGeoTileProviderOsm *provider)
+{
+    Q_UNUSED(provider)
+    for (int i = 0; i < m_providers.size(); i++) {
+        if (m_providers[i]->isHighDpi() != m_highDpi[i]) {
+            int mapId = m_providers[i]->mapType().mapId();
+            m_highDpi[i] = m_providers[i]->isHighDpi();
+
+            // reload cache for mapId i
+            dropTiles(mapId);
+            loadTiles(mapId);
+
+            // send signal to clear scene in all maps created through this provider that use the reloaded tiles
+            emit mapDataUpdated(mapId);
+        }
+    }
+}
+
 void QGeoFileTileCacheOsm::init()
 {
     QGeoFileTileCache::init();
@@ -99,6 +123,42 @@ QSharedPointer<QGeoTileTexture> QGeoFileTileCacheOsm::getFromOfflineStorage(cons
     return QSharedPointer<QGeoTileTexture>();
 }
 
+void QGeoFileTileCacheOsm::dropTiles(int mapId)
+{
+    QList<QGeoTileSpec> keys;
+    keys = textureCache_.keys();
+    for (const QGeoTileSpec &k : keys)
+        if (k.mapId() == mapId)
+            textureCache_.remove(k);
+
+    keys = memoryCache_.keys();
+    for (const QGeoTileSpec &k : keys)
+        if (k.mapId() == mapId)
+            memoryCache_.remove(k);
+
+    keys = diskCache_.keys();
+    for (const QGeoTileSpec &k : keys)
+        if (k.mapId() == mapId)
+            diskCache_.remove(k);
+}
+
+void QGeoFileTileCacheOsm::loadTiles(int mapId)
+{
+    QStringList formats;
+    formats << QLatin1String("*.*");
+
+    QDir dir(directory_);
+    QStringList files = dir.entryList(formats, QDir::Files);
+
+    for (int i = 0; i < files.size(); ++i) {
+        QGeoTileSpec spec = filenameToTileSpec(files.at(i));
+        if (spec.zoom() == -1 || spec.mapId() != mapId)
+            continue;
+        QString filename = dir.filePath(files.at(i));
+        addToDiskCache(spec, filename);
+    }
+}
+
 void QGeoFileTileCacheOsm::initOfflineRegistry()
 {
     // Dealing with duplicates: picking the newest
@@ -128,4 +188,80 @@ void QGeoFileTileCacheOsm::initOfflineRegistry()
     qWarning() << "OSM Offline tiles: "<<count;
 }
 
+QString QGeoFileTileCacheOsm::tileSpecToFilename(const QGeoTileSpec &spec, const QString &format, const QString &directory) const
+{
+    int providerId = spec.mapId() - 1;
+    if (providerId < 0 || providerId >= m_providers.size())
+        return QString();
+    QString filename = spec.plugin();
+    filename += QLatin1String("-");
+    filename += (m_providers[providerId]->isHighDpi()) ? QLatin1Char('h') : QLatin1Char('l');
+    filename += QLatin1String("-");
+    filename += QString::number(spec.mapId());
+    filename += QLatin1String("-");
+    filename += QString::number(spec.zoom());
+    filename += QLatin1String("-");
+    filename += QString::number(spec.x());
+    filename += QLatin1String("-");
+    filename += QString::number(spec.y());
+
+    //Append version if real version number to ensure backwards compatibility and eviction of old tiles
+    if (spec.version() != -1) {
+        filename += QLatin1String("-");
+        filename += QString::number(spec.version());
+    }
+
+    filename += QLatin1String(".");
+    filename += format;
+
+    QDir dir = QDir(directory);
+
+    return dir.filePath(filename);
+}
+
+QGeoTileSpec QGeoFileTileCacheOsm::filenameToTileSpec(const QString &filename) const
+{
+    QGeoTileSpec emptySpec;
+
+    QStringList parts = filename.split('.');
+
+    if (parts.length() != 2)
+        return emptySpec;
+
+    QString name = parts.at(0);
+    QStringList fields = name.split('-');
+
+    int length = fields.length();
+    if (length != 6 && length != 7)
+        return emptySpec;
+
+    QList<int> numbers;
+
+    bool ok = false;
+    for (int i = 2; i < length; ++i) {
+        ok = false;
+        int value = fields.at(i).toInt(&ok);
+        if (!ok)
+            return emptySpec;
+        numbers.append(value);
+    }
+
+    bool highDpi = m_providers[numbers.at(0) - 1]->isHighDpi();
+    if (highDpi && fields.at(1) != QLatin1Char('h'))
+        return emptySpec;
+    else if (!highDpi && fields.at(1) != QLatin1Char('l'))
+        return emptySpec;
+
+    //File name without version, append default
+    if (numbers.length() < 5)
+        numbers.append(-1);
+
+    return QGeoTileSpec(fields.at(0),
+                    numbers.at(0),
+                    numbers.at(1),
+                    numbers.at(2),
+                    numbers.at(3),
+                    numbers.at(4));
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/geoservices/osm/qgeofiletilecacheosm.h b/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
index f48fad7f..52d57747 100644
--- a/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
+++ b/src/plugins/geoservices/osm/qgeofiletilecacheosm.h
@@ -37,6 +37,7 @@
 #ifndef QGEOFILETILECACHEOSM_H
 #define QGEOFILETILECACHEOSM_H
 
+#include "qgeotileproviderosm.h"
 #include <QtLocation/private/qgeofiletilecache_p.h>
 #include <QHash>
 #include <QtConcurrent>
@@ -48,14 +49,24 @@ class QGeoFileTileCacheOsm : public QGeoFileTileCache
 {
     Q_OBJECT
 public:
-    QGeoFileTileCacheOsm(const QString &offlineDirectory = QString(), const QString &directory = QString(), QObject *parent = 0);
+    QGeoFileTileCacheOsm(const QVector<QGeoTileProviderOsm *> &providers, const QString &offlineDirectory = QString(), const QString &directory = QString(), QObject *parent = 0);
     ~QGeoFileTileCacheOsm();
 
     QSharedPointer<QGeoTileTexture> get(const QGeoTileSpec &spec) Q_DECL_OVERRIDE;
 
+Q_SIGNALS:
+    void mapDataUpdated(int mapId);
+
+protected Q_SLOTS:
+    void onProviderResolutionFinished(const QGeoTileProviderOsm *provider);
+
 protected:
     void init() Q_DECL_OVERRIDE;
+    QString tileSpecToFilename(const QGeoTileSpec &spec, const QString &format, const QString &directory) const Q_DECL_OVERRIDE;
+    QGeoTileSpec filenameToTileSpec(const QString &filename) const Q_DECL_OVERRIDE;
     QSharedPointer<QGeoTileTexture> getFromOfflineStorage(const QGeoTileSpec &spec);
+    void dropTiles(int mapId);
+    void loadTiles(int mapId);
 
     void initOfflineRegistry();
 
@@ -64,6 +75,8 @@ protected:
     QAtomicInt m_requestCancel;
     QFuture<void> m_future;
     QMutex storageLock;
+    QVector<QGeoTileProviderOsm *> m_providers;
+    QVector<bool> m_highDpi;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
index 152df9b6..f8e18de9 100644
--- a/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
+++ b/src/plugins/geoservices/osm/qgeotiledmappingmanagerengineosm.cpp
@@ -89,13 +89,13 @@ QGeoTiledMappingManagerEngineOsm::QGeoTiledMappingManagerEngineOsm(const QVarian
     QVector<TileProvider *> providers_terrain;
     QVector<TileProvider *> providers_hiking;
     if (highdpi) {
-        providers_street.push_back(new TileProvider(domain + "street-hires"));
-        providers_satellite.push_back(new TileProvider(domain + "satellite-hires"));
-        providers_cycle.push_back(new TileProvider(domain + "cycle-hires"));
-        providers_transit.push_back(new TileProvider(domain + "transit-hires"));
-        providers_nighttransit.push_back(new TileProvider(domain + "night-transit-hires"));
-        providers_terrain.push_back(new TileProvider(domain + "terrain-hires"));
-        providers_hiking.push_back(new TileProvider(domain + "hiking-hires"));
+        providers_street.push_back(new TileProvider(domain + "street-hires", true));
+        providers_satellite.push_back(new TileProvider(domain + "satellite-hires", true));
+        providers_cycle.push_back(new TileProvider(domain + "cycle-hires", true));
+        providers_transit.push_back(new TileProvider(domain + "transit-hires", true));
+        providers_nighttransit.push_back(new TileProvider(domain + "night-transit-hires", true));
+        providers_terrain.push_back(new TileProvider(domain + "terrain-hires", true));
+        providers_hiking.push_back(new TileProvider(domain + "hiking-hires", true));
     }
     providers_street.push_back(new TileProvider(domain + "street"));
     providers_satellite.push_back(new TileProvider(domain + "satellite"));
@@ -226,7 +226,7 @@ QGeoTiledMappingManagerEngineOsm::QGeoTiledMappingManagerEngineOsm(const QVarian
     if (parameters.contains(QStringLiteral("osm.mapping.offline.directory")))
         m_offlineDirectory = parameters.value(QStringLiteral("osm.mapping.offline.directory")).toString();
 
-    QAbstractGeoTileCache *tileCache = new QGeoFileTileCacheOsm(m_offlineDirectory, m_cacheDirectory);
+    QAbstractGeoTileCache *tileCache = new QGeoFileTileCacheOsm(m_providers, m_offlineDirectory, m_cacheDirectory);
 
     // 50mb of disk cache by default to minimize n. of accesses to public OSM servers
     tileCache->setMaxDiskUsage(50 * 1024 * 1024);
@@ -263,7 +263,10 @@ QGeoTiledMappingManagerEngineOsm::~QGeoTiledMappingManagerEngineOsm()
 
 QGeoMap *QGeoTiledMappingManagerEngineOsm::createMap()
 {
-    return new QGeoTiledMapOsm(this);
+    QGeoTiledMap *map = new QGeoTiledMapOsm(this);
+    connect(qobject_cast<QGeoFileTileCacheOsm *>(tileCache()), &QGeoFileTileCacheOsm::mapDataUpdated
+            , map, &QGeoTiledMap::clearScene);
+    return map;
 }
 
 const QVector<QGeoTileProviderOsm *> &QGeoTiledMappingManagerEngineOsm::providers()
diff --git a/src/plugins/geoservices/osm/qgeotilefetcherosm.cpp b/src/plugins/geoservices/osm/qgeotilefetcherosm.cpp
index c7de7cc9..f7c25d61 100644
--- a/src/plugins/geoservices/osm/qgeotilefetcherosm.cpp
+++ b/src/plugins/geoservices/osm/qgeotilefetcherosm.cpp
@@ -43,6 +43,7 @@
 #include <QtNetwork/QNetworkAccessManager>
 #include <QtNetwork/QNetworkRequest>
 #include <QtLocation/private/qgeotilespec_p.h>
+#include <QtLocation/private/qgeotilefetcher_p_p.h>
 
 
 QT_BEGIN_NAMESPACE
@@ -55,10 +56,30 @@ static bool providersResolved(const QVector<QGeoTileProviderOsm *> &providers)
     return true;
 }
 
+class QGeoTileFetcherOsmPrivate : public QGeoTileFetcherPrivate
+{
+    Q_DECLARE_PUBLIC(QGeoTileFetcherOsm)
+public:
+    QGeoTileFetcherOsmPrivate();
+    virtual ~QGeoTileFetcherOsmPrivate();
+
+private:
+    Q_DISABLE_COPY(QGeoTileFetcherOsmPrivate)
+};
+
+QGeoTileFetcherOsmPrivate::QGeoTileFetcherOsmPrivate() : QGeoTileFetcherPrivate()
+{
+}
+
+QGeoTileFetcherOsmPrivate::~QGeoTileFetcherOsmPrivate()
+{
+}
+
+
 QGeoTileFetcherOsm::QGeoTileFetcherOsm(const QVector<QGeoTileProviderOsm *> &providers,
                                        QNetworkAccessManager *nm,
                                        QObject *parent)
-:   QGeoTileFetcher(parent), m_userAgent("Qt Location based application"),
+:   QGeoTileFetcher(*new QGeoTileFetcherOsmPrivate(), parent), m_userAgent("Qt Location based application"),
     m_providers(providers), m_nm(nm), m_ready(true)
 {
     m_nm->setParent(this);
@@ -69,6 +90,8 @@ QGeoTileFetcherOsm::QGeoTileFetcherOsm(const QVector<QGeoTileProviderOsm *> &pro
                     this, &QGeoTileFetcherOsm::onProviderResolutionFinished);
             connect(provider, &QGeoTileProviderOsm::resolutionError,
                     this, &QGeoTileFetcherOsm::onProviderResolutionError);
+            connect(provider, &QGeoTileProviderOsm::resolutionRequired,
+                    this, &QGeoTileFetcherOsm::restartTimer, Qt::QueuedConnection);
             provider->resolveProvider();
         }
     }
@@ -109,6 +132,14 @@ void QGeoTileFetcherOsm::onProviderResolutionError(const QGeoTileProviderOsm *pr
     emit providerDataUpdated(provider);
 }
 
+void QGeoTileFetcherOsm::restartTimer()
+{
+    Q_D(QGeoTileFetcherOsm);
+
+    if (!d->queue_.isEmpty())
+        d->timer_.start(0, this);
+}
+
 QGeoTiledMapReply *QGeoTileFetcherOsm::getTileImage(const QGeoTileSpec &spec)
 {
     int id = spec.mapId();
diff --git a/src/plugins/geoservices/osm/qgeotilefetcherosm.h b/src/plugins/geoservices/osm/qgeotilefetcherosm.h
index c3f33faa..8897d8ed 100644
--- a/src/plugins/geoservices/osm/qgeotilefetcherosm.h
+++ b/src/plugins/geoservices/osm/qgeotilefetcherosm.h
@@ -47,10 +47,12 @@
 QT_BEGIN_NAMESPACE
 
 class QNetworkAccessManager;
+class QGeoTileFetcherOsmPrivate;
 
 class QGeoTileFetcherOsm : public QGeoTileFetcher
 {
     Q_OBJECT
+    Q_DECLARE_PRIVATE(QGeoTileFetcherOsm)
 
     friend class QGeoMapReplyOsm;
     friend class QGeoTiledMappingManagerEngineOsm;
@@ -70,6 +72,7 @@ protected:
 protected Q_SLOTS:
     void onProviderResolutionFinished(const QGeoTileProviderOsm *provider);
     void onProviderResolutionError(const QGeoTileProviderOsm *provider);
+    void restartTimer();
 
 private:
     QGeoTiledMapReply *getTileImage(const QGeoTileSpec &spec);
diff --git a/src/plugins/geoservices/osm/qgeotileproviderosm.cpp b/src/plugins/geoservices/osm/qgeotileproviderosm.cpp
index d0819bd6..0d99c828 100644
--- a/src/plugins/geoservices/osm/qgeotileproviderosm.cpp
+++ b/src/plugins/geoservices/osm/qgeotileproviderosm.cpp
@@ -113,6 +113,13 @@ int QGeoTileProviderOsm::maximumZoomLevel() const
     return m_provider->maximumZoomLevel();
 }
 
+bool QGeoTileProviderOsm::isHighDpi() const
+{
+    if (!m_provider)
+        return false;
+    return m_provider->isHighDpi();
+}
+
 const QGeoMapType &QGeoTileProviderOsm::mapType() const
 {
     return m_mapType;
@@ -183,8 +190,9 @@ void QGeoTileProviderOsm::onResolutionError(TileProvider *provider)
                 m_provider = p;
                 if (!p->isValid()) {
                     m_status = Idle;
-                    //m_status = Resolving;
-                    //p->resolveProvider();
+//                    m_status = Resolving;
+//                    p->resolveProvider();
+                    emit resolutionRequired();
                 }
                 break;
             }
@@ -230,12 +238,13 @@ static void sort2(int &a, int &b)
     }
 }
 
-TileProvider::TileProvider() : m_status(Invalid), m_nm(nullptr)
+TileProvider::TileProvider() : m_status(Invalid), m_nm(nullptr), m_highDpi(false)
 {
 
 }
 
-TileProvider::TileProvider(const QUrl &urlRedirector) : m_status(Idle), m_urlRedirector(urlRedirector), m_nm(nullptr)
+TileProvider::TileProvider(const QUrl &urlRedirector, bool highDpi)
+:   m_status(Idle), m_urlRedirector(urlRedirector), m_nm(nullptr), m_highDpi(highDpi)
 {
     if (!m_urlRedirector.isValid())
         m_status = Invalid;
@@ -245,11 +254,12 @@ TileProvider::TileProvider(const QString &urlTemplate,
                            const QString &format,
                            const QString &copyRightMap,
                            const QString &copyRightData,
+                           bool highDpi,
                            int minimumZoomLevel,
                            int maximumZoomLevel)
 :   m_status(Invalid), m_nm(nullptr), m_urlTemplate(urlTemplate),
     m_format(format), m_copyRightMap(copyRightMap), m_copyRightData(copyRightData),
-    m_minimumZoomLevel(minimumZoomLevel), m_maximumZoomLevel(maximumZoomLevel)
+    m_minimumZoomLevel(minimumZoomLevel), m_maximumZoomLevel(maximumZoomLevel), m_highDpi(highDpi)
 {
     setupProvider();
 }
@@ -299,7 +309,7 @@ void TileProvider::handleError(QNetworkReply::NetworkError error)
         // prevent accessing the redirection info but not the actual providers.
         m_status = Invalid;
     default:
-        qWarning() << "QGeoTileProviderOsm network error:" << error;
+        //qWarning() << "QGeoTileProviderOsm network error:" << error;
         break;
     }
 }
@@ -543,6 +553,11 @@ int TileProvider::maximumZoomLevel() const
     return m_maximumZoomLevel;
 }
 
+bool TileProvider::isHighDpi() const
+{
+    return m_highDpi;
+}
+
 void TileProvider::setStyleCopyRight(const QString &copyright)
 {
     m_copyRightStyle = copyright;
diff --git a/src/plugins/geoservices/osm/qgeotileproviderosm.h b/src/plugins/geoservices/osm/qgeotileproviderosm.h
index 3e887965..cea832f0 100644
--- a/src/plugins/geoservices/osm/qgeotileproviderosm.h
+++ b/src/plugins/geoservices/osm/qgeotileproviderosm.h
@@ -62,12 +62,13 @@ public:
 
     TileProvider();
     // "Online" constructor. Needs resolution to fetch the parameters
-    TileProvider(const QUrl &urlRedirector);
+    TileProvider(const QUrl &urlRedirector, bool highDpi = false);
     // Offline constructor. Doesn't need URLRedirector and networkmanager
     TileProvider(const QString &urlTemplate,
              const QString &format,
              const QString &copyRightMap,
              const QString &copyRightData,
+             bool highDpi = false,
              int minimumZoomLevel = 0,
              int maximumZoomLevel = 19);
 
@@ -89,6 +90,7 @@ public:
     inline QString format() const;
     inline int minimumZoomLevel() const;
     inline int maximumZoomLevel() const;
+    inline bool isHighDpi() const;
     QUrl tileAddress(int x, int y, int z) const;
 
     // Optional properties, not needed to construct a provider
@@ -106,6 +108,7 @@ public:
     QString m_urlSuffix;
     int m_minimumZoomLevel;
     int m_maximumZoomLevel;
+    bool m_highDpi;
 
     int paramsLUT[3];  //Lookup table to handle possibly shuffled x,y,z
     QString paramsSep[2]; // what goes in between %x, %y and %z
@@ -145,6 +148,7 @@ public:
     QString format() const;
     int minimumZoomLevel() const;
     int maximumZoomLevel() const;
+    bool isHighDpi() const;
     const QGeoMapType &mapType() const;
     bool isValid() const;
     bool isResolved() const;
@@ -152,6 +156,7 @@ public:
 Q_SIGNALS:
     void resolutionFinished(const QGeoTileProviderOsm *provider);
     void resolutionError(const QGeoTileProviderOsm *provider);
+    void resolutionRequired();
 
 public Q_SLOTS:
     void resolveProvider();