support multiple URLs/sources for vector tiles (#58155)

* support multiple URLs/sources for vector tiles

* fix ++ op

* fix test

* address review

* review suggestions

* fix clazy warnings

* fix clazy warnings

* again

src/analysis/processing/qgsalgorithmdownloadvectortiles.cpp

@@ -193,10 +193,14 @@ QVariantMap QgsDownloadVectorTilesAlgorithm::processAlgorithm( const QVariantMap
       if ( feedback->isCanceled() )
         break;
 
-      if ( !rawTile.data.isEmpty() )
+      // TODO: at the moment, it handles single source only of tiles
+      // takes the first one
+      const QByteArray data = rawTile.data.first();
+
+      if ( !data.isEmpty() )
       {
         QByteArray gzipTileData;
-        QgsZipUtils::encodeGzip( rawTile.data, gzipTileData );
+        QgsZipUtils::encodeGzip( data, gzipTileData );
         int rowTMS = pow( 2, rawTile.id.zoomLevel() ) - rawTile.id.row() - 1;
         writer->setTileData( rawTile.id.zoomLevel(), rawTile.id.column(), rowTMS, gzipTileData );
       }

src/core/vectortile/qgsvectortiledataprovider.cpp

@@ -20,6 +20,12 @@
 #include <QNetworkRequest>
 #include <QImage>
 
+
+int QgsVectorTileDataProvider::DATA_COLUMN = QNetworkRequest::User + 1;
+int QgsVectorTileDataProvider::DATA_ROW = QNetworkRequest::User + 2;
+int QgsVectorTileDataProvider::DATA_ZOOM = QNetworkRequest::User + 3;
+int QgsVectorTileDataProvider::DATA_SOURCE_ID = QNetworkRequest::User + 4;
+
 QgsVectorTileDataProvider::QgsVectorTileDataProvider(
   const QString &uri,
   const ProviderOptions &options,
@@ -69,11 +75,11 @@ bool QgsVectorTileDataProvider::supportsAsync() const
   return false;
 }
 
-QNetworkRequest QgsVectorTileDataProvider::tileRequest( const QgsTileMatrixSet &, const QgsTileXYZ &, Qgis::RendererUsage ) const
+QList<QNetworkRequest> QgsVectorTileDataProvider::tileRequests( const QgsTileMatrixSet &, const QgsTileXYZ &, Qgis::RendererUsage ) const
 {
   QGIS_PROTECT_QOBJECT_THREAD_ACCESS
 
-  return QNetworkRequest();
+  return QList<QNetworkRequest>();
 }
 
 QVariantMap QgsVectorTileDataProvider::styleDefinition() const

src/core/vectortile/qgsvectortiledataprovider.h

@@ -42,6 +42,7 @@ class QgsVectorTileMatrixSet;
 class QgsVectorTileDataProviderSharedData
 {
   public:
+
     QgsVectorTileDataProviderSharedData();
 
     /**
@@ -77,6 +78,15 @@ class CORE_EXPORT QgsVectorTileDataProvider : public QgsDataProvider
 
   public:
 
+    //! Role to set column attribute in the request so it can be retrieved later
+    static int DATA_COLUMN;
+    //! Role to set row attribute in the request so it can be retrieved later
+    static int DATA_ROW;
+    //! Role to set zoom attribute in the request so it can be retrieved later
+    static int DATA_ZOOM;
+    //! Role to set source ID attribute in the request so it can be retrieved later
+    static int DATA_SOURCE_ID;
+
     /**
      * Constructor for QgsVectorTileDataProvider, with the specified \a uri.
      */
@@ -107,6 +117,17 @@ class CORE_EXPORT QgsVectorTileDataProvider : public QgsDataProvider
      */
     virtual QString sourcePath() const = 0;
 
+
+    /**
+     * Returns the list of source paths for the data.
+     * \since QGIS 3.40
+     */
+    virtual QgsStringMap sourcePaths() const
+    {
+      return { { QString(), sourcePath() } };
+    }
+
+
     /**
      * Returns a clone of the data provider.
      */
@@ -139,7 +160,7 @@ class CORE_EXPORT QgsVectorTileDataProvider : public QgsDataProvider
      *
      * The default implementation returns an invalid request.
      */
-    virtual QNetworkRequest tileRequest( const QgsTileMatrixSet &tileMatrixSet, const QgsTileXYZ &id, Qgis::RendererUsage usage ) const;
+    virtual QList<QNetworkRequest> tileRequests( const QgsTileMatrixSet &tileMatrixSet, const QgsTileXYZ &id, Qgis::RendererUsage usage ) const;
 
     /**
      * Returns the style definition for the provider, if available.

src/core/vectortile/qgsvectortileloader.cpp

@@ -67,7 +67,9 @@ void QgsVectorTileLoader::downloadBlocking()
     return; // nothing to do
   }
 
-  QgsDebugMsgLevel( QStringLiteral( "Starting event loop with %1 requests" ).arg( mReplies.count() ), 2 );
+  int repliesCount = std::accumulate( mReplies.constBegin(), mReplies.constEnd(), 0, []( int count, QList<QgsTileDownloadManagerReply *> replies ) {return count + replies.count();} );
+  Q_UNUSED( repliesCount )
+  QgsDebugMsgLevel( QStringLiteral( "Starting event loop with %1 requests" ).arg( repliesCount ), 2 );
 
   mEventLoop->exec( QEventLoop::ExcludeUserInputEvents );
 
@@ -78,20 +80,25 @@ void QgsVectorTileLoader::downloadBlocking()
 
 void QgsVectorTileLoader::loadFromNetworkAsync( const QgsTileXYZ &id, const QgsTileMatrixSet &tileMatrixSet, const QgsVectorTileDataProvider *provider, Qgis::RendererUsage usage )
 {
-  QNetworkRequest request = provider->tileRequest( tileMatrixSet, id, usage );
+  const QList<QNetworkRequest> requests = provider->tileRequests( tileMatrixSet, id, usage );
 
-  QgsTileDownloadManagerReply *reply = QgsApplication::tileDownloadManager()->get( request );
-  connect( reply, &QgsTileDownloadManagerReply::finished, this, &QgsVectorTileLoader::tileReplyFinished );
-  mReplies << reply;
+  for ( const QNetworkRequest &request : requests )
+  {
+    QgsTileDownloadManagerReply *reply = QgsApplication::tileDownloadManager()->get( request );
+    connect( reply, &QgsTileDownloadManagerReply::finished, this, &QgsVectorTileLoader::tileReplyFinished );
+    mReplies[id].append( reply );
+  }
 }
 
 void QgsVectorTileLoader::tileReplyFinished()
 {
   QgsTileDownloadManagerReply *reply = qobject_cast<QgsTileDownloadManagerReply *>( sender() );
 
-  int reqX = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 1 ) ).toInt();
-  int reqY = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 2 ) ).toInt();
-  int reqZ = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QNetworkRequest::User + 3 ) ).toInt();
+  int reqX = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QgsVectorTileDataProvider::DATA_COLUMN ) ).toInt();
+  int reqY = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QgsVectorTileDataProvider::DATA_ROW ) ).toInt();
+  int reqZ = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QgsVectorTileDataProvider::DATA_ZOOM ) ).toInt();
+  QString sourceId = reply->request().attribute( static_cast<QNetworkRequest::Attribute>( QgsVectorTileDataProvider::DATA_SOURCE_ID ) ).toString();
+
   QgsTileXYZ tileID( reqX, reqY, reqZ );
 
   if ( reply->error() == QNetworkReply::NoError )
@@ -100,10 +107,15 @@ void QgsVectorTileLoader::tileReplyFinished()
 
     QgsDebugMsgLevel( QStringLiteral( "Tile download successful: " ) + tileID.toString(), 2 );
     QByteArray rawData = reply->data();
-    mReplies.removeOne( reply );
+    mReplies[tileID].removeOne( reply );
+    mPendingRawData[tileID][sourceId] = rawData;
     reply->deleteLater();
 
-    emit tileRequestFinished( QgsVectorTileRawData( tileID, rawData ) );
+    if ( mReplies[tileID].count() == 0 )
+    {
+      mReplies.remove( tileID );
+      emit tileRequestFinished( QgsVectorTileRawData( tileID, mPendingRawData.take( tileID ) ) );
+    }
   }
   else
   {
@@ -116,10 +128,14 @@ void QgsVectorTileLoader::tileReplyFinished()
     }
 
     QgsDebugError( QStringLiteral( "Tile download failed! " ) + reply->errorString() );
-    mReplies.removeOne( reply );
+    mReplies[tileID].removeOne( reply );
     reply->deleteLater();
 
-    emit tileRequestFinished( QgsVectorTileRawData( tileID, QByteArray() ) );
+    if ( mReplies[tileID].count() == 0 )
+    {
+      mReplies.remove( tileID );
+      emit tileRequestFinished( QgsVectorTileRawData( tileID ) );
+    }
   }
 
   if ( mReplies.isEmpty() )
@@ -131,8 +147,12 @@ void QgsVectorTileLoader::tileReplyFinished()
 
 void QgsVectorTileLoader::canceled()
 {
-  QgsDebugMsgLevel( QStringLiteral( "Canceling %1 pending requests" ).arg( mReplies.count() ), 2 );
-  qDeleteAll( mReplies );
+  int repliesCount = std::accumulate( mReplies.constBegin(), mReplies.constEnd(), 0, []( int count, QList<QgsTileDownloadManagerReply *> replies ) {return count + replies.count();} );
+  Q_UNUSED( repliesCount )
+  QgsDebugMsgLevel( QStringLiteral( "Canceling %1 pending requests" ).arg( repliesCount ), 2 );
+  QHash<QgsTileXYZ, QList<QgsTileDownloadManagerReply *>>::iterator it = mReplies.begin();
+  for ( ; it != mReplies.end(); ++it )
+    qDeleteAll( it.value() );
   mReplies.clear();
 
   // stop blocking download

src/core/vectortile/qgsvectortileloader.h

@@ -28,18 +28,23 @@ class QByteArray;
 class QNetworkReply;
 class QEventLoop;
 
+
 /**
  * \ingroup core
- * \brief Keeps track of raw tile data that need to be decoded
+ * \brief Keeps track of raw tile data from one or more sources that need to be decoded
  *
  * \since QGIS 3.14
  */
 class CORE_EXPORT QgsVectorTileRawData
 {
   public:
-    //! Constructs a raw tile object
-    QgsVectorTileRawData( QgsTileXYZ tileID = QgsTileXYZ(), const QByteArray &raw = QByteArray() )
-      : id( tileID ), tileGeometryId( tileID ), data( raw ) {}
+    //! Constructs a raw tile object for single source
+    QgsVectorTileRawData( QgsTileXYZ tileID = QgsTileXYZ(), const QByteArray &data = QByteArray() )
+      : id( tileID ), tileGeometryId( tileID ), data( { { QString(), data } } ) {}
+
+    //! Constructs a raw tile object for one or more sources
+    QgsVectorTileRawData( QgsTileXYZ tileID, const QMap<QString, QByteArray> &data )
+      : id( tileID ), tileGeometryId( tileID ), data( data ) {}
 
     //! Tile position in tile matrix set
     QgsTileXYZ id;
@@ -54,8 +59,8 @@ class CORE_EXPORT QgsVectorTileRawData
      */
     QgsTileXYZ tileGeometryId;
 
-    //! Raw tile data
-    QByteArray data;
+    //! Raw tile data by source ID
+    QMap<QString, QByteArray> data;
 };
 
 
@@ -113,7 +118,10 @@ class CORE_EXPORT QgsVectorTileLoader : public QObject
     QgsFeedback *mFeedback;
 
     //! Running tile requests
-    QList<QgsTileDownloadManagerReply *> mReplies;
+    QHash<QgsTileXYZ, QList<QgsTileDownloadManagerReply *>> mReplies;
+
+    //! Raw data is stored until all sources are fetched
+    QHash<QgsTileXYZ, QMap<QString, QByteArray>> mPendingRawData;
 
     QString mError;
 };