Improve whdload file cache (#90)

* redownload whdload.xml only if Etag header indicates it * keep postData in variable (IGDB)
Gemba · Nov 26, 2024 · bcc22df · bcc22df
1 parent 005240b
commit bcc22df
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 11 deletions.
diff --git a/src/igdb.cpp b/src/igdb.cpp
@@ -70,13 +70,11 @@ void Igdb::getSearchResults(QList<GameEntry> &gameEntries, QString searchName,
     // Request list of games but don't allow re-releases ("game.version_parent =
     // null")
     limiter.exec();
-    netComm->request(baseUrl + "/search/",
-                     "fields "
-                     "game.name,game.platforms.name,game.release_dates.date,"
-                     "game.release_dates.platform; search \"" +
-                         searchName +
-                         "\"; where game != null & game.version_parent = null;",
-                     headers);
+    const QString postData =
+        "fields game.name,game.platforms.name,game.release_dates.date,"
+        "game.release_dates.platform; search \"" +
+        searchName + "\"; where game != null & game.version_parent = null;";
+    netComm->request(baseUrl + "/search/", postData, headers);
     q.exec();
     data = netComm->getData();
 

diff --git a/src/netcomm.cpp b/src/netcomm.cpp
@@ -25,6 +25,7 @@
 
 #include "netcomm.h"
 
+#include <QDebug>
 #include <QNetworkRequest>
 #include <QUrl>
 
@@ -43,8 +44,6 @@ void NetComm::request(QString query, QString postData,
     request.setHeader(QNetworkRequest::UserAgentHeader,
                       "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) "
                       "Gecko/20100101 Firefox/74.0");
-    request.setHeader(QNetworkRequest::ContentTypeHeader,
-                      "application/x-www-form-urlencoded");
 
     if (!headers.isEmpty()) {
         for (const auto &header : headers) {
@@ -53,8 +52,15 @@ void NetComm::request(QString query, QString postData,
     }
 
     if (postData.isNull()) {
+        // GET
+        // No body -> no Content-Type
         reply = manager->getRequest(request);
+    } else if (postData == "HEAD") {
+        reply = manager->headRequest(request);
     } else {
+        // POST
+        request.setHeader(QNetworkRequest::ContentTypeHeader,
+                          "application/x-www-form-urlencoded");
         reply = manager->postRequest(request, postData.toUtf8());
     }
     connect(reply, &QNetworkReply::finished, this, &NetComm::replyReady);
@@ -69,12 +75,23 @@ void NetComm::replyReady() {
     error = reply->error();
     contentType = reply->rawHeader("Content-Type");
     redirUrl = reply->rawHeader("Location");
+    headerPairs = reply->rawHeaderPairs();
     reply->deleteLater();
     emit dataReady();
 }
 
 QByteArray NetComm::getData() { return data; }
 
+QString NetComm::getHeaderValue(const QString headerKey) {
+    for (const auto &h : headerPairs) {
+        if (h.first == headerKey.toUtf8()) {
+            qDebug() << h.first << ":" << h.second;
+            return QString(h.second);
+        }
+    }
+    return "";
+}
+
 QNetworkReply::NetworkError NetComm::getError(const int &verbosity) {
     if (error != QNetworkReply::NoError && verbosity >= 1) {
         switch (error) {

diff --git a/src/netcomm.h b/src/netcomm.h
@@ -43,6 +43,7 @@ class NetComm : public QObject {
     QNetworkReply::NetworkError getError(const int &verbosity = 0);
     QByteArray getContentType();
     QByteArray getRedirUrl();
+    QString getHeaderValue(const QString headerKey);
 
 private slots:
     void replyReady();
@@ -60,6 +61,7 @@ private slots:
     QByteArray contentType;
     QByteArray redirUrl;
     QNetworkReply *reply;
+    QList<QNetworkReply::RawHeaderPair> headerPairs;
 };
 
 #endif // NETCOMM_H
diff --git a/src/netmanager.cpp b/src/netmanager.cpp
@@ -34,6 +34,12 @@ QNetworkReply *NetManager::getRequest(const QNetworkRequest &request) {
     return get(request);
 }
 
+
+QNetworkReply *NetManager::headRequest(const QNetworkRequest &request) {
+    QMutexLocker locker(&requestMutex);
+    return head(request);
+}
+
 QNetworkReply *NetManager::postRequest(const QNetworkRequest &request,
                                        const QByteArray &data) {
     QMutexLocker locker(&requestMutex);

diff --git a/src/netmanager.h b/src/netmanager.h
@@ -36,6 +36,7 @@ class NetManager : public QNetworkAccessManager {
 public:
     NetManager();
     QNetworkReply *getRequest(const QNetworkRequest &request);
+    QNetworkReply *headRequest(const QNetworkRequest &request);
     QNetworkReply *postRequest(const QNetworkRequest &request,
                                const QByteArray &data);
 

diff --git a/src/skyscraper.cpp b/src/skyscraper.cpp
@@ -1084,9 +1084,31 @@ void Skyscraper::prepareScraping() {
 }
 
 void Skyscraper::updateWhdloadDb(NetComm &netComm, QEventLoop &q) {
+    const QString url = "https://raw.githubusercontent.com/HoraceAndTheSpider/"
+                        "Amiberry-XML-Builder/master/whdload_db.xml";
+    netComm.request(url, "HEAD");
+    q.exec();
+
+    QString etag = netComm.getHeaderValue("ETag");
+    QString cachedEtag;
+    QFile whdlEtagFile(Config::getSkyFolder(Config::SkyFolderType::LOG) %
+                       "/whdload_cached_etag.txt");
+    if (whdlEtagFile.open(QIODevice::ReadOnly)) {
+        cachedEtag = QString(whdlEtagFile.readLine());
+        whdlEtagFile.close();
+    }
+    if (!etag.isEmpty() && etag == cachedEtag) {
+        // not expired, no need to download again
+        return;
+    }
+
+    if (!etag.isEmpty() && whdlEtagFile.open(QIODevice::WriteOnly)) {
+        whdlEtagFile.write(etag.toUtf8());
+        whdlEtagFile.close();
+    }
+
     printf("Fetching 'whdload_db.xml', just a sec...");
-    netComm.request("https://raw.githubusercontent.com/HoraceAndTheSpider/"
-                    "Amiberry-XML-Builder/master/whdload_db.xml");
+    netComm.request(url);
     q.exec();
     QByteArray data = netComm.getData();
     QDomDocument tempDoc;