Fixed issue preventing catalogue from being updated due to http cache.

slkh · Mar 30, 2022 · 9aafbbb · 9aafbbb
1 parent 18ec2d0
commit 9aafbbb
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/camel_tools/data/downloader.py b/camel_tools/data/downloader.py
@@ -32,6 +32,7 @@
 import zipfile
 
 import requests
+from requests.structures import CaseInsensitiveDict
 
 
 _STREAM_CHUNK_SIZE = 32768
@@ -111,7 +112,13 @@ def _save_content(url,
 
         try:
             session = requests.Session()
-            response = session.get(url, stream=True)
+
+            headers = CaseInsensitiveDict()
+            headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+            headers["Pragma"] = "no-cache"
+            headers["Expires"] = "0"
+
+            response = session.get(url, stream=True, headers=headers)
 
             curr_size = 0
             total_size = int(response.headers.get('content-length', 0))