Force update script to download the image and calculate SHA512

Previously the precomputed SHA256 was used for most images.
Since we want to check the hash against the openstack backend,
we need to have the SHA512. Sadly, most images creators do not
provide us with that hash pre-computed.

The only solution is to download the images and computing
the hash at runtime.

Signed-off-by: Gondermann <[email protected]>

openstack_image_manager/update.py

@@ -2,6 +2,8 @@
 # source of latest URLs: https://gitlab.com/libosinfo/osinfo-db
 
 from datetime import datetime
+import hashlib
+import math
 import os
 import re
 import shutil
@@ -157,6 +159,39 @@ def mirror_image(
         client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename)
         os.remove(filename)
 
+def size_clean(size):
+    size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
+    i = int(math.floor(math.log(size, 1024)))
+    s = size / 1024 ** i
+    return f"{s:.2f} {size_name[i]}"
+
+def headers_and_sha512(download_url: str):
+    hash_obj = hashlib.new("sha512")
+
+    file_headers = None
+    with requests.get(url=download_url, stream=True, timeout=30) as response:
+        if response.status_code != 200:
+            logger.error(f"Downloading image '{download_url}' failed with error code {response.status_code}")
+            return None, None
+
+        file_headers = response.headers
+        file_size = int(file_headers["Content-Length"])
+        logger.info(f"Image size {size_clean(file_size)}")
+
+        downloadedBytes = 0
+        lastProgress = 0
+        for chunk in response.iter_content(chunk_size=8192):
+            downloadedBytes += 8192
+            progressPercent = (downloadedBytes / file_size) * 100
+            progress = round(min(max(progressPercent, 0), 100))
+            if progress - lastProgress >= 5:
+                logger.info(f"Downloading image: {progress}%")
+                lastProgress = progress
+
+            hash_obj.update(chunk)
+
+    sha512 = hash_obj.hexdigest()
+    return file_headers, f"sha512:{sha512}"
 
 def update_image(image, getter, minio_server, minio_bucket, minio_access_key, minio_secret_key):
     name = image["name"]
@@ -181,6 +216,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi
                 "checksum": None,
                 "url": None,
                 "version": None,
+                "verify_checksum": None
             }
         )
 
@@ -191,12 +227,15 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi
         logger.info(f"Image {name} is up-to-date, nothing to do")
         return 0
 
+    headers, verify_checksum = headers_and_sha512(current_url)
+    if verify_checksum == None:
+        return 0
+
     if current_version is None:
         logger.info(f"Checking {current_url}")
 
-        conn = urlopen(current_url, timeout=30)
         dt = datetime.strptime(
-            conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
+            headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
         )
         current_version = dt.strftime("%Y%m%d")
 
@@ -205,6 +244,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi
         "build_date": datetime.strptime(current_version, "%Y%m%d").date(),
         "checksum": current_checksum,
         "url": current_url,
+        "verify_checksum": verify_checksum,
     }
     logger.info(f"New values are {new_values}")
     image["versions"][0].update(new_values)