diff --git a/openstack_image_manager/update.py b/openstack_image_manager/update.py index 46469a1e..2edf7ee6 100644 --- a/openstack_image_manager/update.py +++ b/openstack_image_manager/update.py @@ -2,6 +2,8 @@ # source of latest URLs: https://gitlab.com/libosinfo/osinfo-db from datetime import datetime +import hashlib +import math import os import re import shutil @@ -157,6 +159,39 @@ def mirror_image( client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename) os.remove(filename) +def size_clean(size): + size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB") + i = int(math.floor(math.log(size, 1024))) + s = size / 1024 ** i + return f"{s:.2f} {size_name[i]}" + +def headers_and_sha512(download_url: str): + hash_obj = hashlib.new("sha512") + + file_headers = None + with requests.get(url=download_url, stream=True, timeout=30) as response: + if response.status_code != 200: + logger.error(f"Downloading image '{download_url}' failed with error code {response.status_code}") + return None, None + + file_headers = response.headers + file_size = int(file_headers["Content-Length"]) + logger.info(f"Image size {size_clean(file_size)}") + + downloadedBytes = 0 + lastProgress = 0 + for chunk in response.iter_content(chunk_size=8192): + downloadedBytes += 8192 + progressPercent = (downloadedBytes / file_size) * 100 + progress = round(min(max(progressPercent, 0), 100)) + if progress - lastProgress >= 5: + logger.info(f"Downloading image: {progress}%") + lastProgress = progress + + hash_obj.update(chunk) + + sha512 = hash_obj.hexdigest() + return file_headers, f"sha512:{sha512}" def update_image(image, getter, minio_server, minio_bucket, minio_access_key, minio_secret_key): name = image["name"] @@ -181,6 +216,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi "checksum": None, "url": None, "version": None, + "verify_checksum": None } ) @@ -191,12 +227,15 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi logger.info(f"Image {name} is up-to-date, nothing to do") return 0 + headers, verify_checksum = headers_and_sha512(current_url) + if verify_checksum == None: + return 0 + if current_version is None: logger.info(f"Checking {current_url}") - conn = urlopen(current_url, timeout=30) dt = datetime.strptime( - conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" + headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" ) current_version = dt.strftime("%Y%m%d") @@ -205,6 +244,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi "build_date": datetime.strptime(current_version, "%Y%m%d").date(), "checksum": current_checksum, "url": current_url, + "verify_checksum": verify_checksum, } logger.info(f"New values are {new_values}") image["versions"][0].update(new_values)