From ff74ed5b346f5b12273b4900667bcb4d12dc031e Mon Sep 17 00:00:00 2001 From: Gondermann Date: Thu, 10 Aug 2023 16:43:14 +0200 Subject: [PATCH] Force update script to download the image and calculate SHA512 Previously the precomputed SHA256 was used for most images. Since we want to check the hash against the openstack backend, we need to have the SHA512. Sadly, most images creators do not provide us with that hash pre-computed. The only solution is to download the images and computing the hash at runtime. Signed-off-by: Gondermann --- openstack_image_manager/update.py | 76 +++++++++++++++++-------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/openstack_image_manager/update.py b/openstack_image_manager/update.py index 51515af7..53697edf 100644 --- a/openstack_image_manager/update.py +++ b/openstack_image_manager/update.py @@ -7,8 +7,9 @@ import shutil import sys import time +import hashlib +import math from urllib.parse import urlparse -from urllib.request import urlopen from loguru import logger from minio import Minio @@ -72,6 +73,11 @@ def mirror_image( client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename) os.remove(filename) +def size_clean(size): + size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB") + i = int(math.floor(math.log(size, 1024))) + s = size / 1024 ** i + return f"{s:.2f} {size_name[i]}" def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secret_key): name = image["name"] @@ -87,54 +93,54 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr logger.info(f"Getting checksums from {latest_checksum_url}") result = requests.get(latest_checksum_url) - checksums = {} - checksum_type = "sha256" + hash_obj = hashlib.new("sha512") + file_headers = None + with requests.get(url=latest_url, stream=True, timeout=30) as response: + if response.status_code != 200: + logger.error(f"Downloading image '{name}' failed with error code {response.status_code}") + return None + + file_headers = response.headers + file_size = int(file_headers["Content-Length"]) + logger.info(f"Image size {size_clean(file_size)}") + + downloadedBytes = 0 + lastProgress = 0 + for chunk in response.iter_content(chunk_size=8192): + downloadedBytes += 8192 + progressPercent = (downloadedBytes / file_size) * 100 + progress = round(min(max(progressPercent, 0), 100)) + if progress - lastProgress >= 5: + logger.info(f"Downloading image: {progress}%") + lastProgress = progress + + hash_obj.update(chunk) + + sha512_value = hash_obj.hexdigest() + filename_pattern = None if image["shortname"] in ["centos-stream-8", "centos-stream-9", "centos-7"]: filename_pattern = latest_filename.replace("HEREBE", "") filename_pattern = filename_pattern.replace("DRAGONS", "") - elif image["shortname"] in ["debian-10", "debian-11", "debian-12"]: - checksum_type = "sha512" + new_latest_filename_list = [] for line in result.text.split("\n"): - if image["shortname"] in ["rocky-8", "rocky-9"]: - splitted_line = re.split("\s+", line) # noqa W605 - if splitted_line[0] == "SHA256": - checksums[latest_filename] = splitted_line[3] - elif image["shortname"] in [ - "ubuntu-14.04", - "ubuntu-16.04", - "ubuntu-16.04-minimal", - "ubuntu-18.04", - "ubuntu-18.04-minimal", - "ubuntu-20.04", - "ubuntu-20.04-minimal", - "ubuntu-22.04", - "ubuntu-22.04-minimal", - ]: - splitted_line = re.split("\s+", line) # noqa W605 - if len(splitted_line) == 2: - checksums[splitted_line[1][1:]] = splitted_line[0] - elif image["shortname"] in ["centos-7"]: + if image["shortname"] in ["centos-7"]: splitted_line = re.split("\s+", line) # noqa W605 if len(splitted_line) == 2: if re.search(filename_pattern, splitted_line[1]): - checksums[splitted_line[1]] = splitted_line[0] + new_latest_filename_list.append(splitted_line[1]) elif image["shortname"] in ["centos-stream-8", "centos-stream-9"]: splitted_line = re.split("\s+", line) # noqa W605 if splitted_line[0] == "SHA256" and re.search( filename_pattern, splitted_line[1][1:-1] ): - checksums[splitted_line[1][1:-1]] = splitted_line[3] - else: - splitted_line = re.split("\s+", line) # noqa W605 - if len(splitted_line) == 2: - checksums[splitted_line[1]] = splitted_line[0] + new_latest_filename_list.append(splitted_line[1][1:-1]) if filename_pattern: - new_latest_filename = natsorted(checksums.keys())[-1] + new_latest_filename = natsorted(new_latest_filename_list)[-1] new_latest_url = latest_url.replace(latest_filename, new_latest_filename) logger.info(f"Latest URL is now {new_latest_url}") @@ -143,7 +149,7 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr latest_filename = new_latest_filename latest_url = new_latest_url - current_checksum = f"{checksum_type}:{checksums[latest_filename]}" + current_checksum = f"sha512:{sha512_value}" logger.info(f"Checksum of current {latest_filename} is {current_checksum}") try: @@ -165,9 +171,8 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr if latest_checksum != current_checksum: logger.info(f"Checking {latest_url}") - conn = urlopen(latest_url, timeout=30) struct = time.strptime( - conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" + file_headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" ) dt = datetime.fromtimestamp(time.mktime(struct)) @@ -250,7 +255,8 @@ def main( minio_access_key, minio_secret_key, ) - data["images"][index] = updated_image + if updated_image: + data["images"][index] = updated_image with open(p, "w+") as fp: ryaml = ruamel.yaml.YAML()