diff --git a/openstack_image_manager/update.py b/openstack_image_manager/update.py index 51515af7..53697edf 100644 --- a/openstack_image_manager/update.py +++ b/openstack_image_manager/update.py @@ -7,8 +7,9 @@ import shutil import sys import time +import hashlib +import math from urllib.parse import urlparse -from urllib.request import urlopen from loguru import logger from minio import Minio @@ -72,6 +73,11 @@ def mirror_image( client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename) os.remove(filename) +def size_clean(size): + size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB") + i = int(math.floor(math.log(size, 1024))) + s = size / 1024 ** i + return f"{s:.2f} {size_name[i]}" def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secret_key): name = image["name"] @@ -87,54 +93,54 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr logger.info(f"Getting checksums from {latest_checksum_url}") result = requests.get(latest_checksum_url) - checksums = {} - checksum_type = "sha256" + hash_obj = hashlib.new("sha512") + file_headers = None + with requests.get(url=latest_url, stream=True, timeout=30) as response: + if response.status_code != 200: + logger.error(f"Downloading image '{name}' failed with error code {response.status_code}") + return None + + file_headers = response.headers + file_size = int(file_headers["Content-Length"]) + logger.info(f"Image size {size_clean(file_size)}") + + downloadedBytes = 0 + lastProgress = 0 + for chunk in response.iter_content(chunk_size=8192): + downloadedBytes += 8192 + progressPercent = (downloadedBytes / file_size) * 100 + progress = round(min(max(progressPercent, 0), 100)) + if progress - lastProgress >= 5: + logger.info(f"Downloading image: {progress}%") + lastProgress = progress + + hash_obj.update(chunk) + + sha512_value = hash_obj.hexdigest() + filename_pattern = None if image["shortname"] in ["centos-stream-8", "centos-stream-9", "centos-7"]: filename_pattern = latest_filename.replace("HEREBE", "") filename_pattern = filename_pattern.replace("DRAGONS", "") - elif image["shortname"] in ["debian-10", "debian-11", "debian-12"]: - checksum_type = "sha512" + new_latest_filename_list = [] for line in result.text.split("\n"): - if image["shortname"] in ["rocky-8", "rocky-9"]: - splitted_line = re.split("\s+", line) # noqa W605 - if splitted_line[0] == "SHA256": - checksums[latest_filename] = splitted_line[3] - elif image["shortname"] in [ - "ubuntu-14.04", - "ubuntu-16.04", - "ubuntu-16.04-minimal", - "ubuntu-18.04", - "ubuntu-18.04-minimal", - "ubuntu-20.04", - "ubuntu-20.04-minimal", - "ubuntu-22.04", - "ubuntu-22.04-minimal", - ]: - splitted_line = re.split("\s+", line) # noqa W605 - if len(splitted_line) == 2: - checksums[splitted_line[1][1:]] = splitted_line[0] - elif image["shortname"] in ["centos-7"]: + if image["shortname"] in ["centos-7"]: splitted_line = re.split("\s+", line) # noqa W605 if len(splitted_line) == 2: if re.search(filename_pattern, splitted_line[1]): - checksums[splitted_line[1]] = splitted_line[0] + new_latest_filename_list.append(splitted_line[1]) elif image["shortname"] in ["centos-stream-8", "centos-stream-9"]: splitted_line = re.split("\s+", line) # noqa W605 if splitted_line[0] == "SHA256" and re.search( filename_pattern, splitted_line[1][1:-1] ): - checksums[splitted_line[1][1:-1]] = splitted_line[3] - else: - splitted_line = re.split("\s+", line) # noqa W605 - if len(splitted_line) == 2: - checksums[splitted_line[1]] = splitted_line[0] + new_latest_filename_list.append(splitted_line[1][1:-1]) if filename_pattern: - new_latest_filename = natsorted(checksums.keys())[-1] + new_latest_filename = natsorted(new_latest_filename_list)[-1] new_latest_url = latest_url.replace(latest_filename, new_latest_filename) logger.info(f"Latest URL is now {new_latest_url}") @@ -143,7 +149,7 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr latest_filename = new_latest_filename latest_url = new_latest_url - current_checksum = f"{checksum_type}:{checksums[latest_filename]}" + current_checksum = f"sha512:{sha512_value}" logger.info(f"Checksum of current {latest_filename} is {current_checksum}") try: @@ -165,9 +171,8 @@ def update_image(image, minio_server, minio_bucket, minio_access_key, minio_secr if latest_checksum != current_checksum: logger.info(f"Checking {latest_url}") - conn = urlopen(latest_url, timeout=30) struct = time.strptime( - conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" + file_headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" ) dt = datetime.fromtimestamp(time.mktime(struct)) @@ -250,7 +255,8 @@ def main( minio_access_key, minio_secret_key, ) - data["images"][index] = updated_image + if updated_image: + data["images"][index] = updated_image with open(p, "w+") as fp: ryaml = ruamel.yaml.YAML()