From e42012d7bef74bf036f7580019add67cb5f8a6a9 Mon Sep 17 00:00:00 2001 From: Gondermann Date: Wed, 9 Aug 2023 16:00:28 +0200 Subject: [PATCH 1/3] Validate provided checksum after successful import Use the 'verify_checksum' hash value in the yaml files to verify the image integrity after it has been successfully imported. Show a warning, if either the hash algorithm or the hash value does not match the expected fields. Fixes #340 Signed-off-by: Gondermann --- openstack_image_manager/manage.py | 47 +++++++++++++++++++++++++++---- test/unit/test_manage.py | 37 ++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/openstack_image_manager/manage.py b/openstack_image_manager/manage.py index 81a2a2b4..1d7ec44c 100644 --- a/openstack_image_manager/manage.py +++ b/openstack_image_manager/manage.py @@ -303,7 +303,7 @@ def process_images(self, images) -> set: if "hidden" in version: versions[version["version"]]["hidden"] = version["hidden"] - if version["version"] == "latest":# + if version["version"] == "latest": if "checksums_url" in version: versions[version["version"]]["checksums_url"] = version["checksums_url"] else: @@ -325,6 +325,9 @@ def process_images(self, images) -> set: if "build_date" in version: versions[version["version"]]["meta"]["image_build_date"] = date.isoformat(version["build_date"]) + if "verify_checksum" in version: + versions[version["version"]]["verify_checksum"] = version["verify_checksum"] + if "id" in version: versions[version["version"]]["id"] = version["id"] except ValueError as e: @@ -626,11 +629,43 @@ def process_image( if not self.CONF.dry_run: import_result = self.import_image(image, name, url, versions, version) if import_result: - logger.info( - "Import of '%s' successfully completed, reloading images" % name - ) - cloud_images = self.get_images() - imported_image = cloud_images.get(name, None) + + hash_check_success = True + if "verify_checksum" in versions[version]: + hash_algo, hash_value = versions[version]["verify_checksum"].split(":", 2) + if hash_algo != import_result.hash_algo: + logger.warning( + "Provided verify_checksum algorithm '%s' does not equal the expected algorithm '%s'" + % (hash_algo, import_result.hash_algo) + ) + logger.warning( + "Verification checksum for '%s' will be ignored..." + % name + ) + elif hash_value != import_result.hash_value: + logger.error( + "Provided verify_checksum for '%s' does not match backend checksum!" + % name + ) + hash_check_success = False + else: + logger.info("Backend checksum matches expected value") + else: + logger.warning( + "No verification checksum for '%s'. Ignoring..." + % name + ) + + if hash_check_success: + logger.info( + "Import of '%s' successfully completed, reloading images" % name + ) + cloud_images = self.get_images() + imported_image = cloud_images.get(name, None) + else: + logger.info("Deleting possibly corrupt image %s" % import_result.id) + self.conn.image.delete_image(import_result.id) + continue else: logger.info( f"Skipping required import of image '{name}', running in dry-run mode" diff --git a/test/unit/test_manage.py b/test/unit/test_manage.py index 4836fb74..fbe810cd 100644 --- a/test/unit/test_manage.py +++ b/test/unit/test_manage.py @@ -34,6 +34,7 @@ build_date: 2021-01-21 url: http://url.com checksum: '1234' + verify_checksum: 'sha512:abcd' ''' # sample image dict as generated from FAKE_YML @@ -59,7 +60,8 @@ 'build_date': date.fromisoformat("2021-01-21"), 'version': '1', 'url': 'http://url.com', - 'checksum': '1234' + 'checksum': '1234', + 'verify_checksum': 'sha512:abcd' } ] } @@ -100,7 +102,8 @@ def setUp(self): self.fake_image = Image(**FAKE_IMAGE_DATA) self.fake_name = '%s (%s)' % (self.fake_image_dict['name'], '1') self.fake_url = 'http://url.com' - self.versions = {'1': {'url': self.fake_url, 'meta': {'image_source': self.fake_url, 'image_build_date': '2021-01-21'}}} + self.fake_verify_checksum = 'sha512:abcd' + self.versions = {'1': {'url': self.fake_url, 'meta': {'image_source': self.fake_url, 'image_build_date': '2021-01-21'}, 'verify_checksum': self.fake_verify_checksum}} self.sorted_versions = ['2', '1'] self.previous_image = self.fake_image self.imported_image = self.fake_image @@ -232,6 +235,7 @@ def test_check_image_age(self, mock_read_image_files, mock_get_images): too_old_images = self.sot.check_image_age() self.assertIn(self.fake_name, too_old_images) + @mock.patch('openstack_image_manager.manage.openstack.image.v2._proxy.Proxy.delete_image') @mock.patch('openstack_image_manager.manage.ImageManager.set_properties') @mock.patch('openstack_image_manager.manage.ImageManager.import_image') @mock.patch('openstack_image_manager.manage.requests.head') @@ -239,7 +243,7 @@ def test_check_image_age(self, mock_read_image_files, mock_get_images): @mock.patch('os.path.isfile') @mock.patch('os.path.exists') def test_process_image(self, mock_path_exists, mock_path_isfile, mock_get_images, mock_requests, - mock_import_image, mock_set_properties): + mock_import_image, mock_set_properties, mock_delete_image): ''' test manage.ImageManager.process_image() ''' mock_requests.return_value.status_code = 200 @@ -249,6 +253,7 @@ def test_process_image(self, mock_path_exists, mock_path_isfile, mock_get_images self.assertEqual(mock_get_images.call_count, 2) mock_requests.assert_called_once_with(self.fake_url) + mock_delete_image.assert_not_called() mock_import_image.assert_called_once_with(self.fake_image_dict, self.fake_name, self.fake_url, @@ -259,6 +264,29 @@ def test_process_image(self, mock_path_exists, mock_path_isfile, mock_get_images mock_get_images.reset_mock() mock_requests.reset_mock() + mock_delete_image.reset_mock() + mock_import_image.reset_mock() + mock_set_properties.reset_mock() + + # test wrong checksum + mock_import_image.return_value.hash_algo = "sha512" + mock_import_image.return_value.hash_value = "wrong-checksum" + result = self.sot.process_image(self.fake_image_dict, self.versions, self.sorted_versions, meta) + + self.assertEqual(mock_get_images.call_count, 1) + mock_requests.assert_called_once_with(self.fake_url) + mock_delete_image.assert_called_once() + mock_import_image.assert_called_once_with(self.fake_image_dict, + self.fake_name, + self.fake_url, + self.versions, + '1') + mock_set_properties.assert_not_called() + self.assertEqual(result, ({self.fake_image_dict['name']}, None, None)) + + mock_get_images.reset_mock() + mock_requests.reset_mock() + mock_delete_image.reset_mock() mock_import_image.reset_mock() mock_set_properties.reset_mock() @@ -272,6 +300,7 @@ def test_process_image(self, mock_path_exists, mock_path_isfile, mock_get_images self.assertEqual(mock_get_images.call_count, 2) mock_requests.assert_not_called() + mock_delete_image.assert_not_called() mock_import_image.assert_called_once_with(self.file_image_dict, self.fake_name, self.file_url, @@ -280,6 +309,7 @@ def test_process_image(self, mock_path_exists, mock_path_isfile, mock_get_images mock_get_images.reset_mock() mock_requests.reset_mock() + mock_delete_image.reset_mock() mock_import_image.reset_mock() mock_set_properties.reset_mock() mock_path_exists.reset_mock() @@ -291,6 +321,7 @@ def test_process_image(self, mock_path_exists, mock_path_isfile, mock_get_images mock_get_images.assert_called_once() mock_requests.assert_called_once_with(self.fake_url) + mock_delete_image.assert_not_called() mock_import_image.assert_not_called() mock_set_properties.assert_not_called() self.assertEqual(result, ({self.fake_image_dict['name']}, None, None)) From 5bc0135e5ca00d8df224f2e4d7398f870b7cbf7e Mon Sep 17 00:00:00 2001 From: Gondermann Date: Thu, 10 Aug 2023 12:42:33 +0200 Subject: [PATCH 2/3] Add verification sha512 sums to images Signed-off-by: Gondermann --- etc/images/almalinux.yml | 2 ++ etc/images/centos.yml | 3 +++ etc/images/cirros.yml | 6 ++++-- etc/images/clearlinux.yml | 3 ++- etc/images/debian.yml | 6 ++++++ etc/images/fedora.yml | 3 ++- etc/images/flatcar.yml | 3 ++- etc/images/gardenlinux.yml | 9 ++++++--- etc/images/kubernetes.yml | 12 ++++++++---- etc/images/octavia.yml | 3 ++- etc/images/opensuse.yml | 3 ++- etc/images/opnsense.yml | 3 ++- etc/images/rockylinux.yml | 2 ++ etc/images/talos.yml | 6 ++++-- etc/images/ubuntu.yml | 9 +++++++++ etc/schema.yaml | 1 + 16 files changed, 57 insertions(+), 17 deletions(-) diff --git a/etc/images/almalinux.yml b/etc/images/almalinux.yml index 4e3f373e..bcabce7f 100644 --- a/etc/images/almalinux.yml +++ b/etc/images/almalinux.yml @@ -30,6 +30,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/almalinux-8/20230524-almalinux-8.qcow2 checksum: sha256:c0ad09255d91288dac590d99c95197d83a2846f1bcbec3f4222fb04265a2a4d7 + verify_checksum: sha512:b764ca6cad19bdb59b0efd653b2898a9557ced3d060d13a05faf4d5b53e676bd5e12801ae862040c9f3854af0228b49ddfbbc727300a4bf7d3d670fc246264a3 build_date: 2023-05-24 - name: AlmaLinux 9 enable: true @@ -61,4 +62,5 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/almalinux-9/20230513-almalinux-9.qcow2 checksum: sha256:207d885ca8140e3106098e946cfc04088b0e21f50d24815051520d452eae0a50 + verify_checksum: sha512:9519eb108ff0bc4ecc381400c08d7b476fdf9e5f425de2656caf7fa5fca3f06cc751823ceecdeb0cbe328165d533b364188214d24186e7af772a58285c06e5c3 build_date: 2023-05-13 diff --git a/etc/images/centos.yml b/etc/images/centos.yml index 8c5f7b4e..1c7f7201 100644 --- a/etc/images/centos.yml +++ b/etc/images/centos.yml @@ -30,6 +30,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/centos-7/20221112-centos-7.qcow2 checksum: sha256:284aab2b23d91318f169ff464bce4d53404a15a0618ceb34562838c59af4adea + verify_checksum: sha512:1b29cd42a53a3b0807962c1c9fbbf7fb360e0d95bb0c962c15fd9fa27ad13e908f45d81558a273ecdc4a5db1b7b3ae8cd14678b56e17e8d7ee5f588d36a8a401 build_date: 2022-11-12 - name: CentOS Stream 8 enable: false @@ -61,6 +62,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/centos-stream-8/20230826-centos-stream-8.qcow2 checksum: sha256:3d0b45bacb0355d7f0542d302a7a71146b425f41b513782b4dc554daffe72e9e + verify_checksum: sha512:19cc5b68cc7ae40060e662dda6cb8bde3b0745702f58becaad849673ec30a28085eba93bc5d409a2398c067392f5215d6f634d8977ff28aaa9b0069e17d48030 build_date: 2023-08-26 - name: CentOS Stream 9 enable: true @@ -92,4 +94,5 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/centos-stream-9/20230821-centos-stream-9.qcow2 checksum: sha256:ce7828ea362f4c63d5a73567ee4adce11248b017473ac3bdfa1c155af8796fe5 + verify_checksum: sha512:4f5af823dfdd952264f4e60ed9cd155448ef3d558af59cc209035c51205e9a2e2052dd83a00ba0082ac949136f6136fd58b9eccb7d3069a95ed1f4895630d689 build_date: 2023-08-21 diff --git a/etc/images/cirros.yml b/etc/images/cirros.yml index 8a22c055..f1cca079 100644 --- a/etc/images/cirros.yml +++ b/etc/images/cirros.yml @@ -25,9 +25,11 @@ images: versions: - version: '0.6.0' url: https://github.com/cirros-dev/cirros/releases/download/0.6.0/cirros-0.6.0-x86_64-disk.img - checksum: "sha256:94e1e2c94dbbae7d4bdc38e68590a1daf73c9de2d03dd693857b4b0a042548e8" + checksum: sha256:94e1e2c94dbbae7d4bdc38e68590a1daf73c9de2d03dd693857b4b0a042548e8 + verify_checksum: sha512:3460af14752a95f57992321bdd90d5a1157363173f4d2b3c45c47a2bff283a1ac0eb8b0c60e9a013488f40fad3c04bf8b4241de00a68624eff7ec55b3b5a5b11 build_date: 2022-09-28 - version: '0.6.1' url: https://github.com/cirros-dev/cirros/releases/download/0.6.1/cirros-0.6.1-x86_64-disk.img - checksum: "sha256:cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1" + checksum: sha256:cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1 + verify_checksum: sha512:df88bac2791254f68941229792539621516bd480aa3d6fe4c0ca16057393d024a4944d644959f323dc01a25e3417c0b581776ab3c8db8da542039f2a67230579 build_date: 2022-11-22 diff --git a/etc/images/clearlinux.yml b/etc/images/clearlinux.yml index 19755353..da90b2db 100644 --- a/etc/images/clearlinux.yml +++ b/etc/images/clearlinux.yml @@ -26,5 +26,6 @@ images: - version: '38700' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/clearlinux/38700/clear-38700-cloudguest.img source: https://cdn.download.clearlinux.org/releases/38700/clear/clear-38700-cloudguest.img.xz - checksum: "sha256:554f2e9f1c5cd3caa7ca9043e9379ab7bac36f8004ae6023f2fb642b80ace173" + checksum: sha256:554f2e9f1c5cd3caa7ca9043e9379ab7bac36f8004ae6023f2fb642b80ace173 + verify_checksum: sha512:67c8e08daa554103b13c6bf56da00334a9bb9634a22ec27591ddf2cc3135175dd343bdce38f2bfb54e21e6fa9e167a6f7b5980a5f116802de2e7cf2ab29bcf39 build_date: 2023-03-31 diff --git a/etc/images/debian.yml b/etc/images/debian.yml index 8c074ef5..4a4df8f1 100644 --- a/etc/images/debian.yml +++ b/etc/images/debian.yml @@ -31,6 +31,8 @@ images: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/debian-10/20230802-debian-10.qcow2 checksum: sha512:a11252378a9573052b48fcec8cb0e5d6ab11338e81e960442eb8a9bf4504d8e25f5c2cd893c010f63ef99201b96e4f9cae7fce6c8f8132e18f1a561b9afd8eb3 + verify_checksum: + sha512:a11252378a9573052b48fcec8cb0e5d6ab11338e81e960442eb8a9bf4504d8e25f5c2cd893c010f63ef99201b96e4f9cae7fce6c8f8132e18f1a561b9afd8eb3 build_date: 2023-08-02 - name: Debian 11 enable: true @@ -62,6 +64,8 @@ images: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/debian-11/20230802-debian-11.qcow2 checksum: sha512:f16512174452ea49e34d302e86bb11b6f821013dde583bf703550998e87819f98c6e411d7737bd53f16c73007f1c039218f9cd40421ec3a6b5b1fe38044018c0 + verify_checksum: + sha512:f16512174452ea49e34d302e86bb11b6f821013dde583bf703550998e87819f98c6e411d7737bd53f16c73007f1c039218f9cd40421ec3a6b5b1fe38044018c0 build_date: 2023-08-02 - name: Debian 12 enable: true @@ -91,6 +95,8 @@ images: - build_date: 2023-08-26 checksum: sha512:b9b01448d1572326622e7e5bcf060b995bf5c66326e826f0149c21807cfbda4000224aeb2f43a1991ab486c05d0e184e13de3dd4e0be48611626377fb10d9263 + verify_checksum: + sha512:b9b01448d1572326622e7e5bcf060b995bf5c66326e826f0149c21807cfbda4000224aeb2f43a1991ab486c05d0e184e13de3dd4e0be48611626377fb10d9263 url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/debian-12/20230826-debian-12.qcow2 version: '20230826' diff --git a/etc/images/fedora.yml b/etc/images/fedora.yml index 9d36bf56..37b1e2b1 100644 --- a/etc/images/fedora.yml +++ b/etc/images/fedora.yml @@ -27,5 +27,6 @@ images: - version: '20230402' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/fedora-37/20230402/Fedora-Cloud-Base-37-1.7.x86_64.qcow2 source: https://download.fedoraproject.org/pub/fedora/linux/releases/37/Cloud/x86_64/images/Fedora-Cloud-Base-37-1.7.x86_64.qcow2 - checksum: "sha256:b5b9bec91eee65489a5745f6ee620573b23337cbb1eb4501ce200b157a01f3a0" + checksum: sha256:b5b9bec91eee65489a5745f6ee620573b23337cbb1eb4501ce200b157a01f3a0 + verify_checksum: sha512:41f6d3e6184245bc8c92977f235030bb9c859b2e65b5e668d1ffbc436e1f655a17289bc6955e82e87c921a11e6e646a5d5cee1771e6012e3c8a24c7c5d01c421 build_date: 2022-11-05 diff --git a/etc/images/flatcar.yml b/etc/images/flatcar.yml index fe0d20dc..347c75fa 100644 --- a/etc/images/flatcar.yml +++ b/etc/images/flatcar.yml @@ -26,5 +26,6 @@ images: - version: '3227.2.4' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/flatcar/3227.2.4/flatcar_production_openstack_image.img source: https://stable.release.flatcar-linux.net/amd64-usr/3227.2.4/flatcar_production_openstack_image.img.bz2 - checksum: "sha256:774a4fd77da47a7c91714e6424891d6887ed7b9b72daadc40d56bb2f7527b4f5" + checksum: sha256:774a4fd77da47a7c91714e6424891d6887ed7b9b72daadc40d56bb2f7527b4f5 + verify_checksum: sha512:fc5a2c8a584ec765b76975ee264b70280049c6d616c9fb003ae2409aba7a5b62b5eae23691a02f727bd8104115c57b54645e6962e54a070b33edd8f2c4ab3569 build_date: 2022-10-27 diff --git a/etc/images/gardenlinux.yml b/etc/images/gardenlinux.yml index a5ad0b9f..b2331631 100644 --- a/etc/images/gardenlinux.yml +++ b/etc/images/gardenlinux.yml @@ -23,13 +23,16 @@ images: versions: - version: '934.8' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-image-gardenlinux/gardenlinux-amd64.934.8.qcow2 - checksum: "sha256:68a4cb768224aec313a8c974cec412fe8b2754f0189a318ce9dbdba13fd89597" + checksum: sha256:68a4cb768224aec313a8c974cec412fe8b2754f0189a318ce9dbdba13fd89597 + verify_checksum: sha512:fafc1bee465c7492fe274ef426158deb84a35053ffd442de2e57921435933b6da2a58bc78de7c819efc376af6acf82f428cecad8fd792a67aeeafa8d011cb43f build_date: 2023-05-12 - version: '934.9' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-image-gardenlinux/gardenlinux-amd64.934.9.qcow2 - checksum: "sha256:314aa8369a46a7c9fd97343a688c75b9c282f654fb05e9bb4d6b94646bf3670c" + checksum: sha256:314aa8369a46a7c9fd97343a688c75b9c282f654fb05e9bb4d6b94646bf3670c + verify_checksum: sha512:be5811738e938dcc5c018e0307a6c1c1784c86a69c920817361f1ec1085761d3b354def13a99c89ac15a7a0b570076774b37d238d961e876813ef475d4f39eb3 build_date: 2023-06-03 - version: '934.10' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-image-gardenlinux/gardenlinux-amd64.934.10.qcow2 - checksum: "sha256:995e40bb7dcf044f82650f8131bc8b440215e5f1023c4edca81da585e7068b4f" + checksum: sha256:995e40bb7dcf044f82650f8131bc8b440215e5f1023c4edca81da585e7068b4f + verify_checksum: sha512:987eae0cafd4e49c2a4feaea188df4dfc0d43e753ce03d4191a72641285518e412a8b3c1866bb5e852f4cb2521f21d6990381d090bfc34de9474794446a4f0b2 build_date: 2023-08-28 diff --git a/etc/images/kubernetes.yml b/etc/images/kubernetes.yml index 7791dd65..807dad41 100644 --- a/etc/images/kubernetes.yml +++ b/etc/images/kubernetes.yml @@ -24,17 +24,21 @@ images: versions: - version: '1.24.15' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v1.24/ubuntu-2204-kube-v1.24.15.qcow2 - checksum: "sha256:12fbfead6400fae2fac20b8fae130e23afc351c44ad120deee2318b4bc7b2e59" + checksum: sha256:12fbfead6400fae2fac20b8fae130e23afc351c44ad120deee2318b4bc7b2e59 + verify_checksum: sha512:925cb6f95ff6029dd58ca4e6d96c194abe7444e013f94db350b25955bc6d64eaf55470a37e9e1aca26b2c40e89870a1669f96752a88840c062a82e1aab76077b build_date: 2023-06-15 - version: '1.25.11' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v1.25/ubuntu-2204-kube-v1.25.11.qcow2 - checksum: "sha256:7870a6c7c1761c572c070ca7b776b39344d8d2fd0dca28718fff433e9dca1a8f" + checksum: sha256:7870a6c7c1761c572c070ca7b776b39344d8d2fd0dca28718fff433e9dca1a8f + verify_checksum: sha512:6e0f2dd88f2887c57c24b01b18cf5c8dcfb9742c358ea9307eb5bdaa2929c28a1d1f2338ecb8fc7fadbac5933647ac9b2ed1ec2dbdcacb26fd50866724317fe3 build_date: 2023-06-21 - version: '1.26.6' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v1.26/ubuntu-2204-kube-v1.26.6.qcow2 - checksum: "sha256:0243eeee8a55ed9d9f920d4b168ec548541dc2504c996d58a20be5f6d6518409" + checksum: sha256:0243eeee8a55ed9d9f920d4b168ec548541dc2504c996d58a20be5f6d6518409 + verify_checksum: sha512:cf9c2ac1e36d6e421ed50e477d4c58fdcf67323e00beae57b3f64f027e6af0f6c880bd228bba7710c2a39fb007482d29c2b3f9e24b79d367711481bb5357afe6 build_date: 2023-06-21 - version: '1.27.3' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-k8s-capi-images/ubuntu-2204-kube-v1.27/ubuntu-2204-kube-v1.27.3.qcow2 - checksum: "sha256:857bbb11f29faba71f9f6dc1c44ed657507e422b42f589e1d96da2711c41d5bf" + checksum: sha256:857bbb11f29faba71f9f6dc1c44ed657507e422b42f589e1d96da2711c41d5bf + verify_checksum: sha512:190e231749b48406270bd34f05cd729488a239de65d35429b92913af39332b1e69dd077d974257e1aa280d27df38c5288298f4174474bb112ab16190cf7c9fdc build_date: 2023-06-21 diff --git a/etc/images/octavia.yml b/etc/images/octavia.yml index 8a605b36..0d25c5b5 100644 --- a/etc/images/octavia.yml +++ b/etc/images/octavia.yml @@ -25,5 +25,6 @@ images: versions: - version: 'ZED-2023-05-05' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-octavia-amphora-image/octavia-amphora-haproxy-zed.qcow2 - checksum: "sha256:f0ec47f052e9549b88b3ae86c805ab05036b2a711761b3e2595cb6325311588d" + checksum: sha256:f0ec47f052e9549b88b3ae86c805ab05036b2a711761b3e2595cb6325311588d + verify_checksum: sha512:7ba43bb611dc419b478eb7aeee2758dac7b6f6b829e6149a08675e7b0992d92f1b4d2e520ea39c4da8195a7436d8f884c1f5bed512fb519f5e05b94a66c27b3d build_date: 2023-05-05 diff --git a/etc/images/opensuse.yml b/etc/images/opensuse.yml index 29ac2145..2d30d749 100644 --- a/etc/images/opensuse.yml +++ b/etc/images/opensuse.yml @@ -26,5 +26,6 @@ images: - version: '20220805' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/opensuse-leap-15.4/20220805/openSUSE-Leap-15.4-JeOS.x86_64-OpenStack-Cloud.qcow2 source: https://ftp.gwdg.de/pub/opensuse/distribution/leap/15.4/appliances/openSUSE-Leap-15.4-JeOS.x86_64-OpenStack-Cloud.qcow2 - checksum: "sha256:2521af728f47bdf7c18d18e1c95b576d2aedd9a8fb3668b4791f7f5ece7a1931" + checksum: sha256:2521af728f47bdf7c18d18e1c95b576d2aedd9a8fb3668b4791f7f5ece7a1931 + verify_checksum: sha512:3e001ab0ae774978a4414e614c1fb23d00eeb43581a409d50ad43a2a42d3edf8feb6159bfce349c16ed3aa23ea0b80f7df1ed4143c4235a803b5db8e904c984d build_date: 2022-08-05 diff --git a/etc/images/opnsense.yml b/etc/images/opnsense.yml index f386ffef..ead3e50b 100644 --- a/etc/images/opnsense.yml +++ b/etc/images/opnsense.yml @@ -27,5 +27,6 @@ images: - version: '23.1' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/opnsense/23.1/OPNsense-23.1-OpenSSL-nano-amd64.img source: https://mirror.ams1.nl.leaseweb.net/opnsense/releases/23.1/OPNsense-23.1-OpenSSL-nano-amd64.img.bz2 - checksum: "sha256:8e2e5ab62f8d2bc5eb72296a0d7913789f370f445216d996a7d4d9e88e5e3733" + checksum: sha256:8e2e5ab62f8d2bc5eb72296a0d7913789f370f445216d996a7d4d9e88e5e3733 + verify_checksum: sha512:06c0e8049ddeeb51c0fc7aaa93e4e6169d464e5bca30a84d55620482c5456066e7ebf30e0ff0fc667795349c2f70779917acd65a4287854984a2ae49acca796c build_date: 2023-01-25 diff --git a/etc/images/rockylinux.yml b/etc/images/rockylinux.yml index 10256804..887b6107 100644 --- a/etc/images/rockylinux.yml +++ b/etc/images/rockylinux.yml @@ -31,6 +31,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/rocky-8/20230518-rocky-8.qcow2 checksum: sha256:086bf68f84c974cfcf533741c5be8752270df681a38f20423cf24b851d5edf77 + verify_checksum: sha512:469c17ac19f7dfa4c65e7137a38130b9fe887916fac8ccf939c310e32c61d0a603aef4fadd83dd1328adeddb75433fdf4623790d207ac76496c0e72b0c720392 build_date: 2023-05-18 - name: Rocky 9 enable: true @@ -63,4 +64,5 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/rocky-9/20230515-rocky-9.qcow2 checksum: sha256:50510f98abe1b20a548102a05a9be83153b0bf634fc502d5c8d1f508f6de1430 + verify_checksum: sha512:5af3ad275dd156385fd0ad7ca53ebba1f204432e46eb8c00afc9e9155212c164b27131521c8fce8d013cdf3d28f56d8d83c140cb0a18920e14fd227ae345f7e7 build_date: 2023-05-15 diff --git a/etc/images/talos.yml b/etc/images/talos.yml index 8d29b581..6c4d7539 100644 --- a/etc/images/talos.yml +++ b/etc/images/talos.yml @@ -26,10 +26,12 @@ images: - version: '1.3.6' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/talos/1.3.6/disk.raw source: https://github.com/siderolabs/talos/releases/download/v1.3.6/openstack-amd64.tar.gz - checksum: "sha256:1db0cfd9bd9c3d3165911ab89ef05dad1d590cc442fb0702564cfe737b3f3dbd" + checksum: sha256:1db0cfd9bd9c3d3165911ab89ef05dad1d590cc442fb0702564cfe737b3f3dbd + verify_checksum: sha512:1ff097bbcc6f32bb3c601b83d0834ef9559ffdffc0d8fc288252adaed2979aafb630cf5d2fc465c30ae8adcbadaa6dac3ae4e8af8febd0b3f9ea97741e0f53ed build_date: 2023-03-14 - version: '1.4.5' url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/talos/1.4.5/disk.raw source: https://github.com/siderolabs/talos/releases/download/v1.4.5/openstack-amd64.tar.gz - checksum: "sha256:fc7dc71a6acece93196c8bace15139469b7bfbe4fa7d001336da0fdda9b437c9" + checksum: sha256:fc7dc71a6acece93196c8bace15139469b7bfbe4fa7d001336da0fdda9b437c9 + verify_checksum: sha512:5a6458a2fb35cc503d7cc29bcc455a3ecd04cb63dce68546b1dc0fe895b031eebd57246be9ecea087b291e3a22346d2021bed212c48c284286e3642662763f8a build_date: 2023-05-30 diff --git a/etc/images/ubuntu.yml b/etc/images/ubuntu.yml index 061f18a2..c41040a3 100644 --- a/etc/images/ubuntu.yml +++ b/etc/images/ubuntu.yml @@ -30,6 +30,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-14.04/20191107-ubuntu-14.04.qcow2 checksum: sha256:3c4ad0defbe729dd3c16d2851d775575d1c5351c85734418d3b89bfdfd28ebd1 + verify_checksum: sha512:68a5054c2588605e779c131589ecf7a737a71b856a31c0e2fd97c1026ad97cf7371ab5e431cf3519f0a924238cc05cc11e27a2a317967c118774020976808d6d build_date: 2019-11-07 - name: Ubuntu 16.04 enable: false @@ -61,6 +62,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-16.04/20211001-ubuntu-16.04.qcow2 checksum: sha256:fff2494a70bcaffb59f26f71ec49e137dd87d319341c35832dbe0ea65ff15140 + verify_checksum: sha512:811b6092b6825f2b1a6c409560e81fd63b6f0f68d906d45f9a35a7bdba482c121db0c8219e5ab9fde306f1ada3e5ab7a717d1570261340642dab8979928ddd11 build_date: 2021-10-01 - name: Ubuntu 16.04 Minimal enable: false @@ -92,6 +94,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-16.04-minimal/20210929-ubuntu-16.04-minimal.qcow2 checksum: sha256:7658ec30373e7ad1a1858744f395a89713d333721d7d1986ee8b71680b81a0a9 + verify_checksum: sha512:2f4ede2f19dd8d8f09bb1a11f604ecd75e80be8ac6d73a0ea340a860ef46c801e580f172f24a6af202b50095feb7758fe80eede7e03acf4ab7510425a910272f build_date: 2021-09-20 - name: Ubuntu 18.04 enable: false @@ -122,6 +125,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-18.04/20230607-ubuntu-18.04.qcow2 checksum: sha256:8dd2e6b5e5aad20c3f836123b300cba9861249408cbb07c359145a65d6bab6b6 + verify_checksum: sha512:6eaa852740f958a2d37d5444452ff202e909950919b332de7b007e19a31c4af6ae66cf2b3bcf2890a259e5e9d8cfd4d87dfbf15d0a4d7d1d87dc482278cc69bd build_date: 2023-06-07 - name: Ubuntu 18.04 Minimal enable: false @@ -153,6 +157,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-18.04-minimal/20230602-ubuntu-18.04-minimal.qcow2 checksum: sha256:2d9755669c499e88d51da0638cd20e0983a248828e2153906c013bea0ee2f45a + verify_checksum: sha512:aac9f92833ffd4c435ddd842cc1856ddab9ba58250fb8bb7ac271c9e54f2ac69bbf6dc9d09b9b9d66d485923ba7d6b21566b6d2462d1904b997edd7754e2fdc6 build_date: 2023-06-02 - name: Ubuntu 20.04 enable: true @@ -183,6 +188,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-20.04/20230823-ubuntu-20.04.qcow2 checksum: sha256:ecfb2e07fce2a273e845ca328b96b64390a0bf0efad70fd1d57a0ebbc0549717 + verify_checksum: sha512:64f640756b746d352173c2ed20cdfbbc87c9f75b08c5b4bec1fbd4ddad6781d8c961edabc32e9abbcd086a080e7d3a15deeeee32a608e698844be830caae5e22 build_date: 2023-08-23 - name: Ubuntu 20.04 Minimal enable: true @@ -214,6 +220,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-20.04-minimal/20230811-ubuntu-20.04-minimal.qcow2 checksum: sha256:95536fe95112fff650012867afbf36d31f61ec06dea4269f8133bd6a1596bf95 + verify_checksum: sha512:8af1aeefa8950934b48761717d395ae4d486dcefd985a03e9e1d55f7a821ab12c285da70277e1c6a66d7767fac66f9558fc14088962c8ce9a650e2d43730da3c build_date: 2023-08-11 - name: Ubuntu 22.04 enable: true @@ -244,6 +251,7 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-22.04/20230823-ubuntu-22.04.qcow2 checksum: sha256:a7845be05e3a5c9e031be35d27ecb6f4c1cdab8df89475cd1943d57ff94294c9 + verify_checksum: sha512:caec92b6a20c7b23869c44e7536d039aec929799c4a7960a9b3c691eb405d3bb4604df1a00eb4b21e02820e5bee2a921b9ea67e142e91700f8a9dd82d38576be build_date: 2023-08-23 - name: Ubuntu 22.04 Minimal enable: true @@ -275,4 +283,5 @@ images: url: https://swift.services.a.regiocloud.tech/swift/v1/AUTH_b182637428444b9aa302bb8d5a5a418c/openstack-images/ubuntu-22.04-minimal/20230815-ubuntu-22.04-minimal.qcow2 checksum: sha256:72ca6ba4f8b7305479ed32ed79890a52aec4f0629762286dbc9a5e5b3997489c + verify_checksum: sha512:48e0145d110dbaa7e5a74f510aad107188993321ecf77126c8397c0611ef89d099857f0ea3a0c39b92f91c60b2a4461992c2f465ba99e63ac9c463cae9967ee1 build_date: 2023-08-15 diff --git a/etc/schema.yaml b/etc/schema.yaml index e7af6715..c55cf00e 100644 --- a/etc/schema.yaml +++ b/etc/schema.yaml @@ -41,6 +41,7 @@ meta: versions: build_date: any(day(), timestamp()) checksum: regex('\w+:([a-f0-9]{32}|[a-f0-9]{40}|[a-f0-9]{64}|[a-f0-9]{128})$', name='valid checksum', required=False) + verify_checksum: regex('sha512:([a-f0-9]{128})$', name='valid verify checksum', required=False) checksums_url: regex('^(http|ftp)s?:\/\/\w+.*', name='valid URL', required=False) hidden: bool(required=False) image_description: str(required=False) From 6a1dd543998a72f4d82ac6915b3208eb9a715208 Mon Sep 17 00:00:00 2001 From: Gondermann Date: Thu, 10 Aug 2023 16:43:14 +0200 Subject: [PATCH 3/3] Force update script to download the image and calculate SHA512 Since we want to check the hash against the openstack backend, we need to have the SHA512. Sadly, most images creators do not provide us with that hash pre-computed. That means we will compute the SHA512 for every new image update in the CI worker by downloading the image. Signed-off-by: Gondermann --- openstack_image_manager/update.py | 116 ++++++++++++++++++++++-------- 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/openstack_image_manager/update.py b/openstack_image_manager/update.py index 46469a1e..3be88e63 100644 --- a/openstack_image_manager/update.py +++ b/openstack_image_manager/update.py @@ -2,13 +2,12 @@ # source of latest URLs: https://gitlab.com/libosinfo/osinfo-db from datetime import datetime +import hashlib +import math import os import re -import shutil import sys -import time from urllib.parse import urlparse -from urllib.request import urlopen from loguru import logger from minio import Minio @@ -105,14 +104,14 @@ def get_latest_debubu(shortname, latest_checksum_url, latest_url, checksum_type= IMAGES = { "almalinux": get_latest_default, "centos": get_latest_default, - "debian": get_latest_debubu, - "rockylinux": get_latest_default, + "debian": get_latest_debubu, + "rockylinux": get_latest_default, "ubuntu": get_latest_debubu, } def mirror_image( - image, minio_server, minio_bucket, minio_access_key, minio_secret_key + image, extracted_file, minio_server, minio_bucket, minio_access_key, minio_secret_key ): client = Minio( minio_server, @@ -121,14 +120,7 @@ def mirror_image( ) version = image["versions"][0] - - path = urlparse(version["url"]) dirname = image["shortname"] - filename, fileextension = os.path.splitext(os.path.basename(path.path)) - - if fileextension not in [".bz2", ".zip", ".xz", ".gz"]: - filename += fileextension - shortname = image["shortname"] format = image["format"] new_version = version["version"] @@ -139,23 +131,73 @@ def mirror_image( logger.info("'%s' available in '%s'" % (new_filename, dirname)) except S3Error: logger.info("'%s' not yet available in '%s'" % (new_filename, dirname)) - logger.info("Downloading '%s'" % version["url"]) - response = requests.get(version["url"], stream=True) - with open(os.path.basename(path.path), "wb") as fp: - shutil.copyfileobj(response.raw, fp) - del response - - if fileextension in [".bz2", ".zip", ".xz", ".gz"]: - logger.info("Decompressing '%s'" % os.path.basename(path.path)) - patoolib.extract_archive(os.path.basename(path.path), outdir=".") - os.remove(os.path.basename(path.path)) logger.info( - "Uploading '%s' to '%s' as '%s'" % (filename, dirname, new_filename) + "Uploading '%s' to '%s' as '%s'" % (extracted_file, dirname, new_filename) ) - client.fput_object(minio_bucket, os.path.join(dirname, new_filename), filename) - os.remove(filename) + client.fput_object(minio_bucket, os.path.join(dirname, new_filename), extracted_file) + + +def size_clean(size): + size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB") + i = int(math.floor(math.log(size, 1024))) + s = size / 1024 ** i + return f"{s:.2f} {size_name[i]}" + + +def download_and_hash(download_url: str): + path = urlparse(download_url) + filename, fileextension = os.path.splitext(os.path.basename(path.path)) + is_archive = fileextension in [".bz2", ".zip", ".xz", ".gz"] + if not is_archive: + filename += fileextension + download_filename = os.path.basename(path.path) + http_headers = None + hash_obj = hashlib.new("sha512") + + with requests.get(url=download_url, stream=True, timeout=30) as response: + if response.status_code != 200: + logger.error(f"Downloading image '{download_url}' failed with error code {response.status_code}") + return None, None, None + + http_headers = response.headers + file_size = int(http_headers["Content-Length"]) + logger.info(f"Image size {size_clean(file_size)}") + + downloadedBytes = 0 + lastProgress = 0 + with open(download_filename, "wb") as fp: + for chunk in response.iter_content(chunk_size=8192): + downloadedBytes += 8192 + progressPercent = (downloadedBytes / file_size) * 100 + progress = round(min(max(progressPercent, 0), 100)) + if progress - lastProgress >= 5: + logger.info(f"Downloading image: {progress}%") + lastProgress = progress + + fp.write(chunk) + + if not is_archive: + hash_obj.update(chunk) + + if not is_archive: + sha512 = hash_obj.hexdigest() + return http_headers, f"sha512:{sha512}", download_filename + else: + assert download_filename not in ["", ".", " ", "/", ".."] + logger.info("Decompressing '%s'" % download_filename) + patoolib.extract_archive(download_filename, outdir=".") + os.remove(download_filename) + + with open(filename, 'rb') as fp: + chunk = fp.read(8192) + while chunk: + hash_obj.update(chunk) + chunk = fp.read(8192) + + sha512 = hash_obj.hexdigest() + return http_headers, f"sha512:{sha512}", filename def update_image(image, getter, minio_server, minio_bucket, minio_access_key, minio_secret_key): @@ -167,7 +209,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi latest_checksum_url = image["latest_checksum_url"] logger.info(f"Getting checksums from {latest_checksum_url}") - + shortname = image["shortname"] current_checksum, current_url, current_version = getter(shortname, latest_checksum_url, latest_url) @@ -181,6 +223,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi "checksum": None, "url": None, "version": None, + "verify_checksum": None, } ) @@ -191,12 +234,20 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi logger.info(f"Image {name} is up-to-date, nothing to do") return 0 + logger.info(f"Image {name} change detected. Downloading Image...") + + headers, verify_checksum, extracted_file = download_and_hash(current_url) + if verify_checksum is None or extracted_file in ["", ".", " ", "/", ".."]: + logger.error(f"Downloading and hashing {name} failed") + return 0 + + logger.info(f"Image {name} has the verification checksum {verify_checksum}") + if current_version is None: - logger.info(f"Checking {current_url}") + logger.info("Using HTTP 'last-modified' header as current version") - conn = urlopen(current_url, timeout=30) dt = datetime.strptime( - conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" + headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" ) current_version = dt.strftime("%Y%m%d") @@ -205,6 +256,7 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi "build_date": datetime.strptime(current_version, "%Y%m%d").date(), "checksum": current_checksum, "url": current_url, + "verify_checksum": verify_checksum, } logger.info(f"New values are {new_values}") image["versions"][0].update(new_values) @@ -220,11 +272,15 @@ def update_image(image, getter, minio_server, minio_bucket, minio_access_key, mi mirror_image( image, + extracted_file, minio_server, minio_bucket, minio_access_key, minio_secret_key, ) + + os.remove(extracted_file) + return 1