From 9378eac9c77584c51e76d3825b96bf617d284994 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:28:03 +0000 Subject: [PATCH 1/7] MD5 Check Sum generate and check --- CHANGELOG.rst | 6 ++++++ brewtils/models.py | 2 ++ brewtils/rest/easy_client.py | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0c7bd9b..ae373c3d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Brewtils Changelog ================== +3.27.3 +------ +TBD + +- Apply MD5 Check Sum of chunked files to ensure files are loaded into memory properly + 3.27.2 ------ 9/12/24 diff --git a/brewtils/models.py b/brewtils/models.py index 5c5aa020..234fab24 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -520,6 +520,7 @@ def __init__( owner=None, job=None, request=None, + md5_sum=None, ): self.id = id self.owner_id = owner_id @@ -532,6 +533,7 @@ def __init__( self.file_size = file_size self.chunks = chunks self.chunk_size = chunk_size + self.md5_sum = md5_sum def __str__(self): return self.file_name diff --git a/brewtils/rest/easy_client.py b/brewtils/rest/easy_client.py index 919f6673..fae23d23 100644 --- a/brewtils/rest/easy_client.py +++ b/brewtils/rest/easy_client.py @@ -4,6 +4,7 @@ from io import BytesIO from pathlib import Path from typing import Any, Callable, List, NoReturn, Optional, Type, Union +from hashlib import md5 import six import wrapt @@ -999,6 +1000,8 @@ def upload_chunked_file( fd = file_to_upload require_close = False + default_file_params["md5_sum"] = md5(fd.getbuffer()).hexdigest() + try: default_file_params["file_name"] = desired_filename or fd.name except AttributeError: @@ -1066,6 +1069,9 @@ def download_chunked_file(self, file_id): file_obj.seek(0) + if "md5_sum" in meta and meta["md5_sum"] != md5(file_obj.getbuffer()).hexdigest(): + raise ValidationError("Requested file %s does not match MD5 SUM." % file_id) + return file_obj def delete_chunked_file(self, file_id): From 2621654eaefe2ae75fb5509c3efa5d3499e971b5 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Fri, 13 Sep 2024 18:29:22 +0000 Subject: [PATCH 2/7] Add schema --- brewtils/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 3d2eb385..21dbf224 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -261,6 +261,7 @@ class FileSchema(BaseSchema): file_size = fields.Int(allow_none=False) chunks = fields.Dict(allow_none=True) chunk_size = fields.Int(allow_none=False) + md5_sum = fields.Str(allow_none=True) class FileChunkSchema(BaseSchema): From 25307b8b1b995aeb8ec3ac35ddb25406c9de070a Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 16 Sep 2024 11:26:27 -0400 Subject: [PATCH 3/7] testing and formatting --- Makefile | 2 +- brewtils/rest/easy_client.py | 5 ++++- test/rest/easy_client_test.py | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fa242155..3cafce4a 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ clean: clean-build clean-docs clean-python clean-test ## remove everything but s # Formatting format: ## Run black formatter in-line - black --target-version py27 $(MODULE_NAME) $(TEST_DIR) + black $(MODULE_NAME) $(TEST_DIR) # Linting diff --git a/brewtils/rest/easy_client.py b/brewtils/rest/easy_client.py index fae23d23..56f7f752 100644 --- a/brewtils/rest/easy_client.py +++ b/brewtils/rest/easy_client.py @@ -1069,7 +1069,10 @@ def download_chunked_file(self, file_id): file_obj.seek(0) - if "md5_sum" in meta and meta["md5_sum"] != md5(file_obj.getbuffer()).hexdigest(): + if ( + "md5_sum" in meta + and meta["md5_sum"] != md5(file_obj.getbuffer()).hexdigest() + ): raise ValidationError("Requested file %s does not match MD5 SUM." % file_id) return file_obj diff --git a/test/rest/easy_client_test.py b/test/rest/easy_client_test.py index b58e990a..0f42ac33 100644 --- a/test/rest/easy_client_test.py +++ b/test/rest/easy_client_test.py @@ -39,6 +39,7 @@ def target_file(): fp.tell = Mock(return_value=0) fp.seek = Mock(return_value=1024) fp.read = Mock(side_effect=iter([b"content", None])) + fp.getbuffer = Mock(return_value=b"content") return fp @@ -596,6 +597,12 @@ def test_upload_chunked_file( client._check_chunked_file_validity = Mock(return_value=(True, {})) resolvable = client.upload_chunked_file(target_file, "desired_name") + _, called_kwargs = rest_client.post_chunked_file.call_args[0] + + assert called_kwargs["md5_sum"] == "9a0364b9e99bb480dd25e1f0284c8555" + assert called_kwargs["file_name"] == "desired_name" + assert called_kwargs["file_size"] == 1024 + assert called_kwargs["chunk_size"] == 261120 assert resolvable == bg_resolvable_chunk def test_upload_file_fail(self, client, rest_client, server_error, target_file): From dc0163a52bbee7b329794e51eaf82e489db1ebb6 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:03:59 -0400 Subject: [PATCH 4/7] Adding MD5 to file status --- brewtils/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brewtils/models.py b/brewtils/models.py index 234fab24..cf4ba699 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -595,6 +595,7 @@ def __init__( chunks_ok=None, operation_complete=None, message=None, + md5_sum=None ): # Top-level file info self.file_id = file_id @@ -605,6 +606,7 @@ def __init__( self.chunks = chunks self.owner_id = owner_id self.owner_type = owner_type + self.md5_sum = md5_sum # Chunk info self.chunk_id = chunk_id self.offset = offset From 2fb573f05576072c954ec6565fb4fd2e63c9e074 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:06:59 -0400 Subject: [PATCH 5/7] formatting --- brewtils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brewtils/models.py b/brewtils/models.py index cf4ba699..06f1ffc9 100644 --- a/brewtils/models.py +++ b/brewtils/models.py @@ -595,7 +595,7 @@ def __init__( chunks_ok=None, operation_complete=None, message=None, - md5_sum=None + md5_sum=None, ): # Top-level file info self.file_id = file_id From d6ea28de56e74e0bbff57bc1c04c1609ecb42310 Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:14:26 -0400 Subject: [PATCH 6/7] Add MD5 to FileStatus --- brewtils/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/brewtils/schemas.py b/brewtils/schemas.py index 21dbf224..ccaeee87 100644 --- a/brewtils/schemas.py +++ b/brewtils/schemas.py @@ -282,6 +282,7 @@ class FileStatusSchema(BaseSchema): chunks = fields.Dict(allow_none=True) owner_id = fields.Str(allow_none=True) owner_type = fields.Str(allow_none=True) + md5_sum = fields.Str(allow_none=True) # Chunk info chunk_id = fields.Str(allow_none=True) offset = fields.Int(allow_none=True) From 9b2bd690327aa5dfd6fd1074ea1b17af3ecaab4f Mon Sep 17 00:00:00 2001 From: TheBurchLog <5104941+TheBurchLog@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:09:32 -0400 Subject: [PATCH 7/7] Update Validation Error for MD5 Checksum --- brewtils/rest/easy_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/brewtils/rest/easy_client.py b/brewtils/rest/easy_client.py index 56f7f752..eece1d78 100644 --- a/brewtils/rest/easy_client.py +++ b/brewtils/rest/easy_client.py @@ -1073,7 +1073,10 @@ def download_chunked_file(self, file_id): "md5_sum" in meta and meta["md5_sum"] != md5(file_obj.getbuffer()).hexdigest() ): - raise ValidationError("Requested file %s does not match MD5 SUM." % file_id) + raise ValidationError( + "Requested file %s MD5 SUM %s does match actual MD5 SUM %s" + % (file_id, meta["md5_sum"], md5(file_obj.getbuffer()).hexdigest()) + ) return file_obj