From a524c1c91732a159f580750a7af30dab5eed04a0 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Wed, 15 May 2024 20:07:59 +0200 Subject: [PATCH 1/4] set `timeout` to unlimited --- pyDataverse/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyDataverse/api.py b/pyDataverse/api.py index 118560b..efd3186 100644 --- a/pyDataverse/api.py +++ b/pyDataverse/api.py @@ -292,7 +292,7 @@ def _sync_request( kwargs = self._filter_kwargs(kwargs) try: - resp = method(**kwargs, follow_redirects=True) + resp = method(**kwargs, follow_redirects=True, timeout=None) if resp.status_code == 401: error_msg = resp.json()["message"] raise ApiAuthorizationError( From 656c1bb1096550672819ff9a91a4b4430984bd28 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 17 May 2024 10:17:16 +0200 Subject: [PATCH 2/4] test bulk file upload --- tests/api/test_upload.py | 56 +++++++++++++++++++++++++++++++++++++--- tests/conftest.py | 24 +++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/tests/api/test_upload.py b/tests/api/test_upload.py index 0d65e3c..34af3a4 100644 --- a/tests/api/test_upload.py +++ b/tests/api/test_upload.py @@ -9,7 +9,6 @@ class TestFileUpload: - def test_file_upload(self): """ Test case for uploading a file to a dataset. @@ -46,6 +45,58 @@ def test_file_upload(self): # Assert assert response.status_code == 200, "File upload failed." + def test_bulk_file_upload(self, create_mock_file): + """ + Test case for uploading bulk files to a dataset. + + This test is meant to check the performance of the file upload feature + and that nothing breaks when uploading multiple files in line. + + This test case performs the following steps: + 0. Create 50 mock files. + 1. Creates a dataset using the provided metadata. + 2. Prepares a file for upload. + 3. Uploads the file to the dataset. + 4. Asserts that the file upload was successful. + + Raises: + AssertionError: If the file upload fails. + + """ + # Arrange + BASE_URL = os.getenv("BASE_URL").rstrip("/") + API_TOKEN = os.getenv("API_TOKEN") + + # Create dataset + metadata = json.load(open("tests/data/file_upload_ds_minimum.json")) + pid = self._create_dataset(BASE_URL, API_TOKEN, metadata) + api = NativeApi(BASE_URL, API_TOKEN) + + with tempfile.TemporaryDirectory() as tmp_dir: + # Create mock files + mock_files = [ + create_mock_file( + filename=f"mock_file_{i}.txt", + dir=tmp_dir, + size=1024**2, # 1MB + ) + for i in range(50) + ] + + for mock_file in mock_files: + # Prepare file upload + df = Datafile({"pid": pid, "filename": os.path.basename(mock_file)}) + + # Act + response = api.upload_datafile( + identifier=pid, + filename=mock_file, + json_str=df.json(), + ) + + # Assert + assert response.status_code == 200, "File upload failed." + def test_file_replacement(self): """ Test case for replacing a file in a dataset. @@ -56,7 +107,7 @@ def test_file_replacement(self): 3. Replace the uploaded datafile with a mutated version. 4. Verify that the file replacement was successful and the content matches the expected content. """ - + # Arrange BASE_URL = os.getenv("BASE_URL").rstrip("/") API_TOKEN = os.getenv("API_TOKEN") @@ -79,7 +130,6 @@ def test_file_replacement(self): # Act with tempfile.TemporaryDirectory() as tempdir: - original = open("tests/data/replace.xyz").read() mutated = "Z" + original[1::] mutated_path = os.path.join(tempdir, "replace.xyz") diff --git a/tests/conftest.py b/tests/conftest.py index 76047a9..8d64c99 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -156,3 +156,27 @@ def import_datafile_full_dict(): "description": "Test datafile", "restrict": False, } + + +@pytest.fixture +def create_mock_file(): + """Returns a function that creates a mock file.""" + + def _create_mock_file(filename: str, dir: str, size: int): + """Create a mock file. + + Args: + filename (str): Filename. + dir (str): Directory. + size (int): Size. + + Returns: + str: Path to the file. + """ + path = os.path.join(dir, filename) + with open(path, "wb") as f: + f.write(os.urandom(size)) + + return path + + return _create_mock_file From d8f74a408c9ff9f305e25c6add9dbf63329e3800 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 17 May 2024 10:20:34 +0200 Subject: [PATCH 3/4] bump patch version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3ccce23..a6d9f9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyDataverse" -version = "0.3.2" +version = "0.3.3" description = "A Python module for Dataverse." authors = [ "Stephan Kasberger ", From 0380ea21b158190ba0bfdd9728820bc1e7881dc5 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Fri, 17 May 2024 10:21:22 +0200 Subject: [PATCH 4/4] bump patch version --- pyDataverse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyDataverse/__init__.py b/pyDataverse/__init__.py index bea7692..7136b6f 100644 --- a/pyDataverse/__init__.py +++ b/pyDataverse/__init__.py @@ -12,7 +12,7 @@ __email__ = "stefan.kasberger@univie.ac.at" __copyright__ = "Copyright (c) 2019 Stefan Kasberger" __license__ = "MIT License" -__version__ = "0.3.2" +__version__ = "0.3.3" __url__ = "https://github.com/GDCC/pyDataverse" __download_url__ = "https://pypi.python.org/pypi/pyDataverse" __description__ = "A Python module for Dataverse."