From 1f70a3baaba6bce68ee98aa1508ff07b4a973634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Nie=C5=BCurawski?= Date: Wed, 23 Jun 2021 14:27:38 +0200 Subject: [PATCH] Retry 400 on upload in legacy API (#604) --- .../hosted_alpha_leaderboard_api_client.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/neptune/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py b/neptune/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py index 00b7eb027..6deadd5fc 100644 --- a/neptune/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py +++ b/neptune/internal/api_clients/hosted_api_clients/hosted_alpha_leaderboard_api_client.py @@ -53,6 +53,7 @@ from neptune.new import exceptions as alpha_exceptions from neptune.new.attributes import constants as alpha_consts from neptune.new.attributes.constants import MONITORING_TRACEBACK_ATTRIBUTE_PATH, SYSTEM_FAILED_ATTRIBUTE_PATH +from neptune.new.exceptions import ClientHttpError from neptune.new.internal import operation as alpha_operation from neptune.new.internal.backends import hosted_file_operations as alpha_hosted_file_operations from neptune.new.internal.backends.api_model import AttributeType @@ -190,7 +191,7 @@ def upload_source_code(self, experiment, source_target_pairs): file_globs=file_globs, reset=True, ) - self._execute_upload_operation(experiment, upload_files_operation) + self._execute_upload_operations_with_400_retry(experiment, upload_files_operation) @with_api_exceptions_handler def get_experiment(self, experiment_id): @@ -454,7 +455,7 @@ def log_artifact(self, experiment, artifact, destination=None): else: raise ValueError("Artifact must be a local path or an IO object") - self._execute_upload_operation(experiment, operation) + self._execute_upload_operations_with_400_retry(experiment, operation) @staticmethod def _get_dest_and_ext(target_name): @@ -594,6 +595,17 @@ def _execute_upload_operation(self, return None + def _execute_upload_operations_with_400_retry( + self, + experiment: Experiment, + upload_operation: alpha_operation.Operation): + while True: + try: + return self._execute_upload_operation(experiment, upload_operation) + except ClientHttpError as ex: + if "Length of stream does not match given range" not in ex.response: + raise ex + @with_api_exceptions_handler def _execute_operations(self, experiment: Experiment, operations: List[alpha_operation.Operation]): experiment_uuid = uuid.UUID(experiment.internal_id)