1539 inevitable recursion error from inside get result (#1859)

* Handled RecursionError on get results with wait * Added test for recursion error * Updated changelogs * Retry on RecursionError * move line to satisfy ci test --------- Co-authored-by: Ara Ghukasyan <[email protected]>
AgnostiqHQ · Dec 18, 2023 · 9cd8c5b · 9cd8c5b
1 parent 6e331d4
commit 9cd8c5b
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Reduced number of assets to upload when submitting a dispatch.
+- Handled RecursionError on get results for a long running workflow
 
 ### Operations
 

diff --git a/covalent/_results_manager/results_manager.py b/covalent/_results_manager/results_manager.py
@@ -496,15 +496,23 @@ def get_result(
         The Result object from the Covalent server
 
     """
+    max_attempts = int(os.getenv("COVALENT_GET_RESULT_RETRIES", 10))
+    num_attempts = 0
+    while num_attempts < max_attempts:
+        try:
+            return _get_result_multistage(
+                dispatch_id=dispatch_id,
+                wait=wait,
+                dispatcher_addr=dispatcher_addr,
+                status_only=status_only,
+                results_dir=results_dir,
+                workflow_output=workflow_output,
+                intermediate_outputs=intermediate_outputs,
+                sublattice_results=sublattice_results,
+                qelectron_db=qelectron_db,
+            )
 
-    return _get_result_multistage(
-        dispatch_id=dispatch_id,
-        wait=wait,
-        dispatcher_addr=dispatcher_addr,
-        status_only=status_only,
-        results_dir=results_dir,
-        workflow_output=workflow_output,
-        intermediate_outputs=intermediate_outputs,
-        sublattice_results=sublattice_results,
-        qelectron_db=qelectron_db,
-    )
+        except RecursionError as re:
+            app_log.error(re)
+            num_attempts += 1
+    raise RuntimeError("Timed out waiting for result. Please retry or check dispatch.")
diff --git a/tests/covalent_tests/results_manager_tests/results_manager_test.py b/tests/covalent_tests/results_manager_tests/results_manager_test.py
@@ -261,6 +261,19 @@ def test_get_result_404(mocker):
         get_result(dispatch_id)
 
 
+def test_get_result_RecursionError(mocker):
+    """Check exception handing for RecursionError."""
+
+    dispatch_id = "test_get_result_RecursionError"
+
+    with pytest.raises(RuntimeError):
+        with mocker.patch(
+            "covalent._results_manager.results_manager._get_result_multistage",
+            side_effect=RecursionError("from test_get_result_RecursionError"),
+        ):
+            get_result(dispatch_id, wait=True)
+
+
 def test_get_status_only(mocker):
     """Check get_result when status_only=True"""