Merge branch 'develop' into electron_pow

.github/workflows/requirements.yml

@@ -39,14 +39,17 @@ jobs:
         run: python -m pip install pip-check-reqs
 
       - name: Check extra core requirements
-        run: pip-extra-reqs -r werkzeug covalent covalent_dispatcher covalent_ui --ignore-requirement=qiskit --ignore-requirement=qiskit-ibm-provider --ignore-requirement=amazon-braket-pennylane-plugin
+        run: pip-extra-reqs -r werkzeug -r python-multipart covalent covalent_dispatcher covalent_ui --ignore-requirement=qiskit --ignore-requirement=qiskit-ibm-provider --ignore-requirement=amazon-braket-pennylane-plugin
 
       - name: Check missing SDK requirements
         run: >
           pip-missing-reqs
           --ignore-module=covalent_ui.*
+          --ignore-module=covalent.*
           --ignore-module=urllib3.*
           --ignore-module=pkg_resources
+          --ignore-module=covalent/_dispatcher_plugins
+          --ignore-module=covalent/_shared_files
           --ignore-file=covalent/executor/**
           --ignore-file=covalent/triggers/**
           --ignore-file=covalent/cloud_resource_manager/**
@@ -58,7 +61,7 @@ jobs:
           pip-missing-reqs
           --ignore-module=covalent_ui.*
           --ignore-module=covalent.*
-          --ignore-module=covalent_dispatcher
+          --ignore-module=covalent_dispatcher.*
           --ignore-module=distributed.*
           covalent_dispatcher
 

.github/workflows/tests.yml

@@ -236,7 +236,7 @@ jobs:
         if: >
           steps.modified-files.outputs.sdk == 'true'
           || env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_tests --cov=covalent --cov-config=.coveragerc
+        run: PYTHONPATH=$PWD/ pytest -vvs  --reruns=5 tests/covalent_tests --cov=covalent --cov-config=.coveragerc
 
       - name: Generate SDK coverage report
         id: sdk-coverage
@@ -248,7 +248,7 @@ jobs:
         if: >
           steps.modified-files.outputs.dispatcher == 'true'
           || env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_dispatcher_tests --cov=covalent_dispatcher --cov-config=.coveragerc
+        run: PYTHONPATH=$PWD/ pytest -vvs  --reruns=5 tests/covalent_dispatcher_tests --cov=covalent_dispatcher --cov-config=.coveragerc
 
       - name: Generate dispatcher coverage report
         id: dispatcher-coverage
@@ -260,7 +260,7 @@ jobs:
         if: >
           steps.modified-files.outputs.ui_backend == 'true'
           || env.BUILD_AND_RUN_ALL
-        run: PYTHONPATH=$PWD/ pytest -vvs --reruns=5 tests/covalent_ui_backend_tests --cov=covalent_ui --cov-config=.coveragerc
+        run: PYTHONPATH=$PWD/ pytest -vvs  --reruns=5 tests/covalent_ui_backend_tests --cov=covalent_ui --cov-config=.coveragerc
 
       - name: Generate UI backend coverage report
         id: ui-backend-coverage

CHANGELOG.md

@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Removed strict version pins on `lmdbm`, `mpire`, `orjson`, and `pennylane`
 - Changed license to Apache
 - Improved error handling in generate_docs.py
+- [Significant Changes] Migrated core server-side code to new data access layer.
 
 ### Added
 
@@ -73,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Rsync command fixed to recursively copy files when using SSH
 - Removed accidentally added migrations build files
 - Updated migration script to add a default value for `qelectron_data_exists` in the `electrons` table since it cannot be nullable
+- Reduced server memory consumption during workflow processing
 
 ### Changed
 

covalent/__init__.py

@@ -25,7 +25,11 @@
 from ._dispatcher_plugins import local_redispatch as redispatch  # nopycln: import
 from ._dispatcher_plugins import stop_triggers  # nopycln: import
 from ._file_transfer import strategies as fs_strategies  # nopycln: import
-from ._results_manager.results_manager import cancel, get_result, sync  # nopycln: import
+from ._results_manager.results_manager import (  # nopycln: import
+    cancel,
+    get_result,
+    get_result_manager,
+)
 from ._shared_files.config import get_config, reload_config, set_config  # nopycln: import
 from ._shared_files.util_classes import RESULT_STATUS as status  # nopycln: import
 from ._workflow import (  # nopycln: import

covalent/_api/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

covalent/_api/apiclient.py

@@ -0,0 +1,128 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""API client"""
+
+import json
+import os
+from typing import Dict
+
+import requests
+from requests.adapters import HTTPAdapter
+
+
+class CovalentAPIClient:
+    """Thin wrapper around Requests to centralize error handling."""
+
+    def __init__(self, dispatcher_addr: str, adapter: HTTPAdapter = None, auto_raise: bool = True):
+        self.dispatcher_addr = dispatcher_addr
+        self.adapter = adapter
+        self.auto_raise = auto_raise
+
+    def prepare_headers(self, **kwargs):
+        extra_headers = CovalentAPIClient.get_extra_headers()
+        headers = kwargs.get("headers", {})
+        if headers:
+            kwargs.pop("headers")
+        headers.update(extra_headers)
+        return headers
+
+    def get(self, endpoint: str, **kwargs):
+        headers = self.prepare_headers(**kwargs)
+        url = self.dispatcher_addr + endpoint
+        try:
+            with requests.Session() as session:
+                if self.adapter:
+                    session.mount("http://", self.adapter)
+
+                r = session.get(url, headers=headers, **kwargs)
+
+            if self.auto_raise:
+                r.raise_for_status()
+
+        except requests.exceptions.ConnectionError:
+            message = f"The Covalent server cannot be reached at {url}. Local servers can be started using `covalent start` in the terminal. If you are using a remote Covalent server, contact your systems administrator to report an outage."
+            print(message)
+            raise
+
+        return r
+
+    def put(self, endpoint: str, **kwargs):
+        headers = self.prepare_headers()
+        url = self.dispatcher_addr + endpoint
+        try:
+            with requests.Session() as session:
+                if self.adapter:
+                    session.mount("http://", self.adapter)
+
+                r = session.put(url, headers=headers, **kwargs)
+
+            if self.auto_raise:
+                r.raise_for_status()
+        except requests.exceptions.ConnectionError:
+            message = f"The Covalent server cannot be reached at {url}. Local servers can be started using `covalent start` in the terminal. If you are using a remote Covalent server, contact your systems administrator to report an outage."
+            print(message)
+            raise
+
+        return r
+
+    def post(self, endpoint: str, **kwargs):
+        headers = self.prepare_headers()
+        url = self.dispatcher_addr + endpoint
+        try:
+            with requests.Session() as session:
+                if self.adapter:
+                    session.mount("http://", self.adapter)
+
+                r = session.post(url, headers=headers, **kwargs)
+
+            if self.auto_raise:
+                r.raise_for_status()
+        except requests.exceptions.ConnectionError:
+            message = f"The Covalent server cannot be reached at {url}. Local servers can be started using `covalent start` in the terminal. If you are using a remote Covalent server, contact your systems administrator to report an outage."
+            print(message)
+            raise
+
+        return r
+
+    def delete(self, endpoint: str, **kwargs):
+        headers = self.prepare_headers()
+        url = self.dispatcher_addr + endpoint
+        try:
+            with requests.Session() as session:
+                if self.adapter:
+                    session.mount("http://", self.adapter)
+
+                r = session.delete(url, headers=headers, **kwargs)
+
+            if self.auto_raise:
+                r.raise_for_status()
+        except requests.exceptions.ConnectionError:
+            message = f"The Covalent server cannot be reached at {url}. Local servers can be started using `covalent start` in the terminal. If you are using a remote Covalent server, contact your systems administrator to report an outage."
+            print(message)
+            raise
+
+        return r
+
+    @classmethod
+    def get_extra_headers(headers: Dict) -> Dict:
+        # This is expected to be a JSONified dictionary
+        data = os.environ.get("COVALENT_EXTRA_HEADERS")
+        if data:
+            return json.loads(data)
+        else:
+            return {}