From 93dce58ba3c03e93aca64a1c9fadf594c363c243 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Thu, 22 Feb 2024 17:19:00 -0600
Subject: [PATCH 01/29] capsule some of the request calls

---
 src/cript/api/api.py         | 135 ++++++++++++++++-------------------
 src/cript/api/data_schema.py |  53 +++++++-------
 2 files changed, 86 insertions(+), 102 deletions(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 869007c33..7e1d15502 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -3,7 +3,6 @@
 import logging
 import os
 import uuid
-import warnings
 from pathlib import Path
 from typing import Any, Dict, Optional, Union
 
@@ -17,9 +16,7 @@
     APIError,
     CRIPTAPIRequiredError,
     CRIPTAPISaveError,
-    CRIPTConnectionError,
     CRIPTDuplicateNameError,
-    InvalidHostError,
 )
 from cript.api.paginator import Paginator
 from cript.api.utils.aws_s3_utils import get_s3_client
@@ -208,7 +205,7 @@ def __init__(self, host: Union[str, None] = None, api_token: Union[str, None] =
             api_token = authentication_dict["api_token"]
             storage_token = authentication_dict["storage_token"]
 
-        self._host = self._prepare_host(host=host)  # type: ignore
+        self._host: str = host.rstrip("/")
         self._api_token = api_token  # type: ignore
         self._storage_token = storage_token  # type: ignore
 
@@ -220,7 +217,7 @@ def __init__(self, host: Union[str, None] = None, api_token: Union[str, None] =
 
         # set a logger instance to use for the class logs
         self._init_logger(default_log_level)
-        self._db_schema = DataSchema(self.host, self.logger)
+        self._db_schema = DataSchema(self)
 
     def __str__(self) -> str:
         """
@@ -283,46 +280,6 @@ def _init_logger(self, log_level=logging.INFO) -> None:
     def logger(self):
         return self._logger
 
-    @beartype
-    def _prepare_host(self, host: str) -> str:
-        """
-        Takes the host URL provided by the user during API object construction (e.g., `https://api.criptapp.org`)
-        and standardizes it for internal use. Performs any required string manipulation to ensure uniformity.
-
-        Parameters
-        ----------
-        host: str
-            The host URL specified during API initialization, typically in the form `https://api.criptapp.org`.
-
-        Warnings
-        --------
-        If the specified host uses the unsafe "http://" protocol, a warning will be raised to consider using HTTPS.
-
-        Raises
-        ------
-        InvalidHostError
-            If the host string does not start with either "http" or "https", an InvalidHostError will be raised.
-            Only HTTP protocol is acceptable at this time.
-
-        Returns
-        -------
-        str
-            A standardized host string formatted for internal use.
-
-        """
-        # strip ending slash to make host always uniform
-        host = host.rstrip("/")
-        host = f"{host}/{self._api_prefix}/{self._api_version}"
-
-        # if host is using unsafe "http://" then give a warning
-        if host.startswith("http://"):
-            warnings.warn("HTTP is an unsafe protocol please consider using HTTPS.")
-
-        if not host.startswith("http"):
-            raise InvalidHostError()
-
-        return host
-
     # Use a property to ensure delayed init of s3_client
     @property
     def _s3_client(self) -> boto3.client:  # type: ignore
@@ -414,24 +371,6 @@ def host(self):
         """
         return self._host
 
-    def _check_initial_host_connection(self) -> None:
-        """
-        tries to create a connection with host and if the host does not respond or is invalid it raises an error
-
-        Raises
-        -------
-        CRIPTConnectionError
-            raised when the host does not give the expected response
-
-        Returns
-        -------
-        None
-        """
-        try:
-            pass
-        except Exception as exc:
-            raise CRIPTConnectionError(self.host, self._api_token) from exc
-
     def save(self, project: Project) -> None:
         """
         This method takes a project node, serializes the class into JSON
@@ -497,7 +436,7 @@ def _internal_save(self, node, save_values: Optional[_InternalSaveValues] = None
 
             # This checks if the current node exists on the back end.
             # if it does exist we use `patch` if it doesn't `post`.
-            test_get_response: Dict = requests.get(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}/", headers=self._http_headers, timeout=_API_TIMEOUT).json()
+            test_get_response: Dict = self._capsule_request(url_path=f"/{node.node_type_snake_case}/{str(node.uuid)}/", method="GET").json()
             patch_request = test_get_response["code"] == 200
 
             # TODO remove once get works properly
@@ -512,13 +451,15 @@ def _internal_save(self, node, save_values: Optional[_InternalSaveValues] = None
                 response = {"code": 200}
                 break
 
+            method = "POST"
             if patch_request:
-                response: Dict = requests.patch(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}/", headers=self._http_headers, data=json_data, timeout=_API_TIMEOUT).json()  # type: ignore
-            else:
-                response: Dict = requests.post(url=f"{self._host}/{node.node_type_snake_case}/", headers=self._http_headers, data=json_data, timeout=_API_TIMEOUT).json()  # type: ignore
-                # if node.node_type != "Project":
-                #     test_success: Dict = requests.get(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}/", headers=self._http_headers, timeout=_API_TIMEOUT).json()
-                #     print("XYZ", json_data, save_values, response, test_success)
+                method = "PATCH"
+
+            response: Dict = self._capsule_request(url=f"/{node.node_type_snake_case}/{str(node.uuid)}/", method=method, data=json_data).json()  # type: ignore
+
+            # if node.node_type != "Project":
+            #     test_success: Dict = requests.get(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}/", headers=self._http_headers, timeout=_API_TIMEOUT).json()
+            #     print("XYZ", json_data, save_values, response, test_success)
 
             # print(json_data, patch_request, response, save_values)
             # If we get an error we may be able to fix, we to handle this extra and save the bad node first.
@@ -997,11 +938,59 @@ def delete_node_by_uuid(self, node_type: str, node_uuid: str) -> None:
         -------
         None
         """
-        delete_node_api_url: str = f"{self._host}/{node_type.lower()}/{node_uuid}/"
 
-        response: Dict = requests.delete(headers=self._http_headers, url=delete_node_api_url, timeout=_API_TIMEOUT).json()
+        response: Dict = self._capsule_request(url_path=f"/{node_type.lower()}/{node_uuid}/", method="DELETE").json()
 
         if response["code"] != 200:
-            raise APIError(api_error=str(response), http_method="DELETE", api_url=delete_node_api_url)
+            raise APIError(api_error=str(response), http_method="DELETE", api_url=f"/{node_type.lower()}/{node_uuid}/")
 
         self.logger.info(f"Deleted '{node_type.title()}' with UUID of '{node_uuid}' from CRIPT API.")
+
+    def _capsule_request(self, url_path: str, method: str, api_request: bool = True, headers: Optional[Dict] = None, timeout: int = _API_TIMEOUT, **kwargs) -> requests.Response:
+        """Helper function that capsules every request call we make against the backend.
+
+        Please *always* use this methods instead of `requests` directly.
+        We can log all request calls this way, which can help debugging immensely.
+
+        Parameters
+        ----------
+        url_path:str
+          URL path that we want to request from. So every thing that follows api.host. You can omit the api prefix and api version if you use api_request=True they are automatically added.
+
+        method: str
+          One of `GET`, `OPTIONS`, `HEAD`, `POST`, `PUT, `PATCH`, or `DELETE` as this will directly passed to `requests.request(...)`. See https://docs.python-requests.org/en/latest/api/ for details.
+
+        headers: Dict
+          HTTPS headers to use for the request.
+          If None (default) use the once associated with this API object for authentication.
+
+        timeout:int
+          Time out to be used for the request call.
+
+        kwargs
+          additional keyword arguments that are passed to `request.request`
+        """
+
+        extra_debug_info: bool = True
+
+        if headers is None:
+            headers = self._http_headers
+
+        url: str = self.host
+        if api_request:
+            url += f"/{self._api_prefix}/{self._api_version}"
+        url += url_path
+
+        pre_log_message: str = f"Requesting {method} from {url}"
+        if extra_debug_info:
+            pre_log_message += f"headers {headers} kwargs {kwargs}"
+        pre_log_message += "..."
+        self.logger.debug(pre_log_message)
+
+        response: requests.Response = requests.request(url=url, method=method, headers=headers, timeout=timeout, **kwargs)
+        post_log_message: str = f"Request return with {response.status_code}"
+        if extra_debug_info:
+            post_log_message += f" {response}"
+        self.logger.debug(post_log_message)
+
+        return response
diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py
index 925209f21..cf5a0ff0b 100644
--- a/src/cript/api/data_schema.py
+++ b/src/cript/api/data_schema.py
@@ -2,10 +2,8 @@
 from typing import Union
 
 import jsonschema
-import requests
 from beartype import beartype
 
-from cript.api.api_config import _API_TIMEOUT
 from cript.api.exceptions import APIError, InvalidVocabulary
 from cript.api.utils.helper_functions import _get_node_type_from_json
 from cript.api.vocabulary_categories import VocabCategories
@@ -20,14 +18,30 @@ class DataSchema:
 
     _vocabulary: dict = {}
     _db_schema: dict = {}
-    _logger = None
+    _api = None
     # Advanced User Tip: Disabling Node Validation
     # For experienced users, deactivating node validation during creation can be a time-saver.
     # Note that the complete node graph will still undergo validation before being saved to the back end.
     # Caution: It's advisable to keep validation active while debugging scripts, as disabling it can delay error notifications and complicate the debugging process.
     skip_validation: bool = False
 
-    def _get_db_schema(self, host: str) -> dict:
+    def __init__(self, api):
+        """
+        Initialize DataSchema class with a full hostname to fetch the node validation schema.
+
+        Examples
+        --------
+        ### Create a stand alone DataSchema instance.
+        >>> import cript
+        >>> with cript.API(host="https://api.criptapp.org/") as api:
+        ...    data_schema = cript.api.DataSchema(api)
+        """
+
+        self._db_schema = self._get_db_schema()
+        self._vocabulary = self._get_vocab()
+        self._api = api
+
+    def _get_db_schema(self) -> dict:
         """
         Sends a GET request to CRIPT to get the database schema and returns it.
         The database schema can be used for validating the JSON request
@@ -45,15 +59,13 @@ def _get_db_schema(self, host: str) -> dict:
             return self._db_schema
 
         # fetch db_schema from API
-        if self._logger:
-            self._logger.info(f"Loading node validation schema from {host}/schema/")
+        self._api.logger.info(f"Loading node validation schema from {self._api.host}/schema/")
         # fetch db schema from API
-        response: requests.Response = requests.get(url=f"{host}/schema/", timeout=_API_TIMEOUT)
+        response = self._api._capsule_request(url_path="/schema/", method="GET")
 
         # raise error if not HTTP 200
         response.raise_for_status()
-        if self._logger:
-            self._logger.info(f"Loading node validation schema from {host}/schema/ was successful.")
+        self._api.logger.info(f"Loading node validation schema from {self._api.host}/schema/ was successful.")
 
         # if no error, take the JSON from the API response
         response_dict: dict = response.json()
@@ -63,22 +75,6 @@ def _get_db_schema(self, host: str) -> dict:
 
         return db_schema
 
-    def __init__(self, host: str, logger=None):
-        """
-        Initialize DataSchema class with a full hostname to fetch the node validation schema.
-
-        Examples
-        --------
-        ### Create a stand alone DataSchema instance.
-        >>> import cript
-        >>> with cript.API(host="https://api.criptapp.org/") as api:
-        ...    data_schema = cript.api.DataSchema(api.host)
-        """
-
-        self._db_schema = self._get_db_schema(host)
-        self._vocabulary = self._get_vocab(host)
-        self._logger = logger
-
     def _get_vocab(self, host: str) -> dict:
         """
         gets the entire CRIPT controlled vocabulary and stores it in _vocabulary
@@ -107,10 +103,10 @@ def _get_vocab(self, host: str) -> dict:
         # loop through all vocabulary categories and make a request to each vocabulary category
         # and put them all inside of self._vocab with the keys being the vocab category name
         for category in VocabCategories:
-            vocabulary_category_url: str = f"{host}/cv/{category.value}/"
+            vocabulary_category_url: str = f"/cv/{category.value}/"
 
             # if vocabulary category is not in cache, then get it from API and cache it
-            response: dict = requests.get(url=vocabulary_category_url, timeout=_API_TIMEOUT).json()
+            response: dict = self._api._capsule_request(url_path=vocabulary_category_url, method="GET").json()
 
             if response["code"] != 200:
                 raise APIError(api_error=str(response), http_method="GET", api_url=vocabulary_category_url)
@@ -247,8 +243,7 @@ def is_node_schema_valid(self, node_json: str, is_patch: bool = False, force_val
         else:
             log_message += " (Can be disabled by setting `cript.API.skip_validation = True`.)"
 
-        if self._logger:
-            self._logger.info(log_message)
+        self._api.logger.info(log_message)
 
         # set the schema to test against http POST or PATCH of DB Schema
         schema_http_method: str

From 807d72d6526db3880fe5c4741fbd360792e216c0 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 10:53:49 -0600
Subject: [PATCH 02/29] final touches

---
 conftest.py                  |  3 +++
 src/cript/api/api.py         | 20 ++++++++++----------
 src/cript/api/data_schema.py |  5 ++---
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/conftest.py b/conftest.py
index 51081d5fb..d2a51c84e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -8,6 +8,7 @@
 
 The fixtures are all functional fixtures that stay consistent between all tests.
 """
+import logging
 import os
 
 import pytest
@@ -57,6 +58,8 @@ def cript_api():
         api._BUCKET_NAME = "cript-stage-user-data"
         # using the tests folder name within our cloud storage
         api._BUCKET_DIRECTORY_NAME = "tests"
+        api.extra_api_log_debug_info = True
+        api.logger.setLevel(logging.DEBUG)
 
         yield api
 
diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 7e1d15502..5ffd81e68 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+import traceback
 import uuid
 from pathlib import Path
 from typing import Any, Dict, Optional, Union
@@ -72,6 +73,8 @@ class API:
     _internal_s3_client: Any = None  # type: ignore
     # trunk-ignore-end(cspell)
 
+    extra_api_log_debug_info: bool = False
+
     @beartype
     def __init__(self, host: Union[str, None] = None, api_token: Union[str, None] = None, storage_token: Union[str, None] = None, config_file_path: Union[str, Path] = "", default_log_level=logging.INFO):
         """
@@ -212,9 +215,6 @@ def __init__(self, host: Union[str, None] = None, api_token: Union[str, None] =
         # add Bearer to token for HTTP requests
         self._http_headers = {"Authorization": f"Bearer {self._api_token}", "Content-Type": "application/json"}
 
-        # check that api can connect to CRIPT with host and token
-        self._check_initial_host_connection()
-
         # set a logger instance to use for the class logs
         self._init_logger(default_log_level)
         self._db_schema = DataSchema(self)
@@ -452,10 +452,12 @@ def _internal_save(self, node, save_values: Optional[_InternalSaveValues] = None
                 break
 
             method = "POST"
+            url_path = f"/{node.node_type_snake_case}/"
             if patch_request:
                 method = "PATCH"
+                url_path += f"{str(node.uuid)}/"
 
-            response: Dict = self._capsule_request(url=f"/{node.node_type_snake_case}/{str(node.uuid)}/", method=method, data=json_data).json()  # type: ignore
+            response: Dict = self._capsule_request(url_path=url_path, method=method, data=json_data).json()  # type: ignore
 
             # if node.node_type != "Project":
             #     test_success: Dict = requests.get(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}/", headers=self._http_headers, timeout=_API_TIMEOUT).json()
@@ -971,8 +973,6 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True,
           additional keyword arguments that are passed to `request.request`
         """
 
-        extra_debug_info: bool = True
-
         if headers is None:
             headers = self._http_headers
 
@@ -982,15 +982,15 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True,
         url += url_path
 
         pre_log_message: str = f"Requesting {method} from {url}"
-        if extra_debug_info:
-            pre_log_message += f"headers {headers} kwargs {kwargs}"
+        if self.extra_api_log_debug_info:
+            pre_log_message += f" from {traceback.format_stack(limit=4)} kwargs {kwargs}"
         pre_log_message += "..."
         self.logger.debug(pre_log_message)
 
         response: requests.Response = requests.request(url=url, method=method, headers=headers, timeout=timeout, **kwargs)
         post_log_message: str = f"Request return with {response.status_code}"
-        if extra_debug_info:
-            post_log_message += f" {response}"
+        if self.extra_api_log_debug_info:
+            post_log_message += f" {response.json()}"
         self.logger.debug(post_log_message)
 
         return response
diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py
index cf5a0ff0b..4a8594015 100644
--- a/src/cript/api/data_schema.py
+++ b/src/cript/api/data_schema.py
@@ -36,10 +36,9 @@ def __init__(self, api):
         >>> with cript.API(host="https://api.criptapp.org/") as api:
         ...    data_schema = cript.api.DataSchema(api)
         """
-
+        self._api = api
         self._db_schema = self._get_db_schema()
         self._vocabulary = self._get_vocab()
-        self._api = api
 
     def _get_db_schema(self) -> dict:
         """
@@ -75,7 +74,7 @@ def _get_db_schema(self) -> dict:
 
         return db_schema
 
-    def _get_vocab(self, host: str) -> dict:
+    def _get_vocab(self) -> dict:
         """
         gets the entire CRIPT controlled vocabulary and stores it in _vocabulary
 

From b6eac0b03b139da9a106807034436b681da3530f Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 11:02:38 -0600
Subject: [PATCH 03/29] make mypy happy

---
 src/cript/api/data_schema.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py
index 4a8594015..a529bb72c 100644
--- a/src/cript/api/data_schema.py
+++ b/src/cript/api/data_schema.py
@@ -18,7 +18,6 @@ class DataSchema:
 
     _vocabulary: dict = {}
     _db_schema: dict = {}
-    _api = None
     # Advanced User Tip: Disabling Node Validation
     # For experienced users, deactivating node validation during creation can be a time-saver.
     # Note that the complete node graph will still undergo validation before being saved to the back end.

From a0faedde9bf66ffffe00cf245b8a4b2036536903 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Wed, 21 Feb 2024 09:04:11 -0600
Subject: [PATCH 04/29] add missing file

---
 src/cript/nodes/util/json.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/cript/nodes/util/json.py b/src/cript/nodes/util/json.py
index e03435afe..79dae1f17 100644
--- a/src/cript/nodes/util/json.py
+++ b/src/cript/nodes/util/json.py
@@ -26,6 +26,7 @@ class NodeEncoder(json.JSONEncoder):
 
     Attributes
     ----------
+<<<<<<< HEAD
     handled_ids : Set[str]
         A set to store the UIDs of nodes that have been processed during serialization.
     known_uuid : Set[str]
@@ -33,6 +34,15 @@ class NodeEncoder(json.JSONEncoder):
     condense_to_uuid : Dict[str, Set[str]]
         A set to store the node types that should be condensed to UUID edges in the JSON.
     suppress_attributes : Optional[Dict[str, Set[str]]]
+=======
+    handled_ids : set[str]
+        A set to store the UIDs of nodes that have been processed during serialization.
+    known_uuid : set[str]
+        A set to store the UUIDs of nodes that have been previously encountered in the JSON.
+    condense_to_uuid : dict[str, set[str]]
+        A set to store the node types that should be condensed to UUID edges in the JSON.
+    suppress_attributes : Optional[dict[str, set[str]]]
+>>>>>>> 44cc898 (add missing file)
         A dictionary that allows suppressing specific attributes for nodes with the corresponding UUIDs.
 
     Methods
@@ -43,7 +53,11 @@ class NodeEncoder(json.JSONEncoder):
     ```
 
     ```python
+<<<<<<< HEAD
     _apply_modifications(self, serialize_dict: Dict) -> Tuple[Dict, List[str]]:
+=======
+    _apply_modifications(self, serialize_dict: dict) -> Tuple[dict, list[str]]:
+>>>>>>> 44cc898 (add missing file)
         # Apply modifications to the serialized dictionary based on node types
         # and attributes to be condensed. This internal function handles node
         # condensation and attribute suppression during serialization.
@@ -145,6 +159,7 @@ def _apply_modifications(self, serialize_dict: Dict):
 
         Parameters
         ----------
+<<<<<<< HEAD
         serialize_dict: Dict
 
         Returns

From d6c25f98fd7c0ed87ab6853f6571df913517cc25 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Tue, 20 Feb 2024 19:05:21 -0600
Subject: [PATCH 05/29] implement the new paginator design

---
 src/cript/api/api.py                   |  39 +-----
 src/cript/api/paginator.py             | 181 ++++++++-----------------
 src/cript/nodes/subobjects/#*scratch*# |  19 +++
 src/cript/nodes/subobjects/*scratch*   |  17 +++
 4 files changed, 100 insertions(+), 156 deletions(-)
 create mode 100644 src/cript/nodes/subobjects/#*scratch*#
 create mode 100644 src/cript/nodes/subobjects/*scratch*

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 869007c33..86cc94f5f 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -745,7 +745,7 @@ def search(
         --------
         ???+ Example "Search by Node Type"
             ```python
-            materials_paginator = cript_api.search(
+            materials_iterator = cript_api.search(
                 node_type=cript.Material,
                 search_mode=cript.SearchModes.NODE_TYPE,
                 value_to_search=None
@@ -754,7 +754,7 @@ def search(
 
         ??? Example "Search by Contains name"
             ```python
-            contains_name_paginator = cript_api.search(
+            contains_name_iterator = cript_api.search(
                 node_type=cript.Process,
                 search_mode=cript.SearchModes.CONTAINS_NAME,
                 value_to_search="poly"
@@ -763,7 +763,7 @@ def search(
 
         ??? Example "Search by Exact Name"
             ```python
-            exact_name_paginator = cript_api.search(
+            exact_name_iterator = cript_api.search(
                 node_type=cript.Project,
                 search_mode=cript.SearchModes.EXACT_NAME,
                 value_to_search="Sodium polystyrene sulfonate"
@@ -772,7 +772,7 @@ def search(
 
         ??? Example "Search by UUID"
             ```python
-            uuid_paginator = cript_api.search(
+            uuid_iterator = cript_api.search(
                 node_type=cript.Collection,
                 search_mode=cript.SearchModes.UUID,
                 value_to_search="75fd3ee5-48c2-4fc7-8d0b-842f4fc812b7"
@@ -781,7 +781,7 @@ def search(
 
         ??? Example "Search by BigSmiles"
             ```python
-            paginator = cript_api.search(
+            iterator = cript_api.search(
                 node_type=cript.Material,
                 search_mode=cript.SearchModes.BIGSMILES,
                 value_to_search="{[][$]CC(C)(C(=O)OCCCC)[$][]}"
@@ -802,42 +802,17 @@ def search(
         Returns
         -------
         Paginator
-            paginator object for the user to use to flip through pages of search results
+            An iterator that will present and fetch the results to the user seamlessly
 
         Notes
         -----
         To learn more about working with pagination, please refer to our
         [paginator object documentation](../paginator).
-
-        Additionally, you can utilize the utility function
-        [`load_nodes_from_json(node_json)`](../../utility_functions/#cript.nodes.util.load_nodes_from_json)
-        to convert API JSON responses into Python SDK nodes.
-
-        ???+ Example "Convert API JSON Response to Python SDK Nodes"
-            ```python
-            # Get updated project from API
-            my_paginator = api.search(
-                node_type=cript.Project,
-                search_mode=cript.SearchModes.EXACT_NAME,
-                value_to_search="my project name",
-            )
-
-            # Take specific Project you want from paginator
-            my_project_from_api_dict: dict = my_paginator.current_page_results[0]
-
-            # Deserialize your Project dict into a Project node
-            my_project_node_from_api = cript.load_nodes_from_json(
-                nodes_json=json.dumps(my_project_from_api_dict)
-            )
-            ```
         """
 
         # get node typ from class
         node_type = node_type.node_type_snake_case
 
-        # always putting a page parameter of 0 for all search URLs
-        page_number = 0
-
         api_endpoint: str = ""
 
         # requesting a page of some primary node
@@ -862,7 +837,7 @@ def search(
         else:
             raise RuntimeError("Internal Error: Failed to recognize any search modes. Please report this bug on https://github.com/C-Accel-CRIPT/Python-SDK/issues.")
 
-        return Paginator(http_headers=self._http_headers, api_endpoint=api_endpoint, query=value_to_search, current_page_number=page_number)
+        return Paginator(http_headers=self._http_headers, api_endpoint=api_endpoint, query=value_to_search)
 
     def delete(self, node) -> None:
         """
diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index f829a0d27..0e15ea285 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -7,6 +7,7 @@
 
 from cript.api.api_config import _API_TIMEOUT
 from cript.api.exceptions import APIError
+from cript.nodes.util import load_nodes_from_json
 
 
 class Paginator:
@@ -14,37 +15,30 @@ class Paginator:
     Paginator is used to flip through different pages of data that the API returns when searching.
     > Instead of the user manipulating the URL and parameters, this object handles all of that for them.
 
-    When conducting any kind of search the API returns pages of data and each page contains 10 results.
-    This is equivalent to conducting a Google search when Google returns a limited number of links on the first page
-    and all other results are on the next pages.
+    Using the Paginator object, the user can simply and easily flip through the results of the search.
+    The details, that results are listed as pages are hidden from the user.
+    The pages are automatically requested from the API as needed.
 
-    Using the Paginator object, the user can simply and easily flip through the pages of data the API provides.
+    This object implements a python iterator, so `for node in Paginator` works as expected.
+    It will loop through all results of the search, returning the nodes one by one.
 
     !!! Warning "Do not create paginator objects"
         Please note that you are not required or advised to create a paginator object, and instead the
         Python SDK API object will create a paginator for you, return it, and let you simply use it
 
-
-    Attributes
-    ----------
-    current_page_results: List[dict]
-        List of JSON dictionary results returned from the API
-        ```python
-        [{result 1}, {result 2}, {result 3}, ...]
-        ```
     """
 
     _http_headers: dict
 
-    api_endpoint: str
+    _api_endpoint: str
 
     # if query or page number are None, then it means that api_endpoint does not allow for whatever that is None
     # and that is not added to the URL
     # by default the page_number and query are `None` and they can get filled in
-    query: Union[str, None]
+    _query: Union[str, None]
     _current_page_number: int
-
-    current_page_results: List[dict]
+    _curren_position: int
+    _fetched_nodes: list["BaseNode"]
 
     @beartype
     def __init__(
@@ -52,7 +46,6 @@ def __init__(
         http_headers: dict,
         api_endpoint: str,
         query: Optional[str] = None,
-        current_page_number: int = 0,
     ):
         """
         create a paginator
@@ -78,127 +71,55 @@ def __init__(
             instantiate a paginator
         """
         self._http_headers = http_headers
-        self.api_endpoint = api_endpoint
-        self.query = query
-        self._current_page_number = current_page_number
+        self._api_endpoint = api_endpoint
+        self._query = query
+        self._current_page_number = 0
+        self._fetched_nodes = []
+        self._current_position = 0
 
         # check if it is a string and not None to avoid AttributeError
-        if api_endpoint is not None:
-            # strip the ending slash "/" to make URL uniform and any trailing spaces from either side
-            self.api_endpoint = api_endpoint.rstrip("/").strip()
+        try:
+            self._api_endpoint = self._api_endpoint.rstrip("/").strip()
+        except AttributeError as exc:
+            if self._api_endpoint is not None:
+                raise RuntimeError(f"Invalid type for api_endpoint {self._api_endpoint} for a paginator.") from exc
 
         # check if it is a string and not None to avoid AttributeError
-        if query is not None:
-            # URL encode query
-            self.query = quote(query)
-
-        self.fetch_page_from_api()
-
-    def next_page(self):
-        """
-        flip to the next page of data.
-
-        Examples
-        --------
-        ```python
-        my_paginator.next_page()
-        ```
-        """
-        self.current_page_number += 1
-
-    def previous_page(self):
-        """
-        flip to the next page of data.
-
-        Examples
-        --------
-        ```python
-        my_paginator.previous_page()
-        ```
-        """
-        self.current_page_number -= 1
-
-    @property
-    @beartype
-    def current_page_number(self) -> int:
-        """
-        get the current page number that you are on.
-
-        Setting the page will take you to that specific page of results
-
-        Examples
-        --------
-        ```python
-        my_paginator.current_page = 10
-        ```
-
-        Returns
-        -------
-        current page number: int
-            the current page number of the data
-        """
-        return self._current_page_number
-
-    @current_page_number.setter
-    @beartype
-    def current_page_number(self, new_page_number: int) -> None:
-        """
-        flips to a specific page of data that has been requested
-
-        sets the current_page_number and then sends the request to the API and gets the results of this page number
-
-        Parameters
-        ----------
-        new_page_number (int): specific page of data that the user wants to go to
-
-        Examples
-        --------
-        requests.get("https://api.criptapp.org//api?page=2)
-        requests.get(f"{self.query}?page={self.current_page_number - 1}")
-
-        Raises
-        --------
-        InvalidPageRequest, in case the user tries to get a negative page or a page that doesn't exist
-        """
-        if new_page_number < 0:
-            error_message: str = f"Paginator current page number is invalid because it is negative: " f"{self.current_page_number} please set paginator.current_page_number " f"to a positive page number"
-
-            raise RuntimeError(error_message)
-
-        else:
-            self._current_page_number = new_page_number
-            # when new page number is set, it is then fetched from the API
-            self.fetch_page_from_api()
+        try:
+            self._query = quote(self._query)
+        except AttributeError as exc:
+            if self._query is not None:
+                raise RuntimeError(f"Invalid type for query {self._query} a paginator.") from exc
 
     @beartype
-    def fetch_page_from_api(self) -> List[dict]:
+    def _fetch_next_page(self) -> None:
         """
         1. builds the URL from the query and page number
         1. makes the request to the API
         1. API responds with a JSON that has data or JSON that has data and result
-            1. parses it and correctly sets the current_page_results property
+            1. parses the response
+            2. creates cript.Nodes from the response
+            3. Add the nodes to the fetched_data so the iterator can return them
 
         Raises
         ------
         InvalidSearchRequest
             In case the API responds with an error
+        StopIteration
+            In case there are no further results to fetch
+
 
         Returns
         -------
-        current page results: List[dict]
-            makes a request to the API and gets a page of data
+             None
         """
 
-        # temporary variable to not overwrite api_endpoint
-        temp_api_endpoint: str = self.api_endpoint
-
-        if self.query is not None:
-            temp_api_endpoint = f"{temp_api_endpoint}/?q={self.query}"
-
-        elif self.query is None:
-            temp_api_endpoint = f"{temp_api_endpoint}/?q="
-
-        temp_api_endpoint = f"{temp_api_endpoint}&page={self.current_page_number}"
+        # Composition of the query URL
+        temp_api_endpoint: str = self._api_endpoint
+        temp_api_endpoint += "/?q="
+        if self._query is not None:
+            temp_api_endpoint += f"{self._query}"
+        temp_api_endpoint += f"&page={self._current_page_number}"
 
         response: requests.Response = requests.get(url=temp_api_endpoint, headers=self._http_headers, timeout=_API_TIMEOUT)
 
@@ -215,18 +136,30 @@ def fetch_page_from_api(self) -> List[dict]:
 
         # handling both cases in case there is result inside of data or just data
         try:
-            self.current_page_results = api_response["data"]["result"]
+            current_page_results = api_response["data"]["result"]
         except KeyError:
-            self.current_page_results = api_response["data"]
+            current_page_results = api_response["data"]
         except TypeError:
-            self.current_page_results = api_response["data"]
+            current_page_results = api_response["data"]
 
         if api_response["code"] == 404 and api_response["error"] == "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.":
-            self.current_page_results = []
-            return self.current_page_results
+            current_page_results = []
 
         # if API response is not 200 raise error for the user to debug
         if api_response["code"] != 200:
             raise APIError(api_error=str(response.json()), http_method="GET", api_url=temp_api_endpoint)
+        if len(current_page_results) == 0:
+            raise StopIteration
+
+        node_list = load_nodes_from_json(current_page_results)
+        self._fetched_nodes += node_list
+
+    def __next__(self):
+        if self._current_position >= len(self._fetched_nodes):
+            self._fetch_next_page()
+        self._current_position += 1
+        return self._fetched_nodes[self._current_position - 1]
 
-        return self.current_page_results
+    def __iter__(self):
+        self._current_position = 0
+        return self
diff --git a/src/cript/nodes/subobjects/#*scratch*# b/src/cript/nodes/subobjects/#*scratch*#
new file mode 100644
index 000000000..cadec5f2a
--- /dev/null
+++ b/src/cript/nodes/subobjects/#*scratch*#
@@ -0,0 +1,19 @@
+;; This buffer is for text that is not saved, and for Lisp evaluation.
+;; To create a file, visit it with C-x C-f and enter text in its buffer.
+
+
+
+my_material = cript.Material()
+
+assert my_material.updated_at == None
+
+api.save(Material)
+
+assert my_material.updated_at == None
+
+downloaded_material = api.get(uuid=my_material.uuid)
+
+assert my_material.updated_at != None
+assert my_material.updated_at == "asdfcience"
+
+assert downloaded_material is my_material
diff --git a/src/cript/nodes/subobjects/*scratch* b/src/cript/nodes/subobjects/*scratch*
new file mode 100644
index 000000000..aaed34389
--- /dev/null
+++ b/src/cript/nodes/subobjects/*scratch*
@@ -0,0 +1,17 @@
+;; This buffer is for text that is not saved, and for Lisp evaluation.
+;; To create a file, visit it with C-x C-f and enter text in its buffer.
+
+
+
+my_material = cript.Material()
+
+assert my_material.updated_at == None
+
+api.save(Material)
+
+assert my_material.updated_at == None
+
+downloaded_material = api.get(uuid=my_material.uuid)
+
+assert my_material.updated_at != None
+assert my_material.updated_at == "asdfcience"

From ea4aa42b42f818aebc2f20f7e046643d58e39985 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Tue, 20 Feb 2024 19:05:49 -0600
Subject: [PATCH 06/29] adjust tests to the new design

---
 tests/api/test_api.py                  | 37 +++++++++++---------------
 tests/fixtures/api_fixtures.py         |  3 ++-
 tests/utils/integration_test_helper.py |  9 +++----
 3 files changed, 22 insertions(+), 27 deletions(-)

diff --git a/tests/api/test_api.py b/tests/api/test_api.py
index e4323b7ee..bfeadf5d6 100644
--- a/tests/api/test_api.py
+++ b/tests/api/test_api.py
@@ -169,25 +169,14 @@ def test_api_search_node_type(cript_api: cript.API) -> None:
 
     # test search results
     assert isinstance(materials_paginator, Paginator)
-    assert len(materials_paginator.current_page_results) > 5
-    first_page_first_result = materials_paginator.current_page_results[0]["name"]
-
+    materials_list = list(materials_paginator)
+    # Assure that we paginated more then one page
+    assert materials_paginator._current_page_number > 0
+    assert len(materials_list) > 5
+    first_page_first_result = materials_list[0]["name"]
     # just checking that the word has a few characters in it
     assert len(first_page_first_result) > 3
 
-    # tests that it can correctly go to the next page
-    materials_paginator.next_page()
-    assert len(materials_paginator.current_page_results) > 5
-    second_page_first_result = materials_paginator.current_page_results[0]["name"]
-
-    assert len(second_page_first_result) > 3
-
-    # tests that it can correctly go to the previous page
-    materials_paginator.previous_page()
-    assert len(materials_paginator.current_page_results) > 5
-
-    assert len(first_page_first_result) > 3
-
 
 @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
 def test_api_search_contains_name(cript_api: cript.API) -> None:
@@ -198,9 +187,12 @@ def test_api_search_contains_name(cript_api: cript.API) -> None:
     contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
 
     assert isinstance(contains_name_paginator, Paginator)
-    assert len(contains_name_paginator.current_page_results) > 5
+    contains_name_list = list(contains_name_paginator)
+    # Assure that we paginated more then one page
+    assert contains_name_paginator._current_page_number > 0
+    assert len(contains_name_list) > 5
 
-    contains_name_first_result = contains_name_paginator.current_page_results[0]["name"]
+    contains_name_first_result = contains_name_list["name"]
 
     # just checking that the result has a few characters in it
     assert len(contains_name_first_result) > 3
@@ -215,7 +207,8 @@ def test_api_search_exact_name(cript_api: cript.API) -> None:
     exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
 
     assert isinstance(exact_name_paginator, Paginator)
-    assert len(exact_name_paginator.current_page_results) == 1
+    exact_name_list = list(exact_name_paginator)
+    assert len(exact_name_list) == 1
     assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
 
 
@@ -233,7 +226,8 @@ def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
     uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
 
     assert isinstance(uuid_paginator, Paginator)
-    assert len(uuid_paginator.current_page_results) == 1
+    uuid_list = list(uuid_paginator)
+    assert len(uuid_list) == 1
     assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
     assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
 
@@ -252,7 +246,8 @@ def test_api_search_bigsmiles(cript_api: cript.API) -> None:
     bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
 
     assert isinstance(bigsmiles_paginator, Paginator)
-    assert len(bigsmiles_paginator.current_page_results) >= 1
+    bigsmiles_list = list(bigsmiles_paginator)
+    assert len(bigsmiles_list) >= 1
     # not sure if this will always be in this position in every server environment, so commenting it out for now
     # assert bigsmiles_paginator.current_page_results[1]["name"] == "BCDB_Material_285"
 
diff --git a/tests/fixtures/api_fixtures.py b/tests/fixtures/api_fixtures.py
index 82391b5ee..062c19f75 100644
--- a/tests/fixtures/api_fixtures.py
+++ b/tests/fixtures/api_fixtures.py
@@ -26,6 +26,7 @@ def dynamic_material_data(cript_api: cript.API) -> Dict[str, str]:
 
     exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search=material_name)
 
-    material_uuid: str = exact_name_paginator.current_page_results[0]["uuid"]
+    material = next(exact_name_paginator)
+    material_uuid: str = str(material.uuid)
 
     return {"name": material_name, "uuid": material_uuid}
diff --git a/tests/utils/integration_test_helper.py b/tests/utils/integration_test_helper.py
index 0639a9028..6c319fdbe 100644
--- a/tests/utils/integration_test_helper.py
+++ b/tests/utils/integration_test_helper.py
@@ -58,10 +58,10 @@ def save_integration_node_helper(cript_api: cript.API, project_node: cript.Proje
     my_paginator = cript_api.search(node_type=cript.Project, search_mode=cript.SearchModes.EXACT_NAME, value_to_search=project_node.name)
 
     # get the project from paginator
-    my_project_from_api_dict = my_paginator.current_page_results[0]
+    my_project_from_api_node = my_paginator.next()
 
     print("\n\n================= API Response Node ============================")
-    print(json.dumps(my_project_from_api_dict, sort_keys=False, indent=2))
+    print(json.dumps(my_project_from_api_node.json, sort_keys=False, indent=2))
     print("==============================================================")
 
     # Configure keys and blocks to be ignored by deepdiff using exclude_regex_path
@@ -82,7 +82,7 @@ def save_integration_node_helper(cript_api: cript.API, project_node: cript.Proje
         r"root(\[.*\])?\['model_version'\]",
     ]
     # Compare the JSONs
-    diff = DeepDiff(json.loads(project_node.json), my_project_from_api_dict, exclude_regex_paths=exclude_regex_paths)
+    diff = DeepDiff(json.loads(project_node.json), json.loads(my_project_from_api_node.json), exclude_regex_paths=exclude_regex_paths)
     # with open("la", "a") as file_handle:
     #     file_handle.write(str(diff) + "\n")
 
@@ -92,9 +92,8 @@ def save_integration_node_helper(cript_api: cript.API, project_node: cript.Proje
     # assert not list(diff.get("dictionary_item_added", []))
 
     # try to convert api JSON project to node
-    my_project_from_api = cript.load_nodes_from_json(json.dumps(my_project_from_api_dict))
     print("\n\n=================== Project Node Deserialized =========================")
-    print(my_project_from_api.get_json(sort_keys=False, condense_to_uuid={}, indent=2).json)
+    print(my_project_from_api_node.get_json(sort_keys=False, condense_to_uuid={}, indent=2).json)
     print("==============================================================")
     print("\n\n\n######################################## TEST Passed ########################################\n\n\n")
 

From dcf33df6118598a9f37d05b361ab047748a423b8 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Tue, 20 Feb 2024 19:07:17 -0600
Subject: [PATCH 07/29] trunk catches

---
 src/cript/api/paginator.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 0e15ea285..a205a411c 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -1,5 +1,5 @@
 from json import JSONDecodeError
-from typing import Dict, List, Optional, Union
+from typing import Dict, Optional, Union
 from urllib.parse import quote
 
 import requests
@@ -37,8 +37,8 @@ class Paginator:
     # by default the page_number and query are `None` and they can get filled in
     _query: Union[str, None]
     _current_page_number: int
-    _curren_position: int
-    _fetched_nodes: list["BaseNode"]
+    _current_position: int
+    _fetched_nodes: list
 
     @beartype
     def __init__(

From 100cb1b6a566ed359ee74682380bbc18e2aa9425 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Tue, 20 Feb 2024 19:07:54 -0600
Subject: [PATCH 08/29] remove wrong file

---
 src/cript/nodes/subobjects/#*scratch*# | 19 -------------------
 src/cript/nodes/subobjects/*scratch*   | 17 -----------------
 2 files changed, 36 deletions(-)
 delete mode 100644 src/cript/nodes/subobjects/#*scratch*#
 delete mode 100644 src/cript/nodes/subobjects/*scratch*

diff --git a/src/cript/nodes/subobjects/#*scratch*# b/src/cript/nodes/subobjects/#*scratch*#
deleted file mode 100644
index cadec5f2a..000000000
--- a/src/cript/nodes/subobjects/#*scratch*#
+++ /dev/null
@@ -1,19 +0,0 @@
-;; This buffer is for text that is not saved, and for Lisp evaluation.
-;; To create a file, visit it with C-x C-f and enter text in its buffer.
-
-
-
-my_material = cript.Material()
-
-assert my_material.updated_at == None
-
-api.save(Material)
-
-assert my_material.updated_at == None
-
-downloaded_material = api.get(uuid=my_material.uuid)
-
-assert my_material.updated_at != None
-assert my_material.updated_at == "asdfcience"
-
-assert downloaded_material is my_material
diff --git a/src/cript/nodes/subobjects/*scratch* b/src/cript/nodes/subobjects/*scratch*
deleted file mode 100644
index aaed34389..000000000
--- a/src/cript/nodes/subobjects/*scratch*
+++ /dev/null
@@ -1,17 +0,0 @@
-;; This buffer is for text that is not saved, and for Lisp evaluation.
-;; To create a file, visit it with C-x C-f and enter text in its buffer.
-
-
-
-my_material = cript.Material()
-
-assert my_material.updated_at == None
-
-api.save(Material)
-
-assert my_material.updated_at == None
-
-downloaded_material = api.get(uuid=my_material.uuid)
-
-assert my_material.updated_at != None
-assert my_material.updated_at == "asdfcience"

From abc0e482e7c2a3ad6af0486ea0eb65b291fb722a Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Wed, 21 Feb 2024 15:16:56 -0600
Subject: [PATCH 09/29] small improvements

---
 src/cript/api/api.py       |   3 +-
 src/cript/api/paginator.py |  14 +++--
 tests/api/test_api.py      | 124 -------------------------------------
 tests/api/test_search.py   | 105 +++++++++++++++++++++++++++++++
 4 files changed, 114 insertions(+), 132 deletions(-)
 create mode 100644 tests/api/test_search.py

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 86cc94f5f..335fa58e1 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -815,9 +815,8 @@ def search(
 
         api_endpoint: str = ""
 
-        # requesting a page of some primary node
         if search_mode == SearchModes.NODE_TYPE:
-            api_endpoint = f"{self._host}/{node_type}"
+            api_endpoint = f"{self._host}/search/{node_type}"
 
         elif search_mode == SearchModes.CONTAINS_NAME:
             api_endpoint = f"{self._host}/search/{node_type}"
diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index a205a411c..cda02c779 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -71,24 +71,25 @@ def __init__(
             instantiate a paginator
         """
         self._http_headers = http_headers
-        self._api_endpoint = api_endpoint
-        self._query = query
+        self._api_endpoint = None
+        self._query = ""
         self._current_page_number = 0
         self._fetched_nodes = []
         self._current_position = 0
 
         # check if it is a string and not None to avoid AttributeError
         try:
-            self._api_endpoint = self._api_endpoint.rstrip("/").strip()
+            self._api_endpoint = api_endpoint.rstrip("/").strip()
         except AttributeError as exc:
             if self._api_endpoint is not None:
                 raise RuntimeError(f"Invalid type for api_endpoint {self._api_endpoint} for a paginator.") from exc
 
         # check if it is a string and not None to avoid AttributeError
         try:
-            self._query = quote(self._query)
-        except AttributeError as exc:
-            if self._query is not None:
+            self._query = quote(query)
+        except TypeError as exc:
+            self._query = ""
+            if query is not None:
                 raise RuntimeError(f"Invalid type for query {self._query} a paginator.") from exc
 
     @beartype
@@ -144,6 +145,7 @@ def _fetch_next_page(self) -> None:
 
         if api_response["code"] == 404 and api_response["error"] == "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.":
             current_page_results = []
+        import json
 
         # if API response is not 200 raise error for the user to debug
         if api_response["code"] != 200:
diff --git a/tests/api/test_api.py b/tests/api/test_api.py
index bfeadf5d6..eb1e64f31 100644
--- a/tests/api/test_api.py
+++ b/tests/api/test_api.py
@@ -11,7 +11,6 @@
 
 import cript
 from conftest import HAS_INTEGRATION_TESTS_ENABLED
-from cript.api.paginator import Paginator
 
 
 def test_api_with_invalid_host() -> None:
@@ -148,126 +147,3 @@ def test_upload_and_download_local_file(cript_api, tmp_path_factory) -> None:
 
     # assert download file contents are the same as uploaded file contents
     assert downloaded_file_contents == file_text
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_node_type(cript_api: cript.API) -> None:
-    """
-    tests the api.search() method with just a node type material search
-
-    just testing that something comes back from the server
-
-    Notes
-    -----
-    * also tests that it can go to the next page and previous page
-    * later this test should be expanded to test things that it should expect an error for as well.
-    * test checks if there are at least 5 things in the paginator
-        *  each page should have a max of 10 results and there should be close to 5k materials in db,
-        * more than enough to at least have 5 in the paginator
-    """
-    materials_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.NODE_TYPE, value_to_search=None)
-
-    # test search results
-    assert isinstance(materials_paginator, Paginator)
-    materials_list = list(materials_paginator)
-    # Assure that we paginated more then one page
-    assert materials_paginator._current_page_number > 0
-    assert len(materials_list) > 5
-    first_page_first_result = materials_list[0]["name"]
-    # just checking that the word has a few characters in it
-    assert len(first_page_first_result) > 3
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_contains_name(cript_api: cript.API) -> None:
-    """
-    tests that it can correctly search with contains name mode
-    searches for a material that contains the name "poly"
-    """
-    contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
-
-    assert isinstance(contains_name_paginator, Paginator)
-    contains_name_list = list(contains_name_paginator)
-    # Assure that we paginated more then one page
-    assert contains_name_paginator._current_page_number > 0
-    assert len(contains_name_list) > 5
-
-    contains_name_first_result = contains_name_list["name"]
-
-    # just checking that the result has a few characters in it
-    assert len(contains_name_first_result) > 3
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_exact_name(cript_api: cript.API) -> None:
-    """
-    tests search method with exact name search
-    searches for material "Sodium polystyrene sulfonate"
-    """
-    exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
-
-    assert isinstance(exact_name_paginator, Paginator)
-    exact_name_list = list(exact_name_paginator)
-    assert len(exact_name_list) == 1
-    assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
-    """
-    tests search with UUID
-    searches for `Sodium polystyrene sulfonate` material via UUID
-
-    The test is made dynamic to work with any server environment
-    1. gets the material via `exact name search` and gets the full node
-    2. takes the UUID from the full node and puts it into the `UUID search`
-    3. asserts everything is as expected
-    """
-    uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
-
-    assert isinstance(uuid_paginator, Paginator)
-    uuid_list = list(uuid_paginator)
-    assert len(uuid_list) == 1
-    assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
-    assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_bigsmiles(cript_api: cript.API) -> None:
-    """
-    tests search method with bigsmiles SearchMode to see if we just get at least one match
-    searches for material
-    "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
-
-    another good example can be "{[][$]CC(C)(C(=O)OCCCC)[$][]}"
-    """
-    bigsmiles_search_value = "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
-
-    bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
-
-    assert isinstance(bigsmiles_paginator, Paginator)
-    bigsmiles_list = list(bigsmiles_paginator)
-    assert len(bigsmiles_list) >= 1
-    # not sure if this will always be in this position in every server environment, so commenting it out for now
-    # assert bigsmiles_paginator.current_page_results[1]["name"] == "BCDB_Material_285"
-
-
-def test_get_my_user_node_from_api(cript_api: cript.API) -> None:
-    """
-    tests that the Python SDK can successfully get the user node associated with the API Token
-    """
-    pass
-
-
-def test_get_my_group_node_from_api(cript_api: cript.API) -> None:
-    """
-    tests that group node that is associated with their API Token can be gotten correctly
-    """
-    pass
-
-
-def test_get_my_projects_from_api(cript_api: cript.API) -> None:
-    """
-    get a page of project nodes that is associated with the API token
-    """
-    pass
diff --git a/tests/api/test_search.py b/tests/api/test_search.py
new file mode 100644
index 000000000..d8f0a0703
--- /dev/null
+++ b/tests/api/test_search.py
@@ -0,0 +1,105 @@
+import pytest
+
+import cript
+from conftest import HAS_INTEGRATION_TESTS_ENABLED
+from cript.api.paginator import Paginator
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_node_type(cript_api: cript.API) -> None:
+    """
+    tests the api.search() method with just a node type material search
+
+    just testing that something comes back from the server
+
+    Notes
+    -----
+    * also tests that it can go to the next page and previous page
+    * later this test should be expanded to test things that it should expect an error for as well.
+    * test checks if there are at least 5 things in the paginator
+        *  each page should have a max of 10 results and there should be close to 5k materials in db,
+        * more than enough to at least have 5 in the paginator
+    """
+    materials_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.NODE_TYPE, value_to_search=None)
+
+    # test search results
+    assert isinstance(materials_paginator, Paginator)
+    materials_list = list(materials_paginator)
+    # Assure that we paginated more then one page
+    assert materials_paginator._current_page_number > 0
+    assert len(materials_list) > 5
+    first_page_first_result = materials_list[0]["name"]
+    # just checking that the word has a few characters in it
+    assert len(first_page_first_result) > 3
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_contains_name(cript_api: cript.API) -> None:
+    """
+    tests that it can correctly search with contains name mode
+    searches for a material that contains the name "poly"
+    """
+    contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
+
+    assert isinstance(contains_name_paginator, Paginator)
+    contains_name_list = list(contains_name_paginator)
+    # Assure that we paginated more then one page
+    assert contains_name_paginator._current_page_number > 0
+    assert len(contains_name_list) > 5
+
+    contains_name_first_result = contains_name_list["name"]
+
+    # just checking that the result has a few characters in it
+    assert len(contains_name_first_result) > 3
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_exact_name(cript_api: cript.API) -> None:
+    """
+    tests search method with exact name search
+    searches for material "Sodium polystyrene sulfonate"
+    """
+    exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
+
+    assert isinstance(exact_name_paginator, Paginator)
+    exact_name_list = list(exact_name_paginator)
+    assert len(exact_name_list) == 1
+    assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
+    """
+    tests search with UUID
+    searches for `Sodium polystyrene sulfonate` material via UUID
+
+    The test is made dynamic to work with any server environment
+    1. gets the material via `exact name search` and gets the full node
+    2. takes the UUID from the full node and puts it into the `UUID search`
+    3. asserts everything is as expected
+    """
+    uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
+
+    assert isinstance(uuid_paginator, Paginator)
+    uuid_list = list(uuid_paginator)
+    assert len(uuid_list) == 1
+    assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
+    assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_bigsmiles(cript_api: cript.API) -> None:
+    """
+    tests search method with bigsmiles SearchMode to see if we just get at least one match
+    searches for material
+    "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
+
+    another good example can be "{[][$]CC(C)(C(=O)OCCCC)[$][]}"
+    """
+    bigsmiles_search_value = "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
+
+    bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
+
+    assert isinstance(bigsmiles_paginator, Paginator)
+    bigsmiles_list = list(bigsmiles_paginator)
+    assert len(bigsmiles_list) >= 1

From 95cff3e708839be81c081bebdf5cca114e9886f6 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Wed, 21 Feb 2024 16:47:15 -0600
Subject: [PATCH 10/29] make mypy happy

---
 src/cript/api/api.py       | 7 +++----
 src/cript/api/paginator.py | 6 ++----
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 335fa58e1..0c89b7dfc 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -733,7 +733,7 @@ def search(
         self,
         node_type: Any,
         search_mode: SearchModes,
-        value_to_search: Optional[str],
+        value_to_search: str = "",
     ) -> Paginator:
         """
         This method is used to perform search on the CRIPT platform.
@@ -748,7 +748,6 @@ def search(
             materials_iterator = cript_api.search(
                 node_type=cript.Material,
                 search_mode=cript.SearchModes.NODE_TYPE,
-                value_to_search=None
             )
             ```
 
@@ -795,7 +794,7 @@ def search(
         search_mode : SearchModes
             Type of search you want to do. You can search by name, `UUID`, `EXACT_NAME`, etc.
             Refer to [valid search modes](../search_modes)
-        value_to_search : Optional[str]
+        value_to_search : str
             What you are searching for can be either a value, and if you are only searching for
             a `NODE_TYPE`, then this value can be empty or `None`
 
@@ -827,7 +826,7 @@ def search(
         elif search_mode == SearchModes.UUID:
             api_endpoint = f"{self._host}/{node_type}/{value_to_search}"
             # putting the value_to_search in the URL instead of a query
-            value_to_search = None
+            value_to_search = ""
 
         elif search_mode == SearchModes.BIGSMILES:
             api_endpoint = f"{self._host}/search/bigsmiles/"
diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index cda02c779..7c1b535fb 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -35,7 +35,7 @@ class Paginator:
     # if query or page number are None, then it means that api_endpoint does not allow for whatever that is None
     # and that is not added to the URL
     # by default the page_number and query are `None` and they can get filled in
-    _query: Union[str, None]
+    _query: str
     _current_page_number: int
     _current_position: int
     _fetched_nodes: list
@@ -45,7 +45,7 @@ def __init__(
         self,
         http_headers: dict,
         api_endpoint: str,
-        query: Optional[str] = None,
+        query: str = "",
     ):
         """
         create a paginator
@@ -71,8 +71,6 @@ def __init__(
             instantiate a paginator
         """
         self._http_headers = http_headers
-        self._api_endpoint = None
-        self._query = ""
         self._current_page_number = 0
         self._fetched_nodes = []
         self._current_position = 0

From 196025be6ca4f29b9fff00fe86c0a52c2eb8036e Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 11:12:07 -0600
Subject: [PATCH 11/29] adjust expected result for doctest

---
 src/cript/api/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 5ffd81e68..90d8b5303 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -232,7 +232,7 @@ def __str__(self) -> str:
         ...     storage_token=os.getenv("CRIPT_STORAGE_TOKEN")
         ... ) as api:
         ...     print(api)
-        CRIPT API Client - Host URL: 'https://api.criptapp.org/api/v1'
+        CRIPT API Client - Host URL: 'https://api.criptapp.org/'
 
         Returns
         -------

From 78b1839e5502ec8c4f1007d2e873835606e29747 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 11:19:47 -0600
Subject: [PATCH 12/29] further smaller fixes

---
 src/cript/api/api.py         | 12 ++++++++++--
 src/cript/nodes/uuid_base.py |  2 +-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 90d8b5303..4ecd5140f 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -367,10 +367,18 @@ def host(self):
         ...     storage_token=os.getenv("CRIPT_STORAGE_TOKEN")
         ... ) as api:
         ...    print(api.host)
-        https://api.criptapp.org/api/v1
+        https://api.criptapp.org/
         """
         return self._host
 
+    @property
+    def api_prefix(self):
+        return self._api_prefix
+
+    @property
+    def api_version(self):
+        return self._api_version
+
     def save(self, project: Project) -> None:
         """
         This method takes a project node, serializes the class into JSON
@@ -978,7 +986,7 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True,
 
         url: str = self.host
         if api_request:
-            url += f"/{self._api_prefix}/{self._api_version}"
+            url += f"/{self.api_prefix}/{self.api_version}"
         url += url_path
 
         pre_log_message: str = f"Requesting {method} from {url}"
diff --git a/src/cript/nodes/uuid_base.py b/src/cript/nodes/uuid_base.py
index 899b278ed..1a3aa6e33 100644
--- a/src/cript/nodes/uuid_base.py
+++ b/src/cript/nodes/uuid_base.py
@@ -52,7 +52,7 @@ def url(self):
         from cript.api.api import _get_global_cached_api
 
         api = _get_global_cached_api()
-        return f"{api.host}/{self.uuid}"
+        return f"{api.host}/{api.api_prefix}/{api.api_version}/{self.uuid}"
 
     def __deepcopy__(self, memo):
         node = super().__deepcopy__(memo)

From f32e06ab08478e56e16e0b3e4537976d54afee1d Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 11:22:37 -0600
Subject: [PATCH 13/29] fix vocab test

---
 tests/api/test_db_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/api/test_db_schema.py b/tests/api/test_db_schema.py
index 9703bb6cd..f665bca61 100644
--- a/tests/api/test_db_schema.py
+++ b/tests/api/test_db_schema.py
@@ -127,7 +127,7 @@ def test_get_controlled_vocabulary_from_api(cript_api: cript.API) -> None:
     checks if it can successfully get the controlled vocabulary list from CRIPT API
     """
     number_of_vocab_categories = 26
-    vocab = cript_api.schema._get_vocab(cript_api.host)
+    vocab = cript_api.schema._get_vocab()
 
     # assertions
     # check vocabulary list is not empty

From d6d70cdf1a22ef46cd4ce949895e6ff3a82d1af8 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 11:31:48 -0600
Subject: [PATCH 14/29] automate cancelling outdated workflow runs

---
 .github/workflows/docs_check.yaml    | 7 +++++++
 .github/workflows/doctest.yaml       | 8 ++++++++
 .github/workflows/mypy.yaml          | 8 ++++++++
 .github/workflows/test_coverage.yaml | 8 ++++++++
 .github/workflows/test_examples.yml  | 8 ++++++++
 .github/workflows/tests.yml          | 8 ++++++++
 6 files changed, 47 insertions(+)

diff --git a/.github/workflows/docs_check.yaml b/.github/workflows/docs_check.yaml
index 649437cb0..a7eaae983 100644
--- a/.github/workflows/docs_check.yaml
+++ b/.github/workflows/docs_check.yaml
@@ -15,6 +15,13 @@ on:
       - main
       - develop
       - "*"
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
 
 jobs:
   build:
diff --git a/.github/workflows/doctest.yaml b/.github/workflows/doctest.yaml
index ef5d93dde..41091a3df 100644
--- a/.github/workflows/doctest.yaml
+++ b/.github/workflows/doctest.yaml
@@ -14,6 +14,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   doctest:
     strategy:
diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml
index 2df11ab3b..c5dbadeb4 100644
--- a/.github/workflows/mypy.yaml
+++ b/.github/workflows/mypy.yaml
@@ -14,6 +14,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   mypy-test:
     strategy:
diff --git a/.github/workflows/test_coverage.yaml b/.github/workflows/test_coverage.yaml
index 27b7d1af2..95654e7ba 100644
--- a/.github/workflows/test_coverage.yaml
+++ b/.github/workflows/test_coverage.yaml
@@ -15,6 +15,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   test-coverage:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test_examples.yml b/.github/workflows/test_examples.yml
index de9db832a..24ef37fc2 100644
--- a/.github/workflows/test_examples.yml
+++ b/.github/workflows/test_examples.yml
@@ -12,6 +12,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   test-examples:
     runs-on: ${{ matrix.os }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d7f433397..ba088676b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -18,6 +18,14 @@ on:
       - develop
       - "*"
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   install:
     runs-on: ${{ matrix.os }}

From 7f64970befc727fda1ce269f6d99467bbad9be67 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 11:32:31 -0600
Subject: [PATCH 15/29] fix doctest

---
 src/cript/api/api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 4ecd5140f..a0855cbe0 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -232,7 +232,7 @@ def __str__(self) -> str:
         ...     storage_token=os.getenv("CRIPT_STORAGE_TOKEN")
         ... ) as api:
         ...     print(api)
-        CRIPT API Client - Host URL: 'https://api.criptapp.org/'
+        CRIPT API Client - Host URL: 'https://api.criptapp.org'
 
         Returns
         -------
@@ -367,7 +367,7 @@ def host(self):
         ...     storage_token=os.getenv("CRIPT_STORAGE_TOKEN")
         ... ) as api:
         ...    print(api.host)
-        https://api.criptapp.org/
+        https://api.criptapp.org
         """
         return self._host
 

From 39ea5f78dc61e716d666db57e47619494c719898 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 12:01:43 -0600
Subject: [PATCH 16/29] fix test expectations

---
 tests/api/test_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/api/test_api.py b/tests/api/test_api.py
index e4323b7ee..38c51e9a9 100644
--- a/tests/api/test_api.py
+++ b/tests/api/test_api.py
@@ -56,7 +56,7 @@ def test_create_api_with_none() -> None:
 
     # assert SDK correctly got env vars to create cript.API with
     # host/api/v1
-    assert api._host == f"{env_var_host}/api/v1"
+    assert api._host == f"{env_var_host}"
     assert api._api_token == os.environ["CRIPT_TOKEN"]
     assert api._storage_token == os.environ["CRIPT_STORAGE_TOKEN"]
 
@@ -80,7 +80,7 @@ def test_config_file() -> None:
 
         api = cript.API(config_file_path=config_file_path)
 
-        assert api._host == config_file_texts["host"] + "/api/v1"
+        assert api._host == config_file_texts["host"]
         assert api._api_token == config_file_texts["api_token"]
 
 

From f51a1c5c7914d863d4d807cf70cb9644a9509e3d Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 13:54:10 -0600
Subject: [PATCH 17/29] remove tests that don't really make sense. I don't
 think we should process user supplied URLs too much.

---
 tests/api/test_api.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/tests/api/test_api.py b/tests/api/test_api.py
index e4323b7ee..6d14d191f 100644
--- a/tests/api/test_api.py
+++ b/tests/api/test_api.py
@@ -14,19 +14,6 @@
 from cript.api.paginator import Paginator
 
 
-def test_api_with_invalid_host() -> None:
-    """
-    this mostly tests the _prepare_host() function to be sure it is working as expected
-    * attempting to create an api client with invalid host appropriately throws a `CRIPTConnectionError`
-    * giving a host that does not start with http such as "criptapp.org" should throw an InvalidHostError
-    """
-    with pytest.raises((requests.ConnectionError, cript.api.exceptions.CRIPTConnectionError)):
-        cript.API(host="https://some_invalid_host", api_token="123456789", storage_token="123456")
-
-    with pytest.raises(cript.api.exceptions.InvalidHostError):
-        cript.API(host="no_http_host.org", api_token="123456789", storage_token="987654321")
-
-
 @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="skipping because API client needs API token")
 def test_api_context(cript_api: cript.API) -> None:
     assert cript.api.api._global_cached_api is not None

From 80e7bb90f4308bac84c165917df9ad381aee38f1 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Fri, 23 Feb 2024 14:01:58 -0600
Subject: [PATCH 18/29] fix trunk check

---
 src/cript/api/paginator.py   |   3 +-
 src/cript/nodes/util/json.py | 106 +++++++++++++++++------------------
 2 files changed, 54 insertions(+), 55 deletions(-)

diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 7c1b535fb..160817ba5 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -1,5 +1,5 @@
 from json import JSONDecodeError
-from typing import Dict, Optional, Union
+from typing import Dict
 from urllib.parse import quote
 
 import requests
@@ -143,7 +143,6 @@ def _fetch_next_page(self) -> None:
 
         if api_response["code"] == 404 and api_response["error"] == "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.":
             current_page_results = []
-        import json
 
         # if API response is not 200 raise error for the user to debug
         if api_response["code"] != 200:
diff --git a/src/cript/nodes/util/json.py b/src/cript/nodes/util/json.py
index 79dae1f17..2fc33393b 100644
--- a/src/cript/nodes/util/json.py
+++ b/src/cript/nodes/util/json.py
@@ -18,50 +18,50 @@
 
 class NodeEncoder(json.JSONEncoder):
     """
-    Custom JSON encoder for serializing CRIPT nodes to JSON.
+        Custom JSON encoder for serializing CRIPT nodes to JSON.
 
-    This encoder is used to convert CRIPT nodes into JSON format while handling unique identifiers (UUIDs) and
-    condensed representations to avoid redundancy in the JSON output.
-    It also allows suppressing specific attributes from being included in the serialized JSON.
+        This encoder is used to convert CRIPT nodes into JSON format while handling unique identifiers (UUIDs) and
+        condensed representations to avoid redundancy in the JSON output.
+        It also allows suppressing specific attributes from being included in the serialized JSON.
 
-    Attributes
-    ----------
-<<<<<<< HEAD
-    handled_ids : Set[str]
-        A set to store the UIDs of nodes that have been processed during serialization.
-    known_uuid : Set[str]
-        A set to store the UUIDs of nodes that have been previously encountered in the JSON.
-    condense_to_uuid : Dict[str, Set[str]]
-        A set to store the node types that should be condensed to UUID edges in the JSON.
-    suppress_attributes : Optional[Dict[str, Set[str]]]
-=======
-    handled_ids : set[str]
-        A set to store the UIDs of nodes that have been processed during serialization.
-    known_uuid : set[str]
-        A set to store the UUIDs of nodes that have been previously encountered in the JSON.
-    condense_to_uuid : dict[str, set[str]]
-        A set to store the node types that should be condensed to UUID edges in the JSON.
-    suppress_attributes : Optional[dict[str, set[str]]]
->>>>>>> 44cc898 (add missing file)
-        A dictionary that allows suppressing specific attributes for nodes with the corresponding UUIDs.
-
-    Methods
-    -------
-    ```python
-    default(self, obj: Any) -> Any:
-        # Convert CRIPT nodes and other objects to their JSON representation.
-    ```
+        Attributes
+        ----------
+    <<<<<<< HEAD
+        handled_ids : Set[str]
+            A set to store the UIDs of nodes that have been processed during serialization.
+        known_uuid : Set[str]
+            A set to store the UUIDs of nodes that have been previously encountered in the JSON.
+        condense_to_uuid : Dict[str, Set[str]]
+            A set to store the node types that should be condensed to UUID edges in the JSON.
+        suppress_attributes : Optional[Dict[str, Set[str]]]
+    =======
+        handled_ids : set[str]
+            A set to store the UIDs of nodes that have been processed during serialization.
+        known_uuid : set[str]
+            A set to store the UUIDs of nodes that have been previously encountered in the JSON.
+        condense_to_uuid : dict[str, set[str]]
+            A set to store the node types that should be condensed to UUID edges in the JSON.
+        suppress_attributes : Optional[dict[str, set[str]]]
+    >>>>>>> 44cc898 (add missing file)
+            A dictionary that allows suppressing specific attributes for nodes with the corresponding UUIDs.
+
+        Methods
+        -------
+        ```python
+        default(self, obj: Any) -> Any:
+            # Convert CRIPT nodes and other objects to their JSON representation.
+        ```
 
-    ```python
-<<<<<<< HEAD
-    _apply_modifications(self, serialize_dict: Dict) -> Tuple[Dict, List[str]]:
-=======
-    _apply_modifications(self, serialize_dict: dict) -> Tuple[dict, list[str]]:
->>>>>>> 44cc898 (add missing file)
-        # Apply modifications to the serialized dictionary based on node types
-        # and attributes to be condensed. This internal function handles node
-        # condensation and attribute suppression during serialization.
-    ```
+        ```python
+    <<<<<<< HEAD
+        _apply_modifications(self, serialize_dict: Dict) -> Tuple[Dict, List[str]]:
+    =======
+        _apply_modifications(self, serialize_dict: dict) -> Tuple[dict, list[str]]:
+    >>>>>>> 44cc898 (add missing file)
+            # Apply modifications to the serialized dictionary based on node types
+            # and attributes to be condensed. This internal function handles node
+            # condensation and attribute suppression during serialization.
+        ```
     """
 
     handled_ids: Set[str] = set()
@@ -150,21 +150,21 @@ def default(self, obj):
 
     def _apply_modifications(self, serialize_dict: Dict):
         """
-        Checks the serialize_dict to see if any other operations are required before it
-        can be considered done. If other operations are required, then it passes it to the other operations
-        and at the end returns the fully finished dict.
+                Checks the serialize_dict to see if any other operations are required before it
+                can be considered done. If other operations are required, then it passes it to the other operations
+                and at the end returns the fully finished dict.
 
-        This function is essentially a big switch case that checks the node type
-        and determines what other operations are required for it.
+                This function is essentially a big switch case that checks the node type
+                and determines what other operations are required for it.
 
-        Parameters
-        ----------
-<<<<<<< HEAD
-        serialize_dict: Dict
+                Parameters
+                ----------
+        <<<<<<< HEAD
+                serialize_dict: Dict
 
-        Returns
-        -------
-        serialize_dict: Dict
+                Returns
+                -------
+                serialize_dict: Dict
         """
 
         def process_attribute(attribute):

From 162c1d30f3c50b10a9a3c346720a3f5660312cd1 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 08:12:52 -0600
Subject: [PATCH 19/29] make API and data schema smarter. Delay init of
 DBschema to raise API connection error at right place.

---
 conftest.py                  |   3 +-
 src/cript/api/api.py         |  13 +++-
 src/cript/api/data_schema.py |  61 +++++----------
 src/cript/api/paginator.py   |   2 +-
 tests/api/test_search.py     | 142 +++++++++++++++++------------------
 5 files changed, 105 insertions(+), 116 deletions(-)

diff --git a/conftest.py b/conftest.py
index d2a51c84e..9e160353b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -50,7 +50,7 @@ def cript_api():
     """
     storage_token = os.getenv("CRIPT_STORAGE_TOKEN")
 
-    with cript.API(host=None, api_token=None, storage_token=storage_token) as api:
+    with cript.API(host=None, api_token=None, storage_token=storage_token, default_log_level=logging.DEBUG) as api:
         # overriding AWS S3 cognito variables to be sure we do not upload test data to production storage
         # staging AWS S3 cognito storage variables
         api._IDENTITY_POOL_ID = "us-east-1:25043452-a922-43af-b8a6-7e938a9e55c1"
@@ -59,7 +59,6 @@ def cript_api():
         # using the tests folder name within our cloud storage
         api._BUCKET_DIRECTORY_NAME = "tests"
         api.extra_api_log_debug_info = True
-        api.logger.setLevel(logging.DEBUG)
 
         yield api
 
diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 3ce13e19f..61ec1f9d5 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -17,6 +17,7 @@
     APIError,
     CRIPTAPIRequiredError,
     CRIPTAPISaveError,
+    CRIPTConnectionError,
     CRIPTDuplicateNameError,
 )
 from cript.api.paginator import Paginator
@@ -217,7 +218,6 @@ def __init__(self, host: Union[str, None] = None, api_token: Union[str, None] =
 
         # set a logger instance to use for the class logs
         self._init_logger(default_log_level)
-        self._db_schema = DataSchema(self)
 
     def __str__(self) -> str:
         """
@@ -316,7 +316,18 @@ def connect(self):
         If this function is called manually, the `API.disconnect` function has to be called later.
 
         For manual connection: nested API object are discouraged.
+
+        Raises
+        -------
+        CRIPTConnectionError
+            raised when the host does not give the expected response
         """
+        # As a form to check our connection, we pull and establish the dataschema
+        try:
+            self._db_schema = DataSchema(self)
+        except APIError as exc:
+            raise CRIPTConnectionError(self.host, self._api_token) from exc
+
         # Store the last active global API (might be None)
         global _global_cached_api
         self._previous_global_cached_api = copy.copy(_global_cached_api)
diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py
index a529bb72c..afb28223f 100644
--- a/src/cript/api/data_schema.py
+++ b/src/cript/api/data_schema.py
@@ -36,8 +36,8 @@ def __init__(self, api):
         ...    data_schema = cript.api.DataSchema(api)
         """
         self._api = api
+        self._vocabulary = {}
         self._db_schema = self._get_db_schema()
-        self._vocabulary = self._get_vocab()
 
     def _get_db_schema(self) -> dict:
         """
@@ -62,7 +62,9 @@ def _get_db_schema(self) -> dict:
         response = self._api._capsule_request(url_path="/schema/", method="GET")
 
         # raise error if not HTTP 200
-        response.raise_for_status()
+        if response["code"] != 200:
+            raise APIError(api_error=str(response), http_method="GET", api_url="/schema")
+
         self._api.logger.info(f"Loading node validation schema from {self._api.host}/schema/ was successful.")
 
         # if no error, take the JSON from the API response
@@ -73,46 +75,19 @@ def _get_db_schema(self) -> dict:
 
         return db_schema
 
-    def _get_vocab(self) -> dict:
+    def _fetch_vocab_entry(self, category: VocabCategories):
         """
-        gets the entire CRIPT controlled vocabulary and stores it in _vocabulary
-
-        1. loops through all controlled vocabulary categories
-            1. if the category already exists in the controlled vocabulary then skip that category and continue
-            1. if the category does not exist in the `_vocabulary` dict,
-            then request it from the API and append it to the `_vocabulary` dict
-        1. at the end the `_vocabulary` should have all the controlled vocabulary and that will be returned
-
-           Examples
-           --------
-           The vocabulary looks like this
-           ```json
-           {'algorithm_key':
-                [
-                    {
-                    'description': "Velocity-Verlet integration algorithm. Parameters: 'integration_timestep'.",
-                    'name': 'velocity_verlet'
-                    },
-            }
-           ```
+        Fetches one the CRIPT controlled vocabulary and stores it in self._vocabulary
         """
 
-        vocabulary: dict = {}
-        # loop through all vocabulary categories and make a request to each vocabulary category
-        # and put them all inside of self._vocab with the keys being the vocab category name
-        for category in VocabCategories:
-            vocabulary_category_url: str = f"/cv/{category.value}/"
-
-            # if vocabulary category is not in cache, then get it from API and cache it
-            response: dict = self._api._capsule_request(url_path=vocabulary_category_url, method="GET").json()
-
-            if response["code"] != 200:
-                raise APIError(api_error=str(response), http_method="GET", api_url=vocabulary_category_url)
+        vocabulary_category_url: str = f"/cv/{category.value}/"
 
-            # add to cache
-            vocabulary[category.value] = response["data"]
-
-        return vocabulary
+        # if vocabulary category is not in cache, then get it from API and cache it
+        response: dict = self._api._capsule_request(url_path=vocabulary_category_url, method="GET").json()
+        if response["code"] != 200:
+            raise APIError(api_error=str(response), http_method="GET", api_url=vocabulary_category_url)
+        # add to cache
+        self._vocabulary[category.value] = response["data"]
 
     @beartype
     def get_vocab_by_category(self, category: VocabCategories) -> list:
@@ -128,7 +103,7 @@ def get_vocab_by_category(self, category: VocabCategories) -> list:
         ...     api_token=os.getenv("CRIPT_TOKEN"),
         ...     storage_token=os.getenv("CRIPT_STORAGE_TOKEN")
         ... ) as api:
-        ...     api.validation_schema.get_vocab_by_category(cript.VocabCategories.MATERIAL_IDENTIFIER_KEY)  # doctest: +SKIP
+        ...     api.schema.get_vocab_by_category(cript.VocabCategories.MATERIAL_IDENTIFIER_KEY)  # doctest: +SKIP
 
         Parameters
         ----------
@@ -140,7 +115,11 @@ def get_vocab_by_category(self, category: VocabCategories) -> list:
         List[dict]
             list of JSON containing the controlled vocabulary
         """
-        return self._vocabulary[category.value]
+        try:
+            return self._vocabulary[category.value]
+        except APIError:
+            self._fetch_vocab_entry(category)
+            return self._vocabulary[category.value]
 
     @beartype
     def _is_vocab_valid(self, vocab_category: VocabCategories, vocab_word: str) -> bool:
@@ -176,7 +155,7 @@ def _is_vocab_valid(self, vocab_category: VocabCategories, vocab_word: str) -> b
         #     return True
 
         # get just the category needed
-        controlled_vocabulary = self._vocabulary[vocab_category.value]
+        controlled_vocabulary = self.get_vocab_by_category(vocab_category)
 
         # TODO this can be faster with a dict of dicts that can do o(1) look up
         #  looping through an unsorted list is an O(n) look up which is slow
diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 160817ba5..62a8b0e92 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -116,7 +116,7 @@ def _fetch_next_page(self) -> None:
         # Composition of the query URL
         temp_api_endpoint: str = self._api_endpoint
         temp_api_endpoint += "/?q="
-        if self._query is not None:
+        if len(self._query) > 0:
             temp_api_endpoint += f"{self._query}"
         temp_api_endpoint += f"&page={self._current_page_number}"
 
diff --git a/tests/api/test_search.py b/tests/api/test_search.py
index d8f0a0703..79861925b 100644
--- a/tests/api/test_search.py
+++ b/tests/api/test_search.py
@@ -20,7 +20,7 @@ def test_api_search_node_type(cript_api: cript.API) -> None:
         *  each page should have a max of 10 results and there should be close to 5k materials in db,
         * more than enough to at least have 5 in the paginator
     """
-    materials_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.NODE_TYPE, value_to_search=None)
+    materials_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.NODE_TYPE)
 
     # test search results
     assert isinstance(materials_paginator, Paginator)
@@ -33,73 +33,73 @@ def test_api_search_node_type(cript_api: cript.API) -> None:
     assert len(first_page_first_result) > 3
 
 
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_contains_name(cript_api: cript.API) -> None:
-    """
-    tests that it can correctly search with contains name mode
-    searches for a material that contains the name "poly"
-    """
-    contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
-
-    assert isinstance(contains_name_paginator, Paginator)
-    contains_name_list = list(contains_name_paginator)
-    # Assure that we paginated more then one page
-    assert contains_name_paginator._current_page_number > 0
-    assert len(contains_name_list) > 5
-
-    contains_name_first_result = contains_name_list["name"]
-
-    # just checking that the result has a few characters in it
-    assert len(contains_name_first_result) > 3
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_exact_name(cript_api: cript.API) -> None:
-    """
-    tests search method with exact name search
-    searches for material "Sodium polystyrene sulfonate"
-    """
-    exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
-
-    assert isinstance(exact_name_paginator, Paginator)
-    exact_name_list = list(exact_name_paginator)
-    assert len(exact_name_list) == 1
-    assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
-    """
-    tests search with UUID
-    searches for `Sodium polystyrene sulfonate` material via UUID
-
-    The test is made dynamic to work with any server environment
-    1. gets the material via `exact name search` and gets the full node
-    2. takes the UUID from the full node and puts it into the `UUID search`
-    3. asserts everything is as expected
-    """
-    uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
-
-    assert isinstance(uuid_paginator, Paginator)
-    uuid_list = list(uuid_paginator)
-    assert len(uuid_list) == 1
-    assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
-    assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
-
-
-@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-def test_api_search_bigsmiles(cript_api: cript.API) -> None:
-    """
-    tests search method with bigsmiles SearchMode to see if we just get at least one match
-    searches for material
-    "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
-
-    another good example can be "{[][$]CC(C)(C(=O)OCCCC)[$][]}"
-    """
-    bigsmiles_search_value = "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
-
-    bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
-
-    assert isinstance(bigsmiles_paginator, Paginator)
-    bigsmiles_list = list(bigsmiles_paginator)
-    assert len(bigsmiles_list) >= 1
+# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+# def test_api_search_contains_name(cript_api: cript.API) -> None:
+#     """
+#     tests that it can correctly search with contains name mode
+#     searches for a material that contains the name "poly"
+#     """
+#     contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
+
+#     assert isinstance(contains_name_paginator, Paginator)
+#     contains_name_list = list(contains_name_paginator)
+#     # Assure that we paginated more then one page
+#     assert contains_name_paginator._current_page_number > 0
+#     assert len(contains_name_list) > 5
+
+#     contains_name_first_result = contains_name_list["name"]
+
+#     # just checking that the result has a few characters in it
+#     assert len(contains_name_first_result) > 3
+
+
+# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+# def test_api_search_exact_name(cript_api: cript.API) -> None:
+#     """
+#     tests search method with exact name search
+#     searches for material "Sodium polystyrene sulfonate"
+#     """
+#     exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
+
+#     assert isinstance(exact_name_paginator, Paginator)
+#     exact_name_list = list(exact_name_paginator)
+#     assert len(exact_name_list) == 1
+#     assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
+
+
+# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+# def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
+#     """
+#     tests search with UUID
+#     searches for `Sodium polystyrene sulfonate` material via UUID
+
+#     The test is made dynamic to work with any server environment
+#     1. gets the material via `exact name search` and gets the full node
+#     2. takes the UUID from the full node and puts it into the `UUID search`
+#     3. asserts everything is as expected
+#     """
+#     uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
+
+#     assert isinstance(uuid_paginator, Paginator)
+#     uuid_list = list(uuid_paginator)
+#     assert len(uuid_list) == 1
+#     assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
+#     assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
+
+
+# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+# def test_api_search_bigsmiles(cript_api: cript.API) -> None:
+#     """
+#     tests search method with bigsmiles SearchMode to see if we just get at least one match
+#     searches for material
+#     "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
+
+#     another good example can be "{[][$]CC(C)(C(=O)OCCCC)[$][]}"
+#     """
+#     bigsmiles_search_value = "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
+
+#     bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
+
+#     assert isinstance(bigsmiles_paginator, Paginator)
+#     bigsmiles_list = list(bigsmiles_paginator)
+#     assert len(bigsmiles_list) >= 1

From 5345649e84bb812b62c9fa35e13de8c0f542ec21 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 08:27:53 -0600
Subject: [PATCH 20/29] full circle and we still don't have the correct search
 results

---
 src/cript/api/api.py         | 14 ++++++------
 src/cript/api/data_schema.py |  7 ++----
 src/cript/api/paginator.py   | 43 ++++++++++++------------------------
 3 files changed, 23 insertions(+), 41 deletions(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 61ec1f9d5..dffc5ecd3 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -322,7 +322,7 @@ def connect(self):
         CRIPTConnectionError
             raised when the host does not give the expected response
         """
-        # As a form to check our connection, we pull and establish the dataschema
+        # As a form to check our connection, we pull and establish the data schema
         try:
             self._db_schema = DataSchema(self)
         except APIError as exc:
@@ -777,27 +777,27 @@ def search(
         api_endpoint: str = ""
 
         if search_mode == SearchModes.NODE_TYPE:
-            api_endpoint = f"{self._host}/search/{node_type}"
+            api_endpoint = f"/search/{node_type}"
 
         elif search_mode == SearchModes.CONTAINS_NAME:
-            api_endpoint = f"{self._host}/search/{node_type}"
+            api_endpoint = f"/search/{node_type}"
 
         elif search_mode == SearchModes.EXACT_NAME:
-            api_endpoint = f"{self._host}/search/exact/{node_type}"
+            api_endpoint = f"/search/exact/{node_type}"
 
         elif search_mode == SearchModes.UUID:
-            api_endpoint = f"{self._host}/{node_type}/{value_to_search}"
+            api_endpoint = f"/{node_type}/{value_to_search}"
             # putting the value_to_search in the URL instead of a query
             value_to_search = ""
 
         elif search_mode == SearchModes.BIGSMILES:
-            api_endpoint = f"{self._host}/search/bigsmiles/"
+            api_endpoint = "/search/bigsmiles/"
 
         # error handling if none of the API endpoints got hit
         else:
             raise RuntimeError("Internal Error: Failed to recognize any search modes. Please report this bug on https://github.com/C-Accel-CRIPT/Python-SDK/issues.")
 
-        return Paginator(http_headers=self._http_headers, api_endpoint=api_endpoint, query=value_to_search)
+        return Paginator(api=self, url_path=api_endpoint, query=value_to_search)
 
     def delete(self, node) -> None:
         """
diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py
index afb28223f..65c25e053 100644
--- a/src/cript/api/data_schema.py
+++ b/src/cript/api/data_schema.py
@@ -59,7 +59,7 @@ def _get_db_schema(self) -> dict:
         # fetch db_schema from API
         self._api.logger.info(f"Loading node validation schema from {self._api.host}/schema/")
         # fetch db schema from API
-        response = self._api._capsule_request(url_path="/schema/", method="GET")
+        response: dict = self._api._capsule_request(url_path="/schema/", method="GET").json()
 
         # raise error if not HTTP 200
         if response["code"] != 200:
@@ -67,11 +67,8 @@ def _get_db_schema(self) -> dict:
 
         self._api.logger.info(f"Loading node validation schema from {self._api.host}/schema/ was successful.")
 
-        # if no error, take the JSON from the API response
-        response_dict: dict = response.json()
-
         # get the data from the API JSON response
-        db_schema = response_dict["data"]
+        db_schema = response["data"]
 
         return db_schema
 
diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 62a8b0e92..0b943d17f 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -5,7 +5,6 @@
 import requests
 from beartype import beartype
 
-from cript.api.api_config import _API_TIMEOUT
 from cript.api.exceptions import APIError
 from cript.nodes.util import load_nodes_from_json
 
@@ -28,13 +27,7 @@ class Paginator:
 
     """
 
-    _http_headers: dict
-
-    _api_endpoint: str
-
-    # if query or page number are None, then it means that api_endpoint does not allow for whatever that is None
-    # and that is not added to the URL
-    # by default the page_number and query are `None` and they can get filled in
+    _url_path: str
     _query: str
     _current_page_number: int
     _current_position: int
@@ -43,8 +36,8 @@ class Paginator:
     @beartype
     def __init__(
         self,
-        http_headers: dict,
-        api_endpoint: str,
+        api,
+        url_path: str = "",
         query: str = "",
     ):
         """
@@ -70,25 +63,18 @@ def __init__(
         None
             instantiate a paginator
         """
-        self._http_headers = http_headers
+        self._api = api
         self._current_page_number = 0
         self._fetched_nodes = []
         self._current_position = 0
 
         # check if it is a string and not None to avoid AttributeError
         try:
-            self._api_endpoint = api_endpoint.rstrip("/").strip()
-        except AttributeError as exc:
-            if self._api_endpoint is not None:
-                raise RuntimeError(f"Invalid type for api_endpoint {self._api_endpoint} for a paginator.") from exc
+            self._url_path = quote(url_path.rstrip("/").strip())
+        except Exception as exc:
+            raise RuntimeError(f"Invalid type for api_endpoint {self._url_path} for a paginator.") from exc
 
-        # check if it is a string and not None to avoid AttributeError
-        try:
-            self._query = quote(query)
-        except TypeError as exc:
-            self._query = ""
-            if query is not None:
-                raise RuntimeError(f"Invalid type for query {self._query} a paginator.") from exc
+        self._query = quote(query)
 
     @beartype
     def _fetch_next_page(self) -> None:
@@ -114,13 +100,11 @@ def _fetch_next_page(self) -> None:
         """
 
         # Composition of the query URL
-        temp_api_endpoint: str = self._api_endpoint
-        temp_api_endpoint += "/?q="
-        if len(self._query) > 0:
-            temp_api_endpoint += f"{self._query}"
-        temp_api_endpoint += f"&page={self._current_page_number}"
+        temp_url_path: str = self._url_path
+        temp_url_path += f"/?q={self._query}"
+        temp_url_path += f"&page={self._current_page_number}"
 
-        response: requests.Response = requests.get(url=temp_api_endpoint, headers=self._http_headers, timeout=_API_TIMEOUT)
+        response: requests.Response = self._api._capsule_request(url_path=temp_url_path, method="GET")
 
         # it is expected that the response will be JSON
         # try to convert response to JSON
@@ -146,7 +130,8 @@ def _fetch_next_page(self) -> None:
 
         # if API response is not 200 raise error for the user to debug
         if api_response["code"] != 200:
-            raise APIError(api_error=str(response.json()), http_method="GET", api_url=temp_api_endpoint)
+            raise APIError(api_error=str(response.json()), http_method="GET", api_url=temp_url_path)
+
         if len(current_page_results) == 0:
             raise StopIteration
 

From 27da74df073db9511f7037be2bc14c2d8b293595 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 08:30:51 -0600
Subject: [PATCH 21/29] reenable tests

---
 tests/api/test_search.py | 140 +++++++++++++++++++--------------------
 1 file changed, 70 insertions(+), 70 deletions(-)

diff --git a/tests/api/test_search.py b/tests/api/test_search.py
index 79861925b..d180365a5 100644
--- a/tests/api/test_search.py
+++ b/tests/api/test_search.py
@@ -33,73 +33,73 @@ def test_api_search_node_type(cript_api: cript.API) -> None:
     assert len(first_page_first_result) > 3
 
 
-# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-# def test_api_search_contains_name(cript_api: cript.API) -> None:
-#     """
-#     tests that it can correctly search with contains name mode
-#     searches for a material that contains the name "poly"
-#     """
-#     contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
-
-#     assert isinstance(contains_name_paginator, Paginator)
-#     contains_name_list = list(contains_name_paginator)
-#     # Assure that we paginated more then one page
-#     assert contains_name_paginator._current_page_number > 0
-#     assert len(contains_name_list) > 5
-
-#     contains_name_first_result = contains_name_list["name"]
-
-#     # just checking that the result has a few characters in it
-#     assert len(contains_name_first_result) > 3
-
-
-# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-# def test_api_search_exact_name(cript_api: cript.API) -> None:
-#     """
-#     tests search method with exact name search
-#     searches for material "Sodium polystyrene sulfonate"
-#     """
-#     exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
-
-#     assert isinstance(exact_name_paginator, Paginator)
-#     exact_name_list = list(exact_name_paginator)
-#     assert len(exact_name_list) == 1
-#     assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
-
-
-# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-# def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
-#     """
-#     tests search with UUID
-#     searches for `Sodium polystyrene sulfonate` material via UUID
-
-#     The test is made dynamic to work with any server environment
-#     1. gets the material via `exact name search` and gets the full node
-#     2. takes the UUID from the full node and puts it into the `UUID search`
-#     3. asserts everything is as expected
-#     """
-#     uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
-
-#     assert isinstance(uuid_paginator, Paginator)
-#     uuid_list = list(uuid_paginator)
-#     assert len(uuid_list) == 1
-#     assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
-#     assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
-
-
-# @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
-# def test_api_search_bigsmiles(cript_api: cript.API) -> None:
-#     """
-#     tests search method with bigsmiles SearchMode to see if we just get at least one match
-#     searches for material
-#     "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
-
-#     another good example can be "{[][$]CC(C)(C(=O)OCCCC)[$][]}"
-#     """
-#     bigsmiles_search_value = "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
-
-#     bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
-
-#     assert isinstance(bigsmiles_paginator, Paginator)
-#     bigsmiles_list = list(bigsmiles_paginator)
-#     assert len(bigsmiles_list) >= 1
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_contains_name(cript_api: cript.API) -> None:
+    """
+    tests that it can correctly search with contains name mode
+    searches for a material that contains the name "poly"
+    """
+    contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
+
+    assert isinstance(contains_name_paginator, Paginator)
+    contains_name_list = list(contains_name_paginator)
+    # Assure that we paginated more then one page
+    assert contains_name_paginator._current_page_number > 0
+    assert len(contains_name_list) > 5
+
+    contains_name_first_result = contains_name_list["name"]
+
+    # just checking that the result has a few characters in it
+    assert len(contains_name_first_result) > 3
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_exact_name(cript_api: cript.API) -> None:
+    """
+    tests search method with exact name search
+    searches for material "Sodium polystyrene sulfonate"
+    """
+    exact_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.EXACT_NAME, value_to_search="Sodium polystyrene sulfonate")
+
+    assert isinstance(exact_name_paginator, Paginator)
+    exact_name_list = list(exact_name_paginator)
+    assert len(exact_name_list) == 1
+    assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
+    """
+    tests search with UUID
+    searches for `Sodium polystyrene sulfonate` material via UUID
+
+    The test is made dynamic to work with any server environment
+    1. gets the material via `exact name search` and gets the full node
+    2. takes the UUID from the full node and puts it into the `UUID search`
+    3. asserts everything is as expected
+    """
+    uuid_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.UUID, value_to_search=dynamic_material_data["uuid"])
+
+    assert isinstance(uuid_paginator, Paginator)
+    uuid_list = list(uuid_paginator)
+    assert len(uuid_list) == 1
+    assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
+    assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
+
+
+@pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
+def test_api_search_bigsmiles(cript_api: cript.API) -> None:
+    """
+    tests search method with bigsmiles SearchMode to see if we just get at least one match
+    searches for material
+    "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
+
+    another good example can be "{[][$]CC(C)(C(=O)OCCCC)[$][]}"
+    """
+    bigsmiles_search_value = "{[][<]C(C)C(=O)O[>][<]}{[$][$]CCC(C)C[$],[$]CC(C(C)C)[$],[$]CC(C)(CC)[$][]}"
+
+    bigsmiles_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.BIGSMILES, value_to_search=bigsmiles_search_value)
+
+    assert isinstance(bigsmiles_paginator, Paginator)
+    bigsmiles_list = list(bigsmiles_paginator)
+    assert len(bigsmiles_list) >= 1

From 376fbdb2a6cef959cb39cb2c44d00d56e7001db3 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 08:35:19 -0600
Subject: [PATCH 22/29] stupid mistakes

---
 src/cript/api/data_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py
index 65c25e053..87c51c12d 100644
--- a/src/cript/api/data_schema.py
+++ b/src/cript/api/data_schema.py
@@ -114,7 +114,7 @@ def get_vocab_by_category(self, category: VocabCategories) -> list:
         """
         try:
             return self._vocabulary[category.value]
-        except APIError:
+        except KeyError:
             self._fetch_vocab_entry(category)
             return self._vocabulary[category.value]
 

From 9fe4b88d7aff55fa048d87f272cafc54b501ee25 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 10:35:24 -0600
Subject: [PATCH 23/29] remove unmerged paths

---
 src/cript/api/api_config.py  |  2 +-
 src/cript/nodes/util/json.py | 67 ++++++++++++++----------------------
 2 files changed, 27 insertions(+), 42 deletions(-)

diff --git a/src/cript/api/api_config.py b/src/cript/api/api_config.py
index bd7de1b6e..b2222e212 100644
--- a/src/cript/api/api_config.py
+++ b/src/cript/api/api_config.py
@@ -6,4 +6,4 @@
 """
 
 # Default maximum time in seconds for all API requests to wait for a response from the backend
-_API_TIMEOUT: int = 120
+_API_TIMEOUT: int = 6 * 150
diff --git a/src/cript/nodes/util/json.py b/src/cript/nodes/util/json.py
index 2fc33393b..1106f5e0d 100644
--- a/src/cript/nodes/util/json.py
+++ b/src/cript/nodes/util/json.py
@@ -18,50 +18,35 @@
 
 class NodeEncoder(json.JSONEncoder):
     """
-        Custom JSON encoder for serializing CRIPT nodes to JSON.
+    Custom JSON encoder for serializing CRIPT nodes to JSON.
 
-        This encoder is used to convert CRIPT nodes into JSON format while handling unique identifiers (UUIDs) and
-        condensed representations to avoid redundancy in the JSON output.
-        It also allows suppressing specific attributes from being included in the serialized JSON.
+    This encoder is used to convert CRIPT nodes into JSON format while handling unique identifiers (UUIDs) and
+    condensed representations to avoid redundancy in the JSON output.
+    It also allows suppressing specific attributes from being included in the serialized JSON.
 
-        Attributes
-        ----------
-    <<<<<<< HEAD
-        handled_ids : Set[str]
-            A set to store the UIDs of nodes that have been processed during serialization.
-        known_uuid : Set[str]
-            A set to store the UUIDs of nodes that have been previously encountered in the JSON.
-        condense_to_uuid : Dict[str, Set[str]]
-            A set to store the node types that should be condensed to UUID edges in the JSON.
-        suppress_attributes : Optional[Dict[str, Set[str]]]
-    =======
-        handled_ids : set[str]
-            A set to store the UIDs of nodes that have been processed during serialization.
-        known_uuid : set[str]
-            A set to store the UUIDs of nodes that have been previously encountered in the JSON.
-        condense_to_uuid : dict[str, set[str]]
-            A set to store the node types that should be condensed to UUID edges in the JSON.
-        suppress_attributes : Optional[dict[str, set[str]]]
-    >>>>>>> 44cc898 (add missing file)
-            A dictionary that allows suppressing specific attributes for nodes with the corresponding UUIDs.
-
-        Methods
-        -------
-        ```python
-        default(self, obj: Any) -> Any:
-            # Convert CRIPT nodes and other objects to their JSON representation.
-        ```
+    Attributes
+    ----------
+    handled_ids : set[str]
+        A set to store the UIDs of nodes that have been processed during serialization.
+    known_uuid : set[str]
+        A set to store the UUIDs of nodes that have been previously encountered in the JSON.
+    condense_to_uuid : dict[str, set[str]]
+        A set to store the node types that should be condensed to UUID edges in the JSON.
+    suppress_attributes : Optional[dict[str, set[str]]]
+
+    Methods
+    -------
+    ```python
+    default(self, obj: Any) -> Any:
+        # Convert CRIPT nodes and other objects to their JSON representation.
+    ```
 
-        ```python
-    <<<<<<< HEAD
-        _apply_modifications(self, serialize_dict: Dict) -> Tuple[Dict, List[str]]:
-    =======
-        _apply_modifications(self, serialize_dict: dict) -> Tuple[dict, list[str]]:
-    >>>>>>> 44cc898 (add missing file)
-            # Apply modifications to the serialized dictionary based on node types
-            # and attributes to be condensed. This internal function handles node
-            # condensation and attribute suppression during serialization.
-        ```
+    ```python
+    _apply_modifications(self, serialize_dict: dict) -> Tuple[dict, list[str]]:
+        # Apply modifications to the serialized dictionary based on node types
+        # and attributes to be condensed. This internal function handles node
+        # condensation and attribute suppression during serialization.
+    ```
     """
 
     handled_ids: Set[str] = set()

From 5519bf9f9d72bb12c27f9f82dd5ab5470af7aa6f Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 10:38:38 -0600
Subject: [PATCH 24/29] add missing page increase

---
 src/cript/api/paginator.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 0b943d17f..7650d4ccf 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -137,6 +137,7 @@ def _fetch_next_page(self) -> None:
 
         node_list = load_nodes_from_json(current_page_results)
         self._fetched_nodes += node_list
+        self._current_page_number += 1
 
     def __next__(self):
         if self._current_position >= len(self._fetched_nodes):

From 8753080189711bc22dc6092d68a73721037b2d02 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 11:50:55 -0600
Subject: [PATCH 25/29] add good changes

---
 src/cript/api/api.py       | 10 ++++++++--
 src/cript/api/paginator.py | 32 +++++++++++++++++++++-----------
 tests/api/test_search.py   | 15 +++++++--------
 3 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index dffc5ecd3..3081399f8 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -775,29 +775,35 @@ def search(
         node_type = node_type.node_type_snake_case
 
         api_endpoint: str = ""
+        page_number: Union[int, None] = None
 
         if search_mode == SearchModes.NODE_TYPE:
             api_endpoint = f"/search/{node_type}"
+            page_number = 0
 
         elif search_mode == SearchModes.CONTAINS_NAME:
             api_endpoint = f"/search/{node_type}"
+            page_number = 0
 
         elif search_mode == SearchModes.EXACT_NAME:
             api_endpoint = f"/search/exact/{node_type}"
+            page_number = None
 
         elif search_mode == SearchModes.UUID:
             api_endpoint = f"/{node_type}/{value_to_search}"
             # putting the value_to_search in the URL instead of a query
             value_to_search = ""
+            page_number = None
 
         elif search_mode == SearchModes.BIGSMILES:
             api_endpoint = "/search/bigsmiles/"
+            page_number = 0
 
         # error handling if none of the API endpoints got hit
         else:
             raise RuntimeError("Internal Error: Failed to recognize any search modes. Please report this bug on https://github.com/C-Accel-CRIPT/Python-SDK/issues.")
 
-        return Paginator(api=self, url_path=api_endpoint, query=value_to_search)
+        return Paginator(api=self, url_path=api_endpoint, page_number=page_number, query=value_to_search)
 
     def delete(self, node) -> None:
         """
@@ -982,7 +988,7 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True,
         response: requests.Response = requests.request(url=url, method=method, headers=headers, timeout=timeout, **kwargs)
         post_log_message: str = f"Request return with {response.status_code}"
         if self.extra_api_log_debug_info:
-            post_log_message += f" {response.json()}"
+            post_log_message += f" {response.text}"
         self.logger.debug(post_log_message)
 
         return response
diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 7650d4ccf..101d12ba0 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -1,5 +1,5 @@
 from json import JSONDecodeError
-from typing import Dict
+from typing import Dict, Optional, Union
 from urllib.parse import quote
 
 import requests
@@ -29,16 +29,18 @@ class Paginator:
 
     _url_path: str
     _query: str
-    _current_page_number: int
+    _initial_page_number: Union[int, None]
     _current_position: int
     _fetched_nodes: list
+    _number_fetched_pages: int = 0
 
     @beartype
     def __init__(
         self,
         api,
-        url_path: str = "",
-        query: str = "",
+        url_path: str,
+        page_number: Union[int, None],
+        query: str,
     ):
         """
         create a paginator
@@ -64,7 +66,8 @@ def __init__(
             instantiate a paginator
         """
         self._api = api
-        self._current_page_number = 0
+        self._initial_page_number = page_number
+        self._number_fetched_pages = 0
         self._fetched_nodes = []
         self._current_position = 0
 
@@ -102,7 +105,9 @@ def _fetch_next_page(self) -> None:
         # Composition of the query URL
         temp_url_path: str = self._url_path
         temp_url_path += f"/?q={self._query}"
-        temp_url_path += f"&page={self._current_page_number}"
+        if self._initial_page_number is not None:
+            temp_url_path += f"&page={self._initial_page_number + self._number_fetched_pages}"
+        self._number_fetched_pages += 1
 
         response: requests.Response = self._api._capsule_request(url_path=temp_url_path, method="GET")
 
@@ -132,18 +137,23 @@ def _fetch_next_page(self) -> None:
         if api_response["code"] != 200:
             raise APIError(api_error=str(response.json()), http_method="GET", api_url=temp_url_path)
 
-        if len(current_page_results) == 0:
-            raise StopIteration
-
         node_list = load_nodes_from_json(current_page_results)
         self._fetched_nodes += node_list
-        self._current_page_number += 1
 
     def __next__(self):
         if self._current_position >= len(self._fetched_nodes):
+            # Without a page number argument, we can only fetch once.
+            if self._initial_page_number is None and self._number_fetched_pages > 0:
+                raise StopIteration
             self._fetch_next_page()
+
         self._current_position += 1
-        return self._fetched_nodes[self._current_position - 1]
+        try:
+            return self._fetched_nodes[self._current_position - 1]
+        except IndexError:  # This is not a random access iteration.
+            # So if fetching a next page wasn't enough to get the index inbound,
+            # The iteration stops
+            raise StopIteration
 
     def __iter__(self):
         self._current_position = 0
diff --git a/tests/api/test_search.py b/tests/api/test_search.py
index d180365a5..860ab43ea 100644
--- a/tests/api/test_search.py
+++ b/tests/api/test_search.py
@@ -37,17 +37,16 @@ def test_api_search_node_type(cript_api: cript.API) -> None:
 def test_api_search_contains_name(cript_api: cript.API) -> None:
     """
     tests that it can correctly search with contains name mode
-    searches for a material that contains the name "poly"
+    searches for a material that contains the name "polystyrene"
     """
-    contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="poly")
+    contains_name_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.CONTAINS_NAME, value_to_search="polystyrene")
 
     assert isinstance(contains_name_paginator, Paginator)
     contains_name_list = list(contains_name_paginator)
     # Assure that we paginated more then one page
-    assert contains_name_paginator._current_page_number > 0
-    assert len(contains_name_list) > 5
+    assert len(contains_name_list) > 2
 
-    contains_name_first_result = contains_name_list["name"]
+    contains_name_first_result = contains_name_list[0].name
 
     # just checking that the result has a few characters in it
     assert len(contains_name_first_result) > 3
@@ -64,7 +63,7 @@ def test_api_search_exact_name(cript_api: cript.API) -> None:
     assert isinstance(exact_name_paginator, Paginator)
     exact_name_list = list(exact_name_paginator)
     assert len(exact_name_list) == 1
-    assert exact_name_paginator.current_page_results[0]["name"] == "Sodium polystyrene sulfonate"
+    assert exact_name_list[0].name == "Sodium polystyrene sulfonate"
 
 
 @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")
@@ -83,8 +82,8 @@ def test_api_search_uuid(cript_api: cript.API, dynamic_material_data) -> None:
     assert isinstance(uuid_paginator, Paginator)
     uuid_list = list(uuid_paginator)
     assert len(uuid_list) == 1
-    assert uuid_paginator.current_page_results[0]["name"] == dynamic_material_data["name"]
-    assert uuid_paginator.current_page_results[0]["uuid"] == dynamic_material_data["uuid"]
+    assert uuid_list[0].name == dynamic_material_data["name"]
+    assert str(uuid_list[0].uuid) == dynamic_material_data["uuid"]
 
 
 @pytest.mark.skipif(not HAS_INTEGRATION_TESTS_ENABLED, reason="requires a real cript_api_token")

From e8b699618692bd9f4a7aad8238a4a27e46ca5c1d Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 11:52:59 -0600
Subject: [PATCH 26/29] less error prone

---
 src/cript/api/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index a0855cbe0..727dfacf0 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -998,7 +998,7 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True,
         response: requests.Response = requests.request(url=url, method=method, headers=headers, timeout=timeout, **kwargs)
         post_log_message: str = f"Request return with {response.status_code}"
         if self.extra_api_log_debug_info:
-            post_log_message += f" {response.json()}"
+            post_log_message += f" {response.text}"
         self.logger.debug(post_log_message)
 
         return response

From 8b61a748321bbaac79b79ce6ec163d86aa373f4a Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 11:58:32 -0600
Subject: [PATCH 27/29] raw is better for debugging

---
 src/cript/api/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cript/api/api.py b/src/cript/api/api.py
index 727dfacf0..c9449356a 100644
--- a/src/cript/api/api.py
+++ b/src/cript/api/api.py
@@ -998,7 +998,7 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True,
         response: requests.Response = requests.request(url=url, method=method, headers=headers, timeout=timeout, **kwargs)
         post_log_message: str = f"Request return with {response.status_code}"
         if self.extra_api_log_debug_info:
-            post_log_message += f" {response.text}"
+            post_log_message += f" {response.raw}"
         self.logger.debug(post_log_message)
 
         return response

From 139a5c240b0346f5cd399bb66280559df933d0e7 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 13:11:46 -0600
Subject: [PATCH 28/29] fix import error

---
 src/cript/api/paginator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py
index 101d12ba0..929954727 100644
--- a/src/cript/api/paginator.py
+++ b/src/cript/api/paginator.py
@@ -1,5 +1,5 @@
 from json import JSONDecodeError
-from typing import Dict, Optional, Union
+from typing import Dict, Union
 from urllib.parse import quote
 
 import requests

From 71f09809fb0aef8b6b85353a6a20cf77a2577b27 Mon Sep 17 00:00:00 2001
From: Ludwig Schneider <ludwigschneider@uchicago.edu>
Date: Mon, 26 Feb 2024 13:36:37 -0600
Subject: [PATCH 29/29] remove obsolete test

---
 tests/api/test_db_schema.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/tests/api/test_db_schema.py b/tests/api/test_db_schema.py
index f665bca61..fd80d4407 100644
--- a/tests/api/test_db_schema.py
+++ b/tests/api/test_db_schema.py
@@ -122,19 +122,6 @@ def test_get_vocabulary_by_category(cript_api: cript.API) -> None:
     assert "pubchem_cid" in material_identifiers
 
 
-def test_get_controlled_vocabulary_from_api(cript_api: cript.API) -> None:
-    """
-    checks if it can successfully get the controlled vocabulary list from CRIPT API
-    """
-    number_of_vocab_categories = 26
-    vocab = cript_api.schema._get_vocab()
-
-    # assertions
-    # check vocabulary list is not empty
-    assert bool(vocab) is True
-    assert len(vocab) == number_of_vocab_categories
-
-
 def test_is_vocab_valid(cript_api: cript.API) -> None:
     """
     tests if the method for vocabulary is validating and invalidating correctly