From 3c8d5db38dd15316e0524337843ed22f7a4b5b37 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 12 Dec 2023 12:09:28 -0500
Subject: [PATCH 01/45] Minor fix to misc_utils.to_integer to handle float
 string.

---
 CHANGELOG.rst           | 5 +++++
 dcicutils/misc_utils.py | 6 +++++-
 pyproject.toml          | 2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 7e042af56..6585ab36b 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -6,6 +6,11 @@ dcicutils
 Change Log
 ----------
 
+8.6.0
+=====
+* Minor fix to misc_utils.to_integer to handle float strings.
+
+
 8.5.0
 =====
 * Moved structured_data.py from smaht-portal to here; new portal_utils and data_readers modules.
diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py
index baecf97c6..777d0a945 100644
--- a/dcicutils/misc_utils.py
+++ b/dcicutils/misc_utils.py
@@ -982,7 +982,11 @@ def to_integer(value: str, fallback: Optional[Any] = None) -> Optional[Any]:
     try:
         return int(value)
     except Exception:
-        return fallback
+        try:
+            return int(float(value))
+        except:
+            pass
+    return fallback
 
 
 def to_float(value: str, fallback: Optional[Any] = None) -> Optional[Any]:
diff --git a/pyproject.toml b/pyproject.toml
index 9a9e81594..58df16158 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0"
+version = "8.5.0.1b1"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From bec08845625b4f1efe00c6eef6c2586f478546eb Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 12 Dec 2023 12:09:52 -0500
Subject: [PATCH 02/45] Minor fix to misc_utils.to_integer to handle float
 string.

---
 dcicutils/misc_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py
index 777d0a945..ebc5d4b7e 100644
--- a/dcicutils/misc_utils.py
+++ b/dcicutils/misc_utils.py
@@ -984,7 +984,7 @@ def to_integer(value: str, fallback: Optional[Any] = None) -> Optional[Any]:
     except Exception:
         try:
             return int(float(value))
-        except:
+        except Exception:
             pass
     return fallback
 

From 2c15cec05ac4d62fbc2865a9a3a6b1afbfbbb4c3 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 12 Dec 2023 12:15:27 -0500
Subject: [PATCH 03/45] Minor fix to misc_utils.to_integer to handle float
 string.

---
 test/test_misc_utils.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py
index 6bf0afebe..8dfa8454c 100644
--- a/test/test_misc_utils.py
+++ b/test/test_misc_utils.py
@@ -31,7 +31,7 @@
     ObsoleteError, CycleError, TopologicalSorter, keys_and_values_to_dict, dict_to_keys_and_values, is_c4_arn,
     deduplicate_list, chunked, parse_in_radix, format_in_radix, managed_property, future_datetime,
     MIN_DATETIME, MIN_DATETIME_UTC, INPUT, builtin_print, map_chunked, to_camel_case, json_file_contents,
-    pad_to, JsonLinesReader, split_string, merge_objects
+    pad_to, JsonLinesReader, split_string, merge_objects, to_integer
 )
 from dcicutils.qa_utils import (
     Occasionally, ControlledTime, override_environ as qa_override_environ, MockFileSystem, printed_output,
@@ -3674,3 +3674,14 @@ def test_merge_objects_8():
                 "xyzzy": [{"foo": None}, {"goo": None}, {"hoo": None}, {"hoo": None}, {"hoo": None}]}
     merge_objects(target, source, True)
     assert target == expected
+
+
+def test_to_integer():
+    assert to_integer("17") == 17
+    assert to_integer("17.0") == 17
+    assert to_integer("17.1") == 17
+    assert to_integer("17.9", "123") == 17
+    assert to_integer("0") == 0
+    assert to_integer("0.0") == 0
+    assert to_integer("asdf") is None
+    assert to_integer("asdf", "myfallback") == "myfallback"

From a447f3fadf95145be837e8f5f511052bcbb1393d Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 12 Dec 2023 14:45:08 -0500
Subject: [PATCH 04/45] Minor fix to structured_data to accumulate unique
 resolved_refs across schemas.

---
 CHANGELOG.rst                | 1 +
 dcicutils/structured_data.py | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 6585ab36b..8c0206d6a 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -9,6 +9,7 @@ Change Log
 8.6.0
 =====
 * Minor fix to misc_utils.to_integer to handle float strings.
+* Minor fix to structured_data to accumulate unique resolved_refs across schemas.
 
 
 8.5.0
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index de25ef114..eab7e7112 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -50,7 +50,7 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp
         self._prune = prune
         self._warnings = {}
         self._errors = {}
-        self._resolved_refs = []
+        self._resolved_refs = set()
         self._validated = False
         self._load_file(file) if file else None
 
@@ -96,7 +96,7 @@ def validation_errors(self) -> List[dict]:
 
     @property
     def resolved_refs(self) -> List[str]:
-        return self._resolved_refs
+        return list(self._resolved_refs)
 
     @property
     def upload_files(self) -> List[str]:
@@ -167,7 +167,7 @@ def _load_reader(self, reader: RowReader, type_name: str) -> None:
         self._note_warning(reader.warnings, "reader")
         if schema:
             self._note_error(schema._unresolved_refs, "ref")
-            self._resolved_refs = schema._resolved_refs
+            self._resolved_refs.update(schema._resolved_refs)
 
     def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
         if self._prune:

From 8f4f6940af49393d6ff0b9b04d855a09aa606d83 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 12 Dec 2023 18:16:53 -0500
Subject: [PATCH 05/45] Changes to structured_data to respect uniqueItems for
 arrays.

---
 CHANGELOG.rst                |  1 +
 dcicutils/misc_utils.py      | 17 +++++++++++------
 dcicutils/structured_data.py | 29 +++++++++++++++++------------
 test/test_misc_utils.py      | 10 ++++++++--
 4 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 8c0206d6a..cc20fc041 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -10,6 +10,7 @@ Change Log
 =====
 * Minor fix to misc_utils.to_integer to handle float strings.
 * Minor fix to structured_data to accumulate unique resolved_refs across schemas.
+* Changes to structured_data to respect uniqueItems for arrays.
 
 
 8.5.0
diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py
index ebc5d4b7e..307c66ca3 100644
--- a/dcicutils/misc_utils.py
+++ b/dcicutils/misc_utils.py
@@ -1469,28 +1469,33 @@ def string_list(s):
     return [p for p in [part.strip() for part in s.split(",")] if p]
 
 
-def split_string(value: str, delimiter: str, escape: Optional[str] = None) -> List[str]:
+def split_string(value: str, delimiter: str, escape: Optional[str] = None, unique: bool = False) -> List[str]:
     """
     Splits the given string into an array of string based on the given delimiter, and an optional escape character.
     """
     if not isinstance(value, str) or not (value := value.strip()):
         return []
-    if not isinstance(escape, str) or not escape:
-        return [item.strip() for item in value.split(delimiter)]
     result = []
+    if not isinstance(escape, str) or not escape:
+        for item in value.split(delimiter):
+            if (item := item.strip()) and (unique is not True or item not in result):
+                result.append(item)
+        return result
     item = r""
     escaped = False
     for c in value:
         if c == delimiter and not escaped:
-            result.append(item.strip())
+            if (item := item.strip()) and (unique is not True or item not in result):
+                result.append(item)
             item = r""
         elif c == escape and not escaped:
             escaped = True
         else:
             item += c
             escaped = False
-    result.append(item.strip())
-    return [item for item in result if item]
+    if (item := item.strip()) and (unique is not True or item not in result):
+        result.append(item)
+    return result
 
 
 def right_trim(list_or_tuple: Union[List[Any], Tuple[Any]],
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index eab7e7112..71400929a 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -237,7 +237,7 @@ def parse_components(column_components: List[str], path: List[Union[str, int]])
             return {array_name: array} if array_name else {column_component: value}
 
         def set_value_internal(data: Union[dict, list], value: Optional[Any], src: Optional[str],
-                               path: List[Union[str, int]], mapv: Optional[Callable]) -> None:
+                               path: List[Union[str, int]], typeinfo: Optional[dict], mapv: Optional[Callable]) -> None:
 
             def set_value_backtrack_object(path_index: int, path_element: str) -> None:
                 nonlocal data, path, original_data
@@ -257,7 +257,7 @@ def set_value_backtrack_object(path_index: int, path_element: str) -> None:
                     set_value_backtrack_object(i, p)
                 data = data[p]
             if (p := path[-1]) == -1 and isinstance(value, str):
-                values = _split_array_string(value)
+                values = _split_array_string(value, unique=typeinfo.get("unique") if typeinfo else False)
                 if mapv:
                     values = [mapv(value, src) for value in values]
                 merge_objects(data, values)
@@ -288,11 +288,13 @@ def ensure_column_consistency(column_name: str) -> None:
         for column_name in column_names or []:
             ensure_column_consistency(column_name)
             rational_column_name = self._schema.rationalize_column_name(column_name) if self._schema else column_name
-            map_value_function = self._schema.get_map_value_function(rational_column_name) if self._schema else None
+            column_typeinfo = self._schema.get_typeinfo(rational_column_name) if self._schema else None
+            map_value_function = column_typeinfo.get("map") if column_typeinfo else None
             if (column_components := _split_dotted_string(rational_column_name)):
                 merge_objects(structured_row_template, parse_components(column_components, path := []), True)
-                self._set_value_functions[column_name] = (lambda data, value, src, path=path, mapv=map_value_function:
-                                                          set_value_internal(data, value, src, path, mapv))
+                self._set_value_functions[column_name] = (
+                    lambda data, value, src, path=path, typeinfo=column_typeinfo, mapv=map_value_function:
+                        set_value_internal(data, value, src, path, typeinfo, mapv))
         return structured_row_template
 
 
@@ -331,10 +333,7 @@ def unresolved_refs(self) -> List[dict]:
     def resolved_refs(self) -> List[str]:
         return list(self._resolved_refs)
 
-    def get_map_value_function(self, column_name: str) -> Optional[Any]:
-        return (self._get_typeinfo(column_name) or {}).get("map")
-
-    def _get_typeinfo(self, column_name: str) -> Optional[dict]:
+    def get_typeinfo(self, column_name: str) -> Optional[dict]:
         if isinstance(info := self._typeinfo.get(column_name), str):
             info = self._typeinfo.get(info)
         if not info and isinstance(info := self._typeinfo.get(self.unadorn_column_name(column_name)), str):
@@ -467,9 +466,15 @@ def _create_typeinfo(self, schema_json: dict, parent_key: Optional[str] = None)
                             raise Exception(f"Array of undefined or multiple types in JSON schema NOT supported: {key}")
                         raise Exception(f"Invalid array type specifier in JSON schema: {key}")
                     key = key + ARRAY_NAME_SUFFIX_CHAR
+                    if unique := (property_value.get("uniqueItems") is True):
+                        pass
                     property_value = array_property_items
                     property_value_type = property_value.get("type")
-                result.update(self._create_typeinfo(array_property_items, parent_key=key))
+                typeinfo = self._create_typeinfo(array_property_items, parent_key=key)
+                if unique:
+                    typeinfo[key]["unique"] = True
+                result.update(typeinfo)
+#               result.update(self._create_typeinfo(array_property_items, parent_key=key))
                 continue
             result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
             if ARRAY_NAME_SUFFIX_CHAR in key:
@@ -615,5 +620,5 @@ def _split_dotted_string(value: str):
     return split_string(value, DOTTED_NAME_DELIMITER_CHAR)
 
 
-def _split_array_string(value: str):
-    return split_string(value, ARRAY_VALUE_DELIMITER_CHAR, ARRAY_VALUE_DELIMITER_ESCAPE_CHAR)
+def _split_array_string(value: str, unique: bool = False):
+    return split_string(value, ARRAY_VALUE_DELIMITER_CHAR, ARRAY_VALUE_DELIMITER_ESCAPE_CHAR, unique=unique)
diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py
index 8dfa8454c..fabf661f6 100644
--- a/test/test_misc_utils.py
+++ b/test/test_misc_utils.py
@@ -3593,8 +3593,8 @@ def test_json_lines_reader_lists():
 
 
 def test_split_array_string():
-    def split_array_string(value: str) -> List[str]:
-        return split_string(value, "|", "\\")
+    def split_array_string(value: str, unique: bool = False) -> List[str]:
+        return split_string(value, "|", "\\", unique=unique)
     assert split_array_string(r"abc|def|ghi") == ["abc", "def", "ghi"]
     assert split_array_string(r"abc\|def|ghi") == ["abc|def", "ghi"]
     assert split_array_string(r"abc\\|def|ghi") == ["abc\\", "def", "ghi"]
@@ -3609,6 +3609,12 @@ def split_array_string(value: str) -> List[str]:
     assert split_array_string(r"|") == []
     assert split_array_string(r"\|") == ["|"]
     assert split_array_string(r"\\|") == ["\\"]
+    assert split_array_string(r"abc|def|abc|ghi", unique=False) == ["abc", "def", "abc", "ghi"]
+    assert split_array_string(r"abc|def|abc|ghi", unique=True) == ["abc", "def", "ghi"]
+    assert split_array_string(r"abc\\\|def\|ghi|jkl|mno|jkl", unique=False) == ["abc\\|def|ghi", "jkl", "mno", "jkl"]
+    assert split_array_string(r"abc\\\|def\|ghi|jkl|mno|jkl", unique=True) == ["abc\\|def|ghi", "jkl", "mno"]
+    assert split_string(r"abc|def|ghi|def", delimiter="|", unique=False) == ["abc", "def", "ghi", "def"]
+    assert split_string(r"abc|def|ghi|def", delimiter="|", unique=True) == ["abc", "def", "ghi"]
 
 
 def test_merge_objects_1():

From a7f86771fc74022a30b1f3933d8fbcbfa46fcc5c Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 12 Dec 2023 18:18:14 -0500
Subject: [PATCH 06/45] Changes to structured_data to respect uniqueItems for
 arrays.

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 58df16158..cbc4d28dc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b1"  # TODO: To become 8.6.0
+version = "8.5.0.1b2"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From 34e7466298ceccbd64ae5fab7e84cf4ea1791986 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Wed, 13 Dec 2023 11:34:34 -0500
Subject: [PATCH 07/45] Added portal_utils.Portal.key_id

---
 CHANGELOG.rst                |  3 +++
 dcicutils/portal_utils.py    | 18 +++++++++++++++++-
 dcicutils/structured_data.py | 12 ++++++++----
 pyproject.toml               |  2 +-
 4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index cc20fc041..16d295207 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -11,6 +11,9 @@ Change Log
 * Minor fix to misc_utils.to_integer to handle float strings.
 * Minor fix to structured_data to accumulate unique resolved_refs across schemas.
 * Changes to structured_data to respect uniqueItems for arrays.
+* Handle no schemas better in structured_data.
+* Added portal_utils.Portal.ping().
+* Minor fix in portal_utils.Portal._uri().
 
 
 8.5.0
diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 82a34f574..33cf7eb87 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -62,6 +62,7 @@ def __init__(self,
             self._server = portal._server
             self._key = portal._key
             self._key_pair = portal._key_pair
+            self._key_id = portal._key_id
             self._key_file = portal._key_file
             return
         self._vapp = None
@@ -70,6 +71,7 @@ def __init__(self,
         self._server = server
         self._key = None
         self._key_pair = None
+        self._key_id = None
         self._key_file = None
         if isinstance(portal, (VirtualApp, TestApp)):
             self._vapp = portal
@@ -95,6 +97,10 @@ def __init__(self,
                 self._key = key_manager.get_keydict_for_server(self._server)
             self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
             self._key_file = key_manager.keys_file
+        if self._key and (key_id := self._key.get("key")):
+            self._key_id = key_id
+        elif self._key_pair and (key_id := self._key_pair[1]):
+            self._key_id = key_id
 
     @property
     def env(self):
@@ -116,6 +122,10 @@ def key(self):
     def key_pair(self):
         return self._key_pair
 
+    @property
+    def key_id(self):
+        return self._key_id
+
     @property
     def key_file(self):
         return self._key_file
@@ -206,13 +216,19 @@ def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
             super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
         return super_type_map_flattened
 
+    def ping(self) -> bool:
+        try:
+            return self.get("/health").status_code == 200
+        except Exception:
+            return False
+
     def _uri(self, uri: str) -> str:
         if not isinstance(uri, str) or not uri:
             return "/"
         if uri.lower().startswith("http://") or uri.lower().startswith("https://"):
             return uri
         uri = re.sub(r"/+", "/", uri)
-        return (self._server + ("/" if uri.startswith("/") else "") + uri) if self._server else uri
+        return (self._server + ("/" if not uri.startswith("/") else "") + uri) if self._server else uri
 
     def _kwargs(self, **kwargs) -> dict:
         result_kwargs = {"headers":
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index 71400929a..ef309e921 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -317,7 +317,8 @@ def __init__(self, schema_json: dict, portal: Optional[Portal] = None) -> None:
 
     @staticmethod
     def load_by_name(name: str, portal: Portal) -> Optional[dict]:
-        return Schema(portal.get_schema(Schema.type_name(name)), portal) if portal else None
+        schema_json = portal.get_schema(Schema.type_name(name)) if portal else None
+        return Schema(schema_json, portal) if schema_json else None
 
     def validate(self, data: dict) -> List[str]:
         errors = []
@@ -565,7 +566,9 @@ def get_metadata(self, object_name: str) -> Optional[dict]:
 
     @lru_cache(maxsize=256)
     def get_schema(self, schema_name: str) -> Optional[dict]:
-        if (schemas := self.get_schemas()) and (schema := schemas.get(schema_name := Schema.type_name(schema_name))):
+        if not (schemas := self.get_schemas()):
+            return None
+        if schema := schemas.get(schema_name := Schema.type_name(schema_name)):
             return schema
         if schema_name == schema_name.upper() and (schema := schemas.get(schema_name.lower().title())):
             return schema
@@ -573,8 +576,9 @@ def get_schema(self, schema_name: str) -> Optional[dict]:
             return schema
 
     @lru_cache(maxsize=1)
-    def get_schemas(self) -> dict:
-        schemas = super().get_schemas()
+    def get_schemas(self) -> Optional[dict]:
+        if not (schemas := super().get_schemas()) or (schemas.get("status") == "error"):
+            return None
         if self._schemas:
             schemas = copy.deepcopy(schemas)
             for user_specified_schema in self._schemas:
diff --git a/pyproject.toml b/pyproject.toml
index cbc4d28dc..0ba2fb089 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b2"  # TODO: To become 8.6.0
+version = "8.5.0.1b3"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From ff035dd2ef915022107b887e9a438d9546e3859f Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Thu, 14 Dec 2023 10:28:19 -0500
Subject: [PATCH 08/45] added autoadd support to structured_data.

---
 CHANGELOG.rst                |  3 +++
 dcicutils/portal_utils.py    | 13 ++++++-------
 dcicutils/structured_data.py | 16 ++++++++++++----
 pyproject.toml               |  2 +-
 4 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 16d295207..85c97e508 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -10,6 +10,9 @@ Change Log
 =====
 * Minor fix to misc_utils.to_integer to handle float strings.
 * Minor fix to structured_data to accumulate unique resolved_refs across schemas.
+* Added ability to autoadd properties structured_data.StructuredDataSet;
+  to automatically pass in submission_centers on submission, and
+  not require that the user explicitly set this in the spreadsheet.
 * Changes to structured_data to respect uniqueItems for arrays.
 * Handle no schemas better in structured_data.
 * Added portal_utils.Portal.ping().
diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 33cf7eb87..e219d0f99 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -23,15 +23,14 @@ class Portal:
     2. From a key dictionary, containing "key" and "secret" property values.
     3. From a key tuple, containing (in order) a key and secret values.
     4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
-       or "fourfront"; and where this file is assumed to contain a dictionary with a key equal to the given "env"
-       value (e.g. smaht-localhost) and with a dictionary value containing "key" and "secret" property values; if
-       an "app" value is not specified but the given "env" value begins with one of the app values then that value
-       will be used, i.e. e.g. if env is "smaht-localhost" and app is unspecified than it is assumed to be "smaht".
+       or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
+       and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
+       if an "app" value is not specified but the given "env" value begins with one of the app values then that value
+       will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
     5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
-       is looked up by the given "server" name and the "server" key dictionary value in the key file.
+       is looked up via the given "server" name and the "server" key dictionary value in the key file.
     6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
-    7. From another Portal object.
-    8. From a a pyramid Router object.
+    7. From another Portal object; or from a a pyramid Router object.
     """
     def __init__(self,
                  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index ef309e921..fdd4735ae 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -42,8 +42,8 @@
 class StructuredDataSet:
 
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-                 schemas: Optional[List[dict]] = None, data: Optional[List[dict]] = None,
-                 order: Optional[List[str]] = None, prune: bool = True) -> None:
+                 schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
+                 data: Optional[List[dict]] = None, order: Optional[List[str]] = None, prune: bool = True) -> None:
         self.data = {} if not data else data  # If portal is None then no schemas nor refs.
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
@@ -52,13 +52,14 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp
         self._errors = {}
         self._resolved_refs = set()
         self._validated = False
+        self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._load_file(file) if file else None
 
     @staticmethod
     def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-             schemas: Optional[List[dict]] = None,
+             schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
              order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
-        return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
+        return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
 
     def validate(self, force: bool = False) -> None:
         if self._validated and not force:
@@ -163,6 +164,8 @@ def _load_reader(self, reader: RowReader, type_name: str) -> None:
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
+                if self._autoadd_properties:
+                    self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
         self._note_warning(reader.warnings, "reader")
         if schema:
@@ -177,6 +180,11 @@ def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
         else:
             self.data[type_name] = [data] if isinstance(data, dict) else data
 
+    def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
+        for name in properties:
+            if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
+                structured_row[name] = properties[name]
+
     def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
         self._note_issue(self._warnings, item, group)
 
diff --git a/pyproject.toml b/pyproject.toml
index 0ba2fb089..e138a0f04 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b3"  # TODO: To become 8.6.0
+version = "8.5.0.1b4"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From ff31a6ed8e819e7072afceabbcecf5d9d3523350 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Thu, 14 Dec 2023 17:04:57 -0500
Subject: [PATCH 09/45] minor code cleanup in structured_data

---
 dcicutils/structured_data.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index fdd4735ae..5f4123230 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -43,8 +43,8 @@ class StructuredDataSet:
 
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
                  schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
-                 data: Optional[List[dict]] = None, order: Optional[List[str]] = None, prune: bool = True) -> None:
-        self.data = {} if not data else data  # If portal is None then no schemas nor refs.
+                 order: Optional[List[str]] = None, prune: bool = True) -> None:
+        self.data = {}
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
         self._prune = prune

From e12a6b5bdfd23a74766e47e82a27b1128b6bf468 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Thu, 14 Dec 2023 17:05:08 -0500
Subject: [PATCH 10/45] minor code cleanup in structured_data

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index e138a0f04..0a641ef31 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b4"  # TODO: To become 8.6.0
+version = "8.5.0.1b5"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From cad51e1e4f6844b1048a4c51f4ce24a1a2902f7f Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Thu, 14 Dec 2023 17:27:10 -0500
Subject: [PATCH 11/45] minor code cleanup in structured_data

---
 dcicutils/structured_data.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index 5f4123230..ccfea9492 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -113,7 +113,7 @@ def upload_files(self) -> List[str]:
     def _load_file(self, file: str) -> None:
         # Returns a dictionary where each property is the name (i.e. the type) of the data,
         # and the value is array of dictionaries for the data itself. Handle these kinds of files:
-        # 1.  Single CSV of JSON file, where the (base) name of the file is the data type name.
+        # 1.  Single CSV, TSV, or JSON file, where the (base) name of the file is the data type name.
         # 2.  Single Excel file containing one or more sheets, where each sheet
         #     represents (i.e. is named for, and contains data for) a different type.
         # 3.  Zip file (.zip or .tar.gz or .tgz or .tar), containing data files to load, where the
@@ -483,7 +483,6 @@ def _create_typeinfo(self, schema_json: dict, parent_key: Optional[str] = None)
                 if unique:
                     typeinfo[key]["unique"] = True
                 result.update(typeinfo)
-#               result.update(self._create_typeinfo(array_property_items, parent_key=key))
                 continue
             result[key] = {"type": property_value_type, "map": self._map_function({**property_value, "column": key})}
             if ARRAY_NAME_SUFFIX_CHAR in key:

From b498207551ba27c9e6e0c9c6d9b03a42d56abc12 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Sun, 17 Dec 2023 16:16:07 -0500
Subject: [PATCH 12/45] Refactor Portal construction.

---
 dcicutils/portal_utils.py    | 233 +++++++++++++++++++++--------------
 dcicutils/structured_data.py |  17 ++-
 pyproject.toml               |   2 +-
 3 files changed, 150 insertions(+), 102 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index e219d0f99..be831454c 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -1,13 +1,15 @@
 from collections import deque
+import io
+import json
 from pyramid.paster import get_app
 from pyramid.router import Router
+import os
 import re
 import requests
 from requests.models import Response as RequestResponse
 from typing import Optional, Type, Union
 from webtest.app import TestApp, TestResponse
-from dcicutils.common import OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT, ORCHESTRATED_APPS
-from dcicutils.creds_utils import CGAPKeyManager, FourfrontKeyManager, SMaHTKeyManager
+from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
 from dcicutils.zip_utils import temporary_file
@@ -33,104 +35,144 @@ class Portal:
     7. From another Portal object; or from a a pyramid Router object.
     """
     def __init__(self,
-                 arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
-                 env: Optional[str] = None, app: Optional[OrchestratedApp] = None, server: Optional[str] = None,
-                 key: Optional[Union[dict, tuple]] = None,
-                 vapp: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
-                 portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None) -> Portal:
-        if vapp and not portal:
-            portal = vapp
-        if ((isinstance(arg, (VirtualApp, TestApp, Router, Portal)) or
-             isinstance(arg, str) and arg.endswith(".ini")) and not portal):
-            portal = arg
-        elif isinstance(arg, str) and not env:
-            env = arg
-        elif (isinstance(arg, dict) or isinstance(arg, tuple)) and not key:
-            key = arg
-        if not app and env:
-            if env.startswith(APP_SMAHT):
-                app = APP_SMAHT
-            elif env.startswith(APP_CGAP):
-                app = APP_CGAP
-            elif env.startswith(APP_FOURFRONT):
-                app = APP_FOURFRONT
-        if isinstance(portal, Portal):
-            self._vapp = portal._vapp
-            self._env = portal._env
-            self._app = portal._app
-            self._server = portal._server
+                 arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
+                 env: Optional[str] = None, server: Optional[str] = None,
+                 app: Optional[OrchestratedApp] = None) -> None:
+
+        def init(unspecified: Optional[list] = []) -> None:
+            self._ini_file = None
+            self._key = None
+            self._key_pair = None
+            self._key_id = None
+            self._keys_file = None
+            self._env = None
+            self._server = None
+            self._app = None
+            self._vapp = None
+            for arg in unspecified:
+                if arg is not None:
+                    raise Exception("Portal init error; extraneous args.")
+
+        def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None:
+            init(unspecified)
+            self._ini_file = portal._ini_file
             self._key = portal._key
             self._key_pair = portal._key_pair
             self._key_id = portal._key_id
-            self._key_file = portal._key_file
-            return
-        self._vapp = None
-        self._env = env
-        self._app = app
-        self._server = server
-        self._key = None
-        self._key_pair = None
-        self._key_id = None
-        self._key_file = None
-        if isinstance(portal, (VirtualApp, TestApp)):
-            self._vapp = portal
-        elif isinstance(portal, (Router, str)):
-            self._vapp = Portal._create_vapp(portal)
-        elif isinstance(key, dict):
-            self._key = key
-            self._key_pair = (key.get("key"), key.get("secret")) if key else None
-            if key_server := key.get("server"):
-                self._server = key_server
-        elif isinstance(key, tuple) and len(key) >= 2:
-            self._key = {"key": key[0], "secret": key[1]}
-            self._key_pair = key
-        elif isinstance(env, str):
-            key_managers = {APP_CGAP: CGAPKeyManager, APP_FOURFRONT: FourfrontKeyManager, APP_SMAHT: SMaHTKeyManager}
-            if not (key_manager := key_managers.get(self._app)) or not (key_manager := key_manager()):
-                raise Exception(f"Invalid app name: {self._app} (valid: {', '.join(ORCHESTRATED_APPS)}).")
-            if isinstance(env, str):
-                self._key = key_manager.get_keydict_for_env(env)
-                if key_server := self._key.get("server"):
-                    self._server = key_server
-            elif isinstance(self._server, str):
-                self._key = key_manager.get_keydict_for_server(self._server)
-            self._key_pair = key_manager.keydict_to_keypair(self._key) if self._key else None
-            self._key_file = key_manager.keys_file
-        if self._key and (key_id := self._key.get("key")):
-            self._key_id = key_id
-        elif self._key_pair and (key_id := self._key_pair[1]):
-            self._key_id = key_id
+            self._keys_file = portal._keys_file
+            self._env = portal._env
+            self._server = portal._server
+            self._app = portal._app
+            self._vapp = portal._vapp
 
-    @property
-    def env(self):
-        return self._env
+        def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            self._vapp = Portal._create_testapp(vapp)
+
+        def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            self._ini_file = ini_file
+            self._vapp = Portal._create_testapp(ini_file)
+
+        def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            if (isinstance(key_id := key.get("key"), str) and key_id and
+                isinstance(secret := key.get("secret"), str) and secret):  # noqa
+                self._key = {"key": key_id, "secret": secret}
+                self._key_id = key_id
+                self._key_pair = (key_id, secret)
+                if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
+                    self._key["server"] = self._server = server
+            if not self._key:
+                raise Exception("Portal init error; from key.")
+
+        def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Optional[list] = []) -> None:
+            if len(key_pair) == 2:
+                init_from_key({"key": key_pair[0], "secret": key_pair[1]}, server, unspecified)
+            else:
+                raise Exception("Portal init error; from key-pair.")
+
+        def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
+                                unspecified: Optional[list] = []) -> None:
+            init(unspecified)
+            with io.open(self.keys_file) as f:
+                keys = json.load(f)
+                if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
+                    init_from_key(key, server)
+                    self._env = env
+                elif isinstance(server, str) and server and (key := [k for k in keys if k.get("server") == server]):
+                    init_from_key(key, server)
+            if not self._keys_file:
+                raise Exception("Portal init error; from key-file.")
+
+        def init_from_env_server_app(env: str, server: str, app: Optional[str],
+                                     unspecified: Optional[list] = None) -> None:
+            return init_from_keys_file(get_default_keys_file(app, env), unspecified)
+
+        def get_default_keys_file(app: Optional[str], env: Optional[str] = None) -> Optional[str]:
+            def is_valid_app(app: Optional[str]) -> bool:  # noqa
+                return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
+            def infer_app_from_env(env: str) -> Optional[str]:  # noqa
+                if isinstance(env, str) and env:
+                    if app := [app for app in ORCHESTRATED_APPS if app.lower().startswith(env.lower())]:
+                        return app[0]
+            if is_valid_app(app) or (app := infer_app_from_env(env)):
+                return os.path.expanduser(f"~/{app.lower()}-keys.json")
+            return None
+
+        if isinstance(arg, Portal):
+            init_from_portal(arg, unspecified=[env, server, app])
+        elif isinstance(arg, (TestApp, VirtualApp, Router)):
+            init_from_vapp(arg, unspecified=[env, server, app])
+        elif isinstance(arg, str) and arg.endswith(".ini"):
+            init_from_ini_file(arg, unspecified=[env, server, app])
+        elif isinstance(arg, dict):
+            init_from_key(arg, server, unspecified=[env, app])
+        elif isinstance(arg, tuple):
+            init_from_key_pair(arg, server, unspecified=[env, app])
+        elif isinstance(arg, str) and arg.endswith(".json"):
+            init_from_keys_file(arg, env, server, unspecified=[app])
+        elif isinstance(arg, str) and arg:
+            init_from_env_server_app(arg, server, app, unspecified=[env])
+        elif isinstance(env, str) and env:
+            init_from_env_server_app(env, server, app, unspecified=[arg])
+        else:
+            raise Exception("Portal construction error [0].")
 
     @property
-    def app(self):
-        return self._app
+    def ini_file(self) -> Optional[str]:
+        return self._ini_file
 
     @property
-    def server(self):
-        return self._server
+    def keys_file(self) -> Optional[str]:
+        return self._keys_file
 
     @property
-    def key(self):
+    def key(self) -> Optional[dict]:
         return self._key
 
     @property
-    def key_pair(self):
+    def key_pair(self) -> Optional[tuple]:
         return self._key_pair
 
     @property
-    def key_id(self):
+    def key_id(self) -> Optional[str]:
         return self._key_id
 
     @property
-    def key_file(self):
-        return self._key_file
+    def env(self) -> Optional[str]:
+        return self._env
+
+    @property
+    def app(self) -> Optional[str]:
+        return self._app
+
+    @property
+    def server(self) -> Optional[str]:
+        return self._server
 
     @property
-    def vapp(self):
+    def vapp(self) -> Optional[TestApp]:
         return self._vapp
 
     def get_metadata(self, object_id: str) -> Optional[dict]:
@@ -147,7 +189,7 @@ def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
         return self.post(f"/{object_type}", data)
 
     def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if isinstance(self._vapp, (VirtualApp, TestApp)):
+        if self._vapp:
             response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
             if response and response.status_code in [301, 302, 303, 307, 308] and follow:
                 response = response.follow()
@@ -156,13 +198,13 @@ def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[Request
 
     def patch(self, uri: str, data: Optional[dict] = None,
               json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if isinstance(self._vapp, (VirtualApp, TestApp)):
+        if self._vapp:
             return self._vapp.patch_json(self._uri(uri), json or data, **self._kwargs(**kwargs))
         return requests.patch(self._uri(uri), json=json or data, **self._kwargs(**kwargs))
 
     def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
              files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if isinstance(self._vapp, (VirtualApp, TestApp)):
+        if self._vapp:
             if files:
                 return self._vapp.post(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
             else:
@@ -254,15 +296,15 @@ def json(self):  # noqa
     @staticmethod
     def create_for_testing(ini_file: Optional[str] = None) -> Portal:
         if isinstance(ini_file, str):
-            return Portal(Portal._create_vapp(ini_file))
+            return Portal(Portal._create_testapp(ini_file))
         minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
         with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
-            return Portal(Portal._create_vapp(ini_file))
+            return Portal(Portal._create_testapp(ini_file))
 
     @staticmethod
     def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
         if isinstance(ini_file, str) and ini_file:
-            return Portal(Portal._create_vapp(ini_file))
+            return Portal(Portal._create_testapp(ini_file))
         minimal_ini_for_testing_local = "\n".join([
             "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
             "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
@@ -283,11 +325,20 @@ def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
             "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
         ])
         with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
-            return Portal(Portal._create_vapp(minimal_ini_file))
+            return Portal(Portal._create_testapp(minimal_ini_file))
 
     @staticmethod
-    def _create_vapp(value: Union[str, Router, TestApp] = "development.ini", app_name: str = "app") -> TestApp:
-        if isinstance(value, TestApp):
-            return value
-        app = value if isinstance(value, Router) else get_app(value, app_name)
-        return TestApp(app, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
+    def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
+        if isinstance(arg, TestApp):
+            return arg
+        elif isinstance(arg, VirtualApp):
+            if not isinstance(arg.wrapped_app, TestApp):
+                raise Exception("Portal._create_testapp VirtualApp argument error.")
+            return arg.wrapped_app
+        if isinstance(arg, Router):
+            router = arg
+        elif isinstance(arg, str) or arg is None:
+            router = get_app(arg or "development.ini", app_name or "app")
+        else:
+            raise Exception("Portal._create_testapp argument error.")
+        return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index ccfea9492..82acc4493 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -550,16 +550,13 @@ class Portal(PortalBase):
 
     def __init__(self,
                  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,
-                 env: Optional[str] = None, app: OrchestratedApp = None, server: Optional[str] = None,
-                 key: Optional[Union[dict, tuple]] = None,
-                 portal: Optional[Union[VirtualApp, TestApp, Router, Portal, str]] = None,
-                 data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> Optional[Portal]:
-        super().__init__(arg, env=env, app=app, server=server, key=key, portal=portal)
-        if isinstance(arg, Portal) and not portal:
-            portal = arg
-        if isinstance(portal, Portal):
-            self._schemas = schemas if schemas is not None else portal._schemas  # Explicitly specified/known schemas.
-            self._data = data if data is not None else portal._data  # Data set being loaded; e.g. by StructuredDataSet.
+                 env: Optional[str] = None, server: Optional[str] = None,
+                 app: Optional[OrchestratedApp] = None,
+                 data: Optional[dict] = None, schemas: Optional[List[dict]] = None) -> None:
+        super().__init__(arg, env=env, server=server, app=app)
+        if isinstance(arg, Portal):
+            self._schemas = schemas if schemas is not None else arg._schemas  # Explicitly specified/known schemas.
+            self._data = data if data is not None else arg._data  # Data set being loaded; e.g. by StructuredDataSet.
         else:
             self._schemas = schemas
             self._data = data
diff --git a/pyproject.toml b/pyproject.toml
index 0a641ef31..e252afde0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b5"  # TODO: To become 8.6.0
+version = "8.5.0.1b6"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From 68789e7cfdd9514b9035eb4b2925c9cdb47b3150 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 00:04:12 -0500
Subject: [PATCH 13/45] Refactor Portal construction.

---
 dcicutils/portal_utils.py | 73 ++++++++++++++++++++++++---------------
 1 file changed, 46 insertions(+), 27 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index be831454c..88d12258b 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -82,7 +82,8 @@ def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list]
                 self._key_id = key_id
                 self._key_pair = (key_id, secret)
                 if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
-                    self._key["server"] = self._server = server
+                    if server := normalize_server(server):
+                        self._key["server"] = self._server = server
             if not self._key:
                 raise Exception("Portal init error; from key.")
 
@@ -94,31 +95,45 @@ def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Opti
 
         def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
                                 unspecified: Optional[list] = []) -> None:
-            init(unspecified)
-            with io.open(self.keys_file) as f:
+            with io.open(keys_file) as f:
                 keys = json.load(f)
-                if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
-                    init_from_key(key, server)
-                    self._env = env
-                elif isinstance(server, str) and server and (key := [k for k in keys if k.get("server") == server]):
-                    init_from_key(key, server)
-            if not self._keys_file:
-                raise Exception("Portal init error; from key-file.")
+            if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
+                init_from_key(key, server)
+                self._keys_file = keys_file
+                self._env = env
+            elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
+                init_from_key(key, server)
+                self._keys_file = keys_file
+            else:
+                raise Exception((f"Portal init error; " +
+                                 f"env ({env}) or server ({server}) not found in keys-file: {keys_file}"))
 
         def init_from_env_server_app(env: str, server: str, app: Optional[str],
                                      unspecified: Optional[list] = None) -> None:
-            return init_from_keys_file(get_default_keys_file(app, env), unspecified)
+            return init_from_keys_file(default_keys_file(app, env), env, server, unspecified)
 
-        def get_default_keys_file(app: Optional[str], env: Optional[str] = None) -> Optional[str]:
+        def default_keys_file(app: Optional[str], env: Optional[str] = None) -> Optional[str]:
             def is_valid_app(app: Optional[str]) -> bool:  # noqa
                 return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
             def infer_app_from_env(env: str) -> Optional[str]:  # noqa
-                if isinstance(env, str) and env:
-                    if app := [app for app in ORCHESTRATED_APPS if app.lower().startswith(env.lower())]:
+                if isinstance(env, str) and (lenv := env.lower()):
+                    if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
                         return app[0]
             if is_valid_app(app) or (app := infer_app_from_env(env)):
-                return os.path.expanduser(f"~/{app.lower()}-keys.json")
-            return None
+                return os.path.expanduser(f"~/.{app.lower()}-keys.json")
+
+        def normalize_server(server: str) -> Optional[str]:
+            prefix = ""
+            if (lserver := server.lower()).startswith("http://"):
+                prefix = "http://"
+            elif lserver.startswith("https://"):
+                prefix = "https://"
+            if prefix:
+                if (server := re.sub(r"/+", "/", server[len(prefix):])).startswith("/"):
+                    server = server[1:]
+                if len(server) > 1 and server.endswith("/"):
+                    server = server[:-1]
+                return prefix + server if server else None
 
         if isinstance(arg, Portal):
             init_from_portal(arg, unspecified=[env, server, app])
@@ -143,10 +158,6 @@ def infer_app_from_env(env: str) -> Optional[str]:  # noqa
     def ini_file(self) -> Optional[str]:
         return self._ini_file
 
-    @property
-    def keys_file(self) -> Optional[str]:
-        return self._keys_file
-
     @property
     def key(self) -> Optional[dict]:
         return self._key
@@ -160,17 +171,21 @@ def key_id(self) -> Optional[str]:
         return self._key_id
 
     @property
-    def env(self) -> Optional[str]:
-        return self._env
+    def keys_file(self) -> Optional[str]:
+        return self._keys_file
 
     @property
-    def app(self) -> Optional[str]:
-        return self._app
+    def env(self) -> Optional[str]:
+        return self._env
 
     @property
     def server(self) -> Optional[str]:
         return self._server
 
+    @property
+    def app(self) -> Optional[str]:
+        return self._app
+
     @property
     def vapp(self) -> Optional[TestApp]:
         return self._vapp
@@ -266,10 +281,11 @@ def ping(self) -> bool:
     def _uri(self, uri: str) -> str:
         if not isinstance(uri, str) or not uri:
             return "/"
-        if uri.lower().startswith("http://") or uri.lower().startswith("https://"):
+        if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
             return uri
-        uri = re.sub(r"/+", "/", uri)
-        return (self._server + ("/" if not uri.startswith("/") else "") + uri) if self._server else uri
+        if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
+            uri = "/" + uri
+        return self._server + uri if self._server else uri
 
     def _kwargs(self, **kwargs) -> dict:
         result_kwargs = {"headers":
@@ -342,3 +358,6 @@ def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_nam
         else:
             raise Exception("Portal._create_testapp argument error.")
         return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
+
+p = Portal("smaht-localhost", server="http://abc.com")
+print(p.server)

From a2632776eb861dfe9326edf8e6e961b697c7775f Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 00:06:36 -0500
Subject: [PATCH 14/45] Refactor Portal construction.

---
 dcicutils/portal_utils.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 88d12258b..9dbc1c933 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -95,8 +95,11 @@ def init_from_key_pair(key_pair: tuple, server: Optional[str], unspecified: Opti
 
         def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str],
                                 unspecified: Optional[list] = []) -> None:
-            with io.open(keys_file) as f:
-                keys = json.load(f)
+            try:
+                with io.open(keys_file) as f:
+                    keys = json.load(f)
+            except Exception:
+                raise Exception(f"Portal init error; cannot open keys-file: {keys_file}")
             if isinstance(env, str) and env and isinstance(key := keys.get(env), dict):
                 init_from_key(key, server)
                 self._keys_file = keys_file
@@ -105,8 +108,7 @@ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str
                 init_from_key(key, server)
                 self._keys_file = keys_file
             else:
-                raise Exception((f"Portal init error; " +
-                                 f"env ({env}) or server ({server}) not found in keys-file: {keys_file}"))
+                raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
 
         def init_from_env_server_app(env: str, server: str, app: Optional[str],
                                      unspecified: Optional[list] = None) -> None:

From 2a51e6cfba59b10d85439500b4dccb50dd8c5c0b Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 06:03:07 -0500
Subject: [PATCH 15/45] typo

---
 dcicutils/portal_utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 9dbc1c933..9bb7d4009 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -360,6 +360,3 @@ def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_nam
         else:
             raise Exception("Portal._create_testapp argument error.")
         return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
-
-p = Portal("smaht-localhost", server="http://abc.com")
-print(p.server)

From 9a0cd498ede3d7d96dde782f70b87e5859f91752 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 06:43:50 -0500
Subject: [PATCH 16/45] typo

---
 dcicutils/portal_utils.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 9bb7d4009..7c905f094 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -15,7 +15,6 @@
 from dcicutils.zip_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
-FILE_SCHEMA_NAME = "File"
 
 
 class Portal:
@@ -34,6 +33,9 @@ class Portal:
     6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
     7. From another Portal object; or from a a pyramid Router object.
     """
+    FILE_SCHEMA_NAME = "File"
+    KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
+
     def __init__(self,
                  arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
                  env: Optional[str] = None, server: Optional[str] = None,
@@ -122,7 +124,7 @@ def infer_app_from_env(env: str) -> Optional[str]:  # noqa
                     if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
                         return app[0]
             if is_valid_app(app) or (app := infer_app_from_env(env)):
-                return os.path.expanduser(f"~/.{app.lower()}-keys.json")
+                return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
 
         def normalize_server(server: str) -> Optional[str]:
             prefix = ""
@@ -240,7 +242,7 @@ def schema_name(name: str) -> str:
 
     def is_file_schema(self, schema_name: str) -> bool:
         if super_type_map := self.get_schemas_super_type_map():
-            if file_super_type := super_type_map.get(FILE_SCHEMA_NAME):
+            if file_super_type := super_type_map.get(Portal.FILE_SCHEMA_NAME):
                 return self.schema_name(schema_name) in file_super_type
         return False
 
@@ -286,7 +288,7 @@ def _uri(self, uri: str) -> str:
         if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
             return uri
         if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
-            uri = "/" + uri
+            uri = "/"
         return self._server + uri if self._server else uri
 
     def _kwargs(self, **kwargs) -> dict:

From 71b6237c0543a7b4d1d46c516c35f5ca5b25510e Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 11:16:09 -0500
Subject: [PATCH 17/45] portal_utils tests

---
 dcicutils/portal_utils.py |   6 +-
 test/test_portal_utils.py | 118 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 123 insertions(+), 1 deletion(-)
 create mode 100644 test/test_portal_utils.py

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 7c905f094..2dcc60de3 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -109,6 +109,10 @@ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str
             elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
                 init_from_key(key, server)
                 self._keys_file = keys_file
+            elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
+                init_from_key(key, server)
+                self._keys_file = keys_file
+                self._env = env
             else:
                 raise Exception(f"Portal init error; {env or server or None} not found in keys-file: {keys_file}")
 
@@ -156,7 +160,7 @@ def normalize_server(server: str) -> Optional[str]:
         elif isinstance(env, str) and env:
             init_from_env_server_app(env, server, app, unspecified=[arg])
         else:
-            raise Exception("Portal construction error [0].")
+            raise Exception("Portal init error; invalid args.")
 
     @property
     def ini_file(self) -> Optional[str]:
diff --git a/test/test_portal_utils.py b/test/test_portal_utils.py
new file mode 100644
index 000000000..25401d2ee
--- /dev/null
+++ b/test/test_portal_utils.py
@@ -0,0 +1,118 @@
+import io
+import json
+import os
+import pytest
+from unittest import mock
+from dcicutils.portal_utils import Portal
+from dcicutils.zip_utils import temporary_file
+
+
+_TEST_KEY_ID = "TTVJOW2A"
+_TEST_SECRET = "3fbswrice6xosnjw"
+_TEST_KEY = {"key": _TEST_KEY_ID, "secret": _TEST_SECRET}
+_TEST_KEY_PAIR = (_TEST_KEY_ID, _TEST_SECRET)
+
+
+def test_portal_constructor_a():
+
+    def assert_for_server(server, expected):
+        portal = Portal(_TEST_KEY, server=server) if server is not None else Portal(_TEST_KEY)
+        assert portal.key_id == _TEST_KEY_ID
+        assert portal.key_pair == _TEST_KEY_PAIR
+        assert portal.server == expected
+        assert portal.keys_file is None
+        assert portal.app is None
+        assert portal.env is None
+        assert portal.vapp is None
+        portal = Portal({**_TEST_KEY, "server": server}) if server is not None else Portal(_TEST_KEY)
+        assert portal.key_id == _TEST_KEY_ID
+        assert portal.key_pair == _TEST_KEY_PAIR
+        assert portal.server == expected
+        assert portal.keys_file is None
+        assert portal.app is None
+        assert portal.env is None
+        assert portal.vapp is None
+
+    assert_for_server(None, None)
+    assert_for_server("http://localhost:8000", "http://localhost:8000")
+    assert_for_server("http://localhost:8000/", "http://localhost:8000")
+    assert_for_server("hTtP://localhost:8000//", "http://localhost:8000")
+    assert_for_server("hTtP://localhost:8000//", "http://localhost:8000")
+    assert_for_server("Http://xyzzy.com//abc/", "http://xyzzy.com/abc")
+    assert_for_server("xhttp://localhost:8000", None)
+    assert_for_server("http:/localhost:8000", None)
+
+
+def test_portal_constructor_b():
+
+    keys_file_content = json.dumps({
+        "smaht-local": {
+            "key": "ABCDEFGHI",
+            "secret": "adfxdloiebvhzp",
+            "server": "http://localhost:8080/"
+        },
+        "smaht-remote": {
+            "key": "GHIDEFABC",
+            "secret": "zpadfxdloiebvh",
+            "server": "https://smaht.hms.harvard.edu/"
+        }
+    }, indent=4)
+
+    with temporary_file(name=".smaht-keys.json", content=keys_file_content) as keys_file:
+
+        portal = Portal(keys_file, env="smaht-local")
+        assert portal.key_id == "ABCDEFGHI"
+        assert portal.key_pair == ("ABCDEFGHI", "adfxdloiebvhzp")
+        assert portal.server == "http://localhost:8080"
+        assert portal.key == {"key": "ABCDEFGHI", "secret": "adfxdloiebvhzp", "server": "http://localhost:8080"}
+        assert portal.keys_file == keys_file
+        assert portal.env == "smaht-local"
+        assert portal.app is None
+        assert portal.vapp is None
+        assert portal.ini_file is None
+
+        portal = Portal(keys_file, env="smaht-remote")
+        assert portal.key_id == "GHIDEFABC"
+        assert portal.key_pair == ("GHIDEFABC", "zpadfxdloiebvh")
+        assert portal.server == "https://smaht.hms.harvard.edu"
+        assert portal.key == {"key": "GHIDEFABC", "secret": "zpadfxdloiebvh", "server": "https://smaht.hms.harvard.edu"}
+        assert portal.keys_file == keys_file
+        assert portal.env == "smaht-remote"
+        assert portal.app is None
+        assert portal.vapp is None
+        assert portal.ini_file is None
+
+        Portal.KEYS_FILE_DIRECTORY = os.path.dirname(keys_file)
+        portal = Portal("smaht-local", app="smaht")
+        assert portal.key_id == "ABCDEFGHI"
+        assert portal.key_pair == ("ABCDEFGHI", "adfxdloiebvhzp")
+        assert portal.server == "http://localhost:8080"
+        assert portal.key == {"key": "ABCDEFGHI", "secret": "adfxdloiebvhzp", "server": "http://localhost:8080"}
+        assert portal.keys_file == keys_file
+        assert portal.env == "smaht-local"
+        assert portal.app is None
+        assert portal.vapp is None
+        assert portal.ini_file is None
+
+def test_portal_constructor_c():
+
+    keys_file_content = json.dumps({
+        "cgap-local": {
+            "key": "ABCDEFGHI",
+            "secret": "adfxdloiebvhzp",
+            "server": "http://localhost:8080/"
+        }
+    }, indent=4)
+
+    with temporary_file(name=".cgap-keys.json", content=keys_file_content) as keys_file:
+
+        portal = Portal(keys_file)
+        assert portal.key_id == "ABCDEFGHI"
+        assert portal.key_pair == ("ABCDEFGHI", "adfxdloiebvhzp")
+        assert portal.server == "http://localhost:8080"
+        assert portal.key == {"key": "ABCDEFGHI", "secret": "adfxdloiebvhzp", "server": "http://localhost:8080"}
+        assert portal.keys_file == keys_file
+        assert portal.env == "cgap-local"
+        assert portal.app is None
+        assert portal.vapp is None
+        assert portal.ini_file is None

From 3cc9bbadf221a5c2ad2659e303316867c3a1363e Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 11:17:39 -0500
Subject: [PATCH 18/45] portal_utils tests

---
 test/test_portal_utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/test_portal_utils.py b/test/test_portal_utils.py
index 25401d2ee..6f8510a5a 100644
--- a/test/test_portal_utils.py
+++ b/test/test_portal_utils.py
@@ -1,8 +1,5 @@
-import io
 import json
 import os
-import pytest
-from unittest import mock
 from dcicutils.portal_utils import Portal
 from dcicutils.zip_utils import temporary_file
 
@@ -94,6 +91,7 @@ def test_portal_constructor_b():
         assert portal.vapp is None
         assert portal.ini_file is None
 
+
 def test_portal_constructor_c():
 
     keys_file_content = json.dumps({

From f10e4ac49403001491249d87219e5e13cb37bb0d Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 11:20:33 -0500
Subject: [PATCH 19/45] portal_utils tests

---
 dcicutils/portal_utils.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 2dcc60de3..0d8064da8 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -118,17 +118,7 @@ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str
 
         def init_from_env_server_app(env: str, server: str, app: Optional[str],
                                      unspecified: Optional[list] = None) -> None:
-            return init_from_keys_file(default_keys_file(app, env), env, server, unspecified)
-
-        def default_keys_file(app: Optional[str], env: Optional[str] = None) -> Optional[str]:
-            def is_valid_app(app: Optional[str]) -> bool:  # noqa
-                return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
-            def infer_app_from_env(env: str) -> Optional[str]:  # noqa
-                if isinstance(env, str) and (lenv := env.lower()):
-                    if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
-                        return app[0]
-            if is_valid_app(app) or (app := infer_app_from_env(env)):
-                return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
+            return init_from_keys_file(self._default_keys_file(app, env), env, server, unspecified)
 
         def normalize_server(server: str) -> Optional[str]:
             prefix = ""
@@ -286,6 +276,16 @@ def ping(self) -> bool:
         except Exception:
             return False
 
+    def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
+        def is_valid_app(app: Optional[str]) -> bool:  # noqa
+            return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
+        def infer_app_from_env(env: str) -> Optional[str]:  # noqa
+            if isinstance(env, str) and (lenv := env.lower()):
+                if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
+                    return app[0]
+        if is_valid_app(app) or (app := infer_app_from_env(env)):
+            return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
+
     def _uri(self, uri: str) -> str:
         if not isinstance(uri, str) or not uri:
             return "/"

From 16bbd38541d909f074bd1f7bac09f59cae952e2a Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 11:30:02 -0500
Subject: [PATCH 20/45] portal_utils tests

---
 dcicutils/portal_utils.py |  7 +++++++
 test/test_portal_utils.py | 13 +++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 0d8064da8..f567b79be 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -46,6 +46,7 @@ def init(unspecified: Optional[list] = []) -> None:
             self._key = None
             self._key_pair = None
             self._key_id = None
+            self._secret = None
             self._keys_file = None
             self._env = None
             self._server = None
@@ -61,6 +62,7 @@ def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None
             self._key = portal._key
             self._key_pair = portal._key_pair
             self._key_id = portal._key_id
+            self._secret = portal._secret
             self._keys_file = portal._keys_file
             self._env = portal._env
             self._server = portal._server
@@ -82,6 +84,7 @@ def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list]
                 isinstance(secret := key.get("secret"), str) and secret):  # noqa
                 self._key = {"key": key_id, "secret": secret}
                 self._key_id = key_id
+                self._secret = secret
                 self._key_pair = (key_id, secret)
                 if ((isinstance(server, str) and server) or (isinstance(server := key.get("server"), str) and server)):
                     if server := normalize_server(server):
@@ -168,6 +171,10 @@ def key_pair(self) -> Optional[tuple]:
     def key_id(self) -> Optional[str]:
         return self._key_id
 
+    @property
+    def secret(self) -> Optional[str]:
+        return self._secret
+
     @property
     def keys_file(self) -> Optional[str]:
         return self._keys_file
diff --git a/test/test_portal_utils.py b/test/test_portal_utils.py
index 6f8510a5a..0869802d2 100644
--- a/test/test_portal_utils.py
+++ b/test/test_portal_utils.py
@@ -106,6 +106,7 @@ def test_portal_constructor_c():
 
         portal = Portal(keys_file)
         assert portal.key_id == "ABCDEFGHI"
+        assert portal.secret == "adfxdloiebvhzp"
         assert portal.key_pair == ("ABCDEFGHI", "adfxdloiebvhzp")
         assert portal.server == "http://localhost:8080"
         assert portal.key == {"key": "ABCDEFGHI", "secret": "adfxdloiebvhzp", "server": "http://localhost:8080"}
@@ -114,3 +115,15 @@ def test_portal_constructor_c():
         assert portal.app is None
         assert portal.vapp is None
         assert portal.ini_file is None
+
+        portal_copy = Portal(portal)
+        assert portal.ini_file == portal_copy.ini_file
+        assert portal.key == portal_copy.key
+        assert portal.key_pair == portal_copy.key_pair
+        assert portal.key_id == portal_copy.key_id
+        assert portal.keys_file == portal_copy.keys_file
+        assert portal.env == portal_copy.env
+        assert portal.server == portal_copy.server
+        assert portal.app == portal_copy.app
+        assert portal.vapp == portal_copy.vapp
+        assert portal.secret == portal_copy.secret

From 5a7baaf14c12963f476e145fa860bfa7f7399b85 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 11:31:14 -0500
Subject: [PATCH 21/45] portal_utils tests

---
 dcicutils/portal_utils.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index f567b79be..51f03d4c2 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -210,26 +210,26 @@ def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
 
     def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
         if self._vapp:
-            response = self._vapp.get(self._uri(uri), **self._kwargs(**kwargs))
+            response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
             if response and response.status_code in [301, 302, 303, 307, 308] and follow:
                 response = response.follow()
             return self._response(response)
-        return requests.get(self._uri(uri), allow_redirects=follow, **self._kwargs(**kwargs))
+        return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
 
     def patch(self, uri: str, data: Optional[dict] = None,
               json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
         if self._vapp:
-            return self._vapp.patch_json(self._uri(uri), json or data, **self._kwargs(**kwargs))
-        return requests.patch(self._uri(uri), json=json or data, **self._kwargs(**kwargs))
+            return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
+        return requests.patch(self.url(uri), json=json or data, **self._kwargs(**kwargs))
 
     def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
              files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
         if self._vapp:
             if files:
-                return self._vapp.post(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+                return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
             else:
-                return self._vapp.post_json(self._uri(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return requests.post(self._uri(uri), json=json or data, files=files, **self._kwargs(**kwargs))
+                return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+        return requests.post(self.url(uri), json=json or data, files=files, **self._kwargs(**kwargs))
 
     def get_schema(self, schema_name: str) -> Optional[dict]:
         return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
@@ -283,17 +283,7 @@ def ping(self) -> bool:
         except Exception:
             return False
 
-    def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
-        def is_valid_app(app: Optional[str]) -> bool:  # noqa
-            return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
-        def infer_app_from_env(env: str) -> Optional[str]:  # noqa
-            if isinstance(env, str) and (lenv := env.lower()):
-                if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
-                    return app[0]
-        if is_valid_app(app) or (app := infer_app_from_env(env)):
-            return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
-
-    def _uri(self, uri: str) -> str:
+    def url(self, uri: str) -> str:
         if not isinstance(uri, str) or not uri:
             return "/"
         if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
@@ -311,6 +301,16 @@ def _kwargs(self, **kwargs) -> dict:
             result_kwargs["timeout"] = timeout
         return result_kwargs
 
+    def _default_keys_file(self, app: Optional[str], env: Optional[str] = None) -> Optional[str]:
+        def is_valid_app(app: Optional[str]) -> bool:  # noqa
+            return app and app.lower() in [name.lower() for name in ORCHESTRATED_APPS]
+        def infer_app_from_env(env: str) -> Optional[str]:  # noqa
+            if isinstance(env, str) and (lenv := env.lower()):
+                if app := [app for app in ORCHESTRATED_APPS if lenv.startswith(app.lower())]:
+                    return app[0]
+        if is_valid_app(app) or (app := infer_app_from_env(env)):
+            return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
+
     def _response(self, response) -> Optional[RequestResponse]:
         if response and isinstance(getattr(response.__class__, "json"), property):
             class RequestResponseWrapper:  # For consistency change json property to method.

From 58ab91437b36b6aea89e84fc4fb274d6ecf27ff9 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 11:33:50 -0500
Subject: [PATCH 22/45] portal_utils tests

---
 test/test_portal_utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/test_portal_utils.py b/test/test_portal_utils.py
index 0869802d2..b9e12dfc3 100644
--- a/test/test_portal_utils.py
+++ b/test/test_portal_utils.py
@@ -13,16 +13,20 @@
 def test_portal_constructor_a():
 
     def assert_for_server(server, expected):
+
         portal = Portal(_TEST_KEY, server=server) if server is not None else Portal(_TEST_KEY)
         assert portal.key_id == _TEST_KEY_ID
+        assert portal.secret == _TEST_SECRET
         assert portal.key_pair == _TEST_KEY_PAIR
         assert portal.server == expected
         assert portal.keys_file is None
         assert portal.app is None
         assert portal.env is None
         assert portal.vapp is None
+
         portal = Portal({**_TEST_KEY, "server": server}) if server is not None else Portal(_TEST_KEY)
         assert portal.key_id == _TEST_KEY_ID
+        assert portal.secret == _TEST_SECRET
         assert portal.key_pair == _TEST_KEY_PAIR
         assert portal.server == expected
         assert portal.keys_file is None
@@ -59,6 +63,7 @@ def test_portal_constructor_b():
 
         portal = Portal(keys_file, env="smaht-local")
         assert portal.key_id == "ABCDEFGHI"
+        assert portal.secret == "adfxdloiebvhzp"
         assert portal.key_pair == ("ABCDEFGHI", "adfxdloiebvhzp")
         assert portal.server == "http://localhost:8080"
         assert portal.key == {"key": "ABCDEFGHI", "secret": "adfxdloiebvhzp", "server": "http://localhost:8080"}
@@ -70,6 +75,7 @@ def test_portal_constructor_b():
 
         portal = Portal(keys_file, env="smaht-remote")
         assert portal.key_id == "GHIDEFABC"
+        assert portal.secret == "zpadfxdloiebvh"
         assert portal.key_pair == ("GHIDEFABC", "zpadfxdloiebvh")
         assert portal.server == "https://smaht.hms.harvard.edu"
         assert portal.key == {"key": "GHIDEFABC", "secret": "zpadfxdloiebvh", "server": "https://smaht.hms.harvard.edu"}
@@ -82,6 +88,7 @@ def test_portal_constructor_b():
         Portal.KEYS_FILE_DIRECTORY = os.path.dirname(keys_file)
         portal = Portal("smaht-local", app="smaht")
         assert portal.key_id == "ABCDEFGHI"
+        assert portal.secret == "adfxdloiebvhzp"
         assert portal.key_pair == ("ABCDEFGHI", "adfxdloiebvhzp")
         assert portal.server == "http://localhost:8080"
         assert portal.key == {"key": "ABCDEFGHI", "secret": "adfxdloiebvhzp", "server": "http://localhost:8080"}
@@ -121,6 +128,7 @@ def test_portal_constructor_c():
         assert portal.key == portal_copy.key
         assert portal.key_pair == portal_copy.key_pair
         assert portal.key_id == portal_copy.key_id
+        assert portal.secret == portal_copy.secret
         assert portal.keys_file == portal_copy.keys_file
         assert portal.env == portal_copy.env
         assert portal.server == portal_copy.server

From ce6a5bbf3584c1c491ff79049cb41aa3f5c0ca0c Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 14:09:05 -0500
Subject: [PATCH 23/45] portal_utils tests

---
 dcicutils/portal_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 51f03d4c2..4563b4d9b 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -220,7 +220,7 @@ def patch(self, uri: str, data: Optional[dict] = None,
               json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
         if self._vapp:
             return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
-        return requests.patch(self.url(uri), json=json or data, **self._kwargs(**kwargs))
+        return requests.patch(self.url(uri), data=json or data, **self._kwargs(**kwargs))
 
     def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
              files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
@@ -229,7 +229,7 @@ def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = Non
                 return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
             else:
                 return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return requests.post(self.url(uri), json=json or data, files=files, **self._kwargs(**kwargs))
+        return requests.post(self.url(uri), data=json or data, files=files, **self._kwargs(**kwargs))
 
     def get_schema(self, schema_name: str) -> Optional[dict]:
         return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)

From b63d4b294c36ce1a1686f2a2439729c8143b9821 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Mon, 18 Dec 2023 15:59:12 -0500
Subject: [PATCH 24/45] portal_utils tests

---
 dcicutils/portal_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 4563b4d9b..c88b34090 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -220,7 +220,7 @@ def patch(self, uri: str, data: Optional[dict] = None,
               json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
         if self._vapp:
             return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
-        return requests.patch(self.url(uri), data=json or data, **self._kwargs(**kwargs))
+        return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
 
     def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
              files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
@@ -229,7 +229,7 @@ def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = Non
                 return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
             else:
                 return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return requests.post(self.url(uri), data=json or data, files=files, **self._kwargs(**kwargs))
+        return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
 
     def get_schema(self, schema_name: str) -> Optional[dict]:
         return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)

From 14e9f9f6b2a43d5a7e79120764f6ec08875a091c Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 09:52:34 -0500
Subject: [PATCH 25/45] more refactoring of portal_utils

---
 dcicutils/portal_utils.py    | 154 ++++++++++++++++++++---------------
 dcicutils/structured_data.py |  11 +--
 2 files changed, 94 insertions(+), 71 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index c88b34090..5bf7a6031 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -1,13 +1,17 @@
 from collections import deque
 import io
 import json
-from pyramid.paster import get_app
-from pyramid.router import Router
+from pyramid.config import Configurator as PyramidConfigurator
+from pyramid.paster import get_app as pyramid_get_app
+from pyramid.response import Response as PyramidResponse
+from pyramid.router import Router as PyramidRouter
 import os
 import re
 import requests
 from requests.models import Response as RequestResponse
-from typing import Optional, Type, Union
+from typing import Callable, Dict, List, Optional, Type, Union
+from uuid import uuid4 as uuid
+# from waitress import serve
 from webtest.app import TestApp, TestResponse
 from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
@@ -37,7 +41,7 @@ class Portal:
     KEYS_FILE_DIRECTORY = os.path.expanduser(f"~")
 
     def __init__(self,
-                 arg: Optional[Union[Portal, TestApp, VirtualApp, Router, dict, tuple, str]] = None,
+                 arg: Optional[Union[Portal, TestApp, VirtualApp, PyramidRouter, dict, tuple, str]] = None,
                  env: Optional[str] = None, server: Optional[str] = None,
                  app: Optional[OrchestratedApp] = None) -> None:
 
@@ -69,14 +73,14 @@ def init_from_portal(portal: Portal, unspecified: Optional[list] = None) -> None
             self._app = portal._app
             self._vapp = portal._vapp
 
-        def init_from_vapp(vapp: Union[TestApp, VirtualApp, Router], unspecified: Optional[list] = []) -> None:
+        def init_from_vapp(vapp: Union[TestApp, VirtualApp, PyramidRouter], unspecified: Optional[list] = []) -> None:
             init(unspecified)
-            self._vapp = Portal._create_testapp(vapp)
+            self._vapp = Portal._create_vapp(vapp)
 
         def init_from_ini_file(ini_file: str, unspecified: Optional[list] = []) -> None:
             init(unspecified)
             self._ini_file = ini_file
-            self._vapp = Portal._create_testapp(ini_file)
+            self._vapp = Portal._create_vapp(ini_file)
 
         def init_from_key(key: dict, server: Optional[str], unspecified: Optional[list] = []) -> None:
             init(unspecified)
@@ -138,7 +142,7 @@ def normalize_server(server: str) -> Optional[str]:
 
         if isinstance(arg, Portal):
             init_from_portal(arg, unspecified=[env, server, app])
-        elif isinstance(arg, (TestApp, VirtualApp, Router)):
+        elif isinstance(arg, (TestApp, VirtualApp, PyramidRouter)):
             init_from_vapp(arg, unspecified=[env, server, app])
         elif isinstance(arg, str) and arg.endswith(".ini"):
             init_from_ini_file(arg, unspecified=[env, server, app])
@@ -201,35 +205,36 @@ def get_metadata(self, object_id: str) -> Optional[dict]:
     def patch_metadata(self, object_id: str, data: str) -> Optional[dict]:
         if self._key:
             return patch_metadata(obj_id=object_id, patch_item=data, key=self._key)
-        return self.patch(f"/{object_id}", data)
+        return self.patch(f"/{object_id}", data).json()
 
     def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
         if self._key:
             return post_metadata(schema_name=object_type, post_item=data, key=self._key)
-        return self.post(f"/{object_type}", data)
+        return self.post(f"/{object_type}", data).json()
 
     def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if self._vapp:
-            response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
-            if response and response.status_code in [301, 302, 303, 307, 308] and follow:
-                response = response.follow()
-            return self._response(response)
-        return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
+        if not self._vapp:
+            return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
+        response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
+        if response and response.status_code in [301, 302, 303, 307, 308] and follow:
+            response = response.follow()
+        return self._response(response)
 
     def patch(self, uri: str, data: Optional[dict] = None,
               json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if self._vapp:
-            return self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs))
-        return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
+        if not self._vapp:
+            return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
+        return self._response(self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs)))
 
     def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
              files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if self._vapp:
-            if files:
-                return self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-            else:
-                return self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
+        if not self._vapp:
+            return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
+        if files:
+            response = self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+        else:
+            response = self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
+        return self._response(response)
 
     def get_schema(self, schema_name: str) -> Optional[dict]:
         return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
@@ -311,7 +316,7 @@ def infer_app_from_env(env: str) -> Optional[str]:  # noqa
         if is_valid_app(app) or (app := infer_app_from_env(env)):
             return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
 
-    def _response(self, response) -> Optional[RequestResponse]:
+    def _response(self, response: TestResponse) -> Optional[RequestResponse]:
         if response and isinstance(getattr(response.__class__, "json"), property):
             class RequestResponseWrapper:  # For consistency change json property to method.
                 def __init__(self, response, **kwargs):
@@ -325,51 +330,72 @@ def json(self):  # noqa
         return response
 
     @staticmethod
-    def create_for_testing(ini_file: Optional[str] = None) -> Portal:
-        if isinstance(ini_file, str):
-            return Portal(Portal._create_testapp(ini_file))
-        minimal_ini_for_unit_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
-        with temporary_file(content=minimal_ini_for_unit_testing, suffix=".ini") as ini_file:
-            return Portal(Portal._create_testapp(ini_file))
-
-    @staticmethod
-    def create_for_testing_local(ini_file: Optional[str] = None) -> Portal:
-        if isinstance(ini_file, str) and ini_file:
-            return Portal(Portal._create_testapp(ini_file))
-        minimal_ini_for_testing_local = "\n".join([
-            "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
-            "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
-            "multiauth.groupfinder = encoded.authorization.smaht_groupfinder",
-            "multiauth.policies = auth0 session remoteuser accesskey",
-            "multiauth.policy.session.namespace = mailto",
-            "multiauth.policy.session.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.session.base = pyramid.authentication.SessionAuthenticationPolicy",
-            "multiauth.policy.remoteuser.namespace = remoteuser",
-            "multiauth.policy.remoteuser.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.remoteuser.base = pyramid.authentication.RemoteUserAuthenticationPolicy",
-            "multiauth.policy.accesskey.namespace = accesskey",
-            "multiauth.policy.accesskey.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.accesskey.base = encoded.authentication.BasicAuthAuthenticationPolicy",
-            "multiauth.policy.accesskey.check = encoded.authentication.basic_auth_check",
-            "multiauth.policy.auth0.use = encoded.authentication.NamespacedAuthenticationPolicy",
-            "multiauth.policy.auth0.namespace = auth0",
-            "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
-        ])
-        with temporary_file(content=minimal_ini_for_testing_local, suffix=".ini") as minimal_ini_file:
-            return Portal(Portal._create_testapp(minimal_ini_file))
+    def create_for_testing(arg: Optional[Union[str, bool, List[dict], dict, Callable]] = None) -> Portal:
+        if isinstance(arg, list) or isinstance(arg, dict) or isinstance(arg, Callable):
+            return Portal(Portal._create_router_for_testing(arg))
+        if isinstance(arg, str) and arg.endswith(".ini"):
+            return Portal(Portal._create_vapp(arg))
+        if arg == "local" or arg is True:
+            minimal_ini_for_testing = "\n".join([
+                "[app:app]\nuse = egg:encoded\nfile_upload_bucket = dummy",
+                "sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata",
+                "multiauth.groupfinder = encoded.authorization.smaht_groupfinder",
+                "multiauth.policies = auth0 session remoteuser accesskey",
+                "multiauth.policy.session.namespace = mailto",
+                "multiauth.policy.session.use = encoded.authentication.NamespacedAuthenticationPolicy",
+                "multiauth.policy.session.base = pyramid.authentication.SessionAuthenticationPolicy",
+                "multiauth.policy.remoteuser.namespace = remoteuser",
+                "multiauth.policy.remoteuser.use = encoded.authentication.NamespacedAuthenticationPolicy",
+                "multiauth.policy.remoteuser.base = pyramid.authentication.RemoteUserAuthenticationPolicy",
+                "multiauth.policy.accesskey.namespace = accesskey",
+                "multiauth.policy.accesskey.use = encoded.authentication.NamespacedAuthenticationPolicy",
+                "multiauth.policy.accesskey.base = encoded.authentication.BasicAuthAuthenticationPolicy",
+                "multiauth.policy.accesskey.check = encoded.authentication.basic_auth_check",
+                "multiauth.policy.auth0.use = encoded.authentication.NamespacedAuthenticationPolicy",
+                "multiauth.policy.auth0.namespace = auth0",
+                "multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy"
+            ])
+        else:
+            minimal_ini_for_testing = "[app:app]\nuse = egg:encoded\nsqlalchemy.url = postgresql://dummy\n"
+        with temporary_file(content=minimal_ini_for_testing, suffix=".ini") as ini_file:
+            return Portal(Portal._create_vapp(ini_file))
 
     @staticmethod
-    def _create_testapp(arg: Union[TestApp, VirtualApp, Router, str] = None, app_name: Optional[str] = None) -> TestApp:
+    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
         if isinstance(arg, TestApp):
             return arg
         elif isinstance(arg, VirtualApp):
             if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_testapp VirtualApp argument error.")
+                raise Exception("Portal._create_vapp VirtualApp argument error.")
             return arg.wrapped_app
-        if isinstance(arg, Router):
+        if isinstance(arg, PyramidRouter):
             router = arg
-        elif isinstance(arg, str) or arg is None:
-            router = get_app(arg or "development.ini", app_name or "app")
+        elif isinstance(arg, str) or not arg:
+            router = pyramid_get_app(arg or "development.ini", "app")
         else:
-            raise Exception("Portal._create_testapp argument error.")
+            raise Exception("Portal._create_vapp argument error.")
         return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
+
+    @staticmethod
+    def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Callable]]]] = None):
+        if isinstance(endpoints, dict):
+            endpoints = [endpoints]
+        elif isinstance(endpoints, Callable):
+            endpoints = [{"path": "/", "method": "GET", "function": endpoints}]
+        if not isinstance(endpoints, list) or not endpoints:
+            endpoints = [{"path": "/", "method": "GET", "function": lambda request: {"status": "OK"}}]
+        with PyramidConfigurator() as config:
+            nendpoints = 0
+            for endpoint in endpoints:
+                if (endpoint_path := endpoint.get("path")) and (endpoint_function := endpoint.get("function")):
+                    endpoint_method = endpoint.get("method", "GET")
+                    def endpoint_wrapper(request):
+                        response = endpoint_function(request)
+                        return PyramidResponse(json.dumps(response), content_type="application/json; charset=utf-8")
+                    endpoint_id = str(uuid())
+                    config.add_route(endpoint_id, endpoint_path)
+                    config.add_view(endpoint_wrapper, route_name=endpoint_id, request_method=endpoint_method)
+                    nendpoints += 1
+            if nendpoints == 0:
+                return Portal._create_router_for_testing([])
+            return config.make_wsgi_app()
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index 82acc4493..f13612653 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -6,7 +6,7 @@
 from pyramid.router import Router
 import re
 import sys
-from typing import Any, Callable, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
 from webtest.app import TestApp
 from dcicutils.common import OrchestratedApp
 from dcicutils.data_readers import CsvReader, Excel, RowReader
@@ -616,12 +616,9 @@ def _ref_exists_single(self, type_name: str, value: str) -> bool:
         return self.get_metadata(f"/{type_name}/{value}") is not None
 
     @staticmethod
-    def create_for_testing(ini_file: Optional[str] = None, schemas: Optional[List[dict]] = None) -> Portal:
-        return Portal(PortalBase.create_for_testing(ini_file), schemas=schemas)
-
-    @staticmethod
-    def create_for_testing_local(ini_file: Optional[str] = None, schemas: Optional[List[dict]] = None) -> Portal:
-        return Portal(PortalBase.create_for_testing_local(ini_file), schemas=schemas)
+    def create_for_testing(arg: Optional[Union[str, bool, List[dict], dict, Callable]] = None,
+                           schemas: Optional[List[dict]] = None) -> Portal:
+        return Portal(PortalBase.create_for_testing(arg), schemas=schemas)
 
 
 def _split_dotted_string(value: str):

From 02f7d0ba392b08cb5d7a7ac2ce361c58e0a9bf15 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 09:53:15 -0500
Subject: [PATCH 26/45] flake8

---
 dcicutils/portal_utils.py    | 2 +-
 dcicutils/structured_data.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 5bf7a6031..3ffda89b9 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -389,7 +389,7 @@ def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Cal
             for endpoint in endpoints:
                 if (endpoint_path := endpoint.get("path")) and (endpoint_function := endpoint.get("function")):
                     endpoint_method = endpoint.get("method", "GET")
-                    def endpoint_wrapper(request):
+                    def endpoint_wrapper(request):  # noqa
                         response = endpoint_function(request)
                         return PyramidResponse(json.dumps(response), content_type="application/json; charset=utf-8")
                     endpoint_id = str(uuid())
diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py
index f13612653..a889ee3a3 100644
--- a/dcicutils/structured_data.py
+++ b/dcicutils/structured_data.py
@@ -6,7 +6,7 @@
 from pyramid.router import Router
 import re
 import sys
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, List, Optional, Tuple, Type, Union
 from webtest.app import TestApp
 from dcicutils.common import OrchestratedApp
 from dcicutils.data_readers import CsvReader, Excel, RowReader

From 6a9569508deaf5edb78740023778c7d3bd1bcf38 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 09:57:50 -0500
Subject: [PATCH 27/45] version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index e252afde0..0798f592f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b6"  # TODO: To become 8.6.0
+version = "8.5.0.1b7"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From 60d6b751979b96adea1fe3eee6938285fcff648a Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 10:46:56 -0500
Subject: [PATCH 28/45] Add ability to actually start a Portal instance purely
 for test purposes.

---
 dcicutils/portal_utils.py | 17 +++++++++++++++--
 pyproject.toml            |  2 +-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 3ffda89b9..dc332fc87 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -9,10 +9,11 @@
 import re
 import requests
 from requests.models import Response as RequestResponse
+from threading import Thread
 from typing import Callable, Dict, List, Optional, Type, Union
 from uuid import uuid4 as uuid
-# from waitress import serve
 from webtest.app import TestApp, TestResponse
+from wsgiref.simple_server import make_server as wsgi_make_server
 from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
@@ -377,7 +378,7 @@ def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) ->
         return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
 
     @staticmethod
-    def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Callable]]]] = None):
+    def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Callable]]]] = None) -> PyramidRouter:
         if isinstance(endpoints, dict):
             endpoints = [endpoints]
         elif isinstance(endpoints, Callable):
@@ -399,3 +400,15 @@ def endpoint_wrapper(request):  # noqa
             if nendpoints == 0:
                 return Portal._create_router_for_testing([])
             return config.make_wsgi_app()
+
+    def start(self, port: int = 8080, asynchronous: bool = False) -> Optional[Thread]:
+        if isinstance(self._vapp, TestApp) and hasattr(self._vapp, "app") and isinstance(self._vapp.app, PyramidRouter):
+            def start_server() -> None:  # noqa
+                with wsgi_make_server('0.0.0.0', port, self._vapp.app) as server:
+                    server.serve_forever()
+            if asynchronous:
+                server_thread = Thread(target=start_server)
+                server_thread.daemon = True
+                server_thread.start()
+                return server_thread
+            start_server(self._vapp.app, port)
diff --git a/pyproject.toml b/pyproject.toml
index 0798f592f..13b2419e9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b7"  # TODO: To become 8.6.0
+version = "8.5.0.1b8"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From bbc1d31c8f1dce9d156e997b05f575e4bc9d3d43 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 10:51:17 -0500
Subject: [PATCH 29/45] Add ability to actually start a Portal instance purely
 for test purposes.

---
 dcicutils/portal_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index dc332fc87..9ad536fda 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -404,11 +404,11 @@ def endpoint_wrapper(request):  # noqa
     def start(self, port: int = 8080, asynchronous: bool = False) -> Optional[Thread]:
         if isinstance(self._vapp, TestApp) and hasattr(self._vapp, "app") and isinstance(self._vapp.app, PyramidRouter):
             def start_server() -> None:  # noqa
-                with wsgi_make_server('0.0.0.0', port, self._vapp.app) as server:
+                with wsgi_make_server("0.0.0.0", port, self._vapp.app) as server:
                     server.serve_forever()
             if asynchronous:
                 server_thread = Thread(target=start_server)
                 server_thread.daemon = True
                 server_thread.start()
                 return server_thread
-            start_server(self._vapp.app, port)
+            start_server()

From 58481ac33f5b76e9a3a64951b21ee652f6d953fa Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 10:56:04 -0500
Subject: [PATCH 30/45] Add ability to actually start a Portal instance purely
 for test purposes.

---
 dcicutils/portal_utils.py | 2 +-
 pyproject.toml            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 9ad536fda..df6625ce7 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -401,7 +401,7 @@ def endpoint_wrapper(request):  # noqa
                 return Portal._create_router_for_testing([])
             return config.make_wsgi_app()
 
-    def start(self, port: int = 8080, asynchronous: bool = False) -> Optional[Thread]:
+    def start_for_testing(self, port: int = 8080, asynchronous: bool = False) -> Optional[Thread]:
         if isinstance(self._vapp, TestApp) and hasattr(self._vapp, "app") and isinstance(self._vapp.app, PyramidRouter):
             def start_server() -> None:  # noqa
                 with wsgi_make_server("0.0.0.0", port, self._vapp.app) as server:
diff --git a/pyproject.toml b/pyproject.toml
index 13b2419e9..03c3f776a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b8"  # TODO: To become 8.6.0
+version = "8.5.0.1b9"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From 6f72a4e9f5653dd4f3f74ba6e9ca06735c50c0b4 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 14:28:49 -0500
Subject: [PATCH 31/45] typo

---
 dcicutils/portal_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index df6625ce7..b514a16c0 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -367,14 +367,14 @@ def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) ->
             return arg
         elif isinstance(arg, VirtualApp):
             if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_vapp VirtualApp argument error.")
+                raise Exception("Portal._create_vapp VirtualApp arguement error.")
             return arg.wrapped_app
         if isinstance(arg, PyramidRouter):
             router = arg
         elif isinstance(arg, str) or not arg:
             router = pyramid_get_app(arg or "development.ini", "app")
         else:
-            raise Exception("Portal._create_vapp argument error.")
+            raise Exception("Portal._create_vapp arguement error.")
         return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
 
     @staticmethod

From d59d35de7841e8c70fdafea2d4c4f82da2488bf1 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 14:29:29 -0500
Subject: [PATCH 32/45] typo

---
 dcicutils/portal_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index b514a16c0..df6625ce7 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -367,14 +367,14 @@ def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) ->
             return arg
         elif isinstance(arg, VirtualApp):
             if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_vapp VirtualApp arguement error.")
+                raise Exception("Portal._create_vapp VirtualApp argument error.")
             return arg.wrapped_app
         if isinstance(arg, PyramidRouter):
             router = arg
         elif isinstance(arg, str) or not arg:
             router = pyramid_get_app(arg or "development.ini", "app")
         else:
-            raise Exception("Portal._create_vapp arguement error.")
+            raise Exception("Portal._create_vapp argument error.")
         return TestApp(router, {"HTTP_ACCEPT": "application/json", "REMOTE_USER": "TEST"})
 
     @staticmethod

From 6294aa2957217160906d7185426345cfae7701af Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 15:12:15 -0500
Subject: [PATCH 33/45] minor portal_utils refactoring

---
 dcicutils/portal_utils.py | 81 ++++++++++++++++++++-------------------
 1 file changed, 42 insertions(+), 39 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index df6625ce7..3f84da3cd 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -8,7 +8,7 @@
 import os
 import re
 import requests
-from requests.models import Response as RequestResponse
+from requests.models import Response
 from threading import Thread
 from typing import Callable, Dict, List, Optional, Type, Union
 from uuid import uuid4 as uuid
@@ -20,6 +20,7 @@
 from dcicutils.zip_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
+OptionalResponse = Optional[Union[Response, TestResponse]]
 
 
 class Portal:
@@ -200,6 +201,29 @@ def app(self) -> Optional[str]:
     def vapp(self) -> Optional[TestApp]:
         return self._vapp
 
+    def get(self, url: str, follow: bool = True, **kwargs) -> OptionalResponse:
+        if not self._vapp:
+            return requests.get(self.url(url), allow_redirects=follow, **self._kwargs(**kwargs))
+        response = self._vapp.get(self.url(url), **self._kwargs(**kwargs))
+        if response and response.status_code in [301, 302, 303, 307, 308] and follow:
+            response = response.follow()
+        return self._response(response)
+
+    def patch(self, url: str, data: Optional[dict] = None, json: Optional[dict] = None, **kwargs) -> OptionalResponse:
+        if not self._vapp:
+            return requests.patch(self.url(url), data=data, json=json, **self._kwargs(**kwargs))
+        return self._response(self._vapp.patch_json(self.url(url), json or data, **self._kwargs(**kwargs)))
+
+    def post(self, url: str, data: Optional[dict] = None, json: Optional[dict] = None,
+             files: Optional[dict] = None, **kwargs) -> OptionalResponse:
+        if not self._vapp:
+            return requests.post(self.url(url), data=data, json=json, files=files, **self._kwargs(**kwargs))
+        if files:
+            response = self._vapp.post(self.url(url), json or data, upload_files=files, **self._kwargs(**kwargs))
+        else:
+            response = self._vapp.post_json(self.url(url), json or data, upload_files=files, **self._kwargs(**kwargs))
+        return self._response(response)
+
     def get_metadata(self, object_id: str) -> Optional[dict]:
         return get_metadata(obj_id=object_id, vapp=self._vapp, key=self._key)
 
@@ -213,29 +237,14 @@ def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
             return post_metadata(schema_name=object_type, post_item=data, key=self._key)
         return self.post(f"/{object_type}", data).json()
 
-    def get(self, uri: str, follow: bool = True, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if not self._vapp:
-            return requests.get(self.url(uri), allow_redirects=follow, **self._kwargs(**kwargs))
-        response = self._vapp.get(self.url(uri), **self._kwargs(**kwargs))
-        if response and response.status_code in [301, 302, 303, 307, 308] and follow:
-            response = response.follow()
-        return self._response(response)
+    def get_health(self) -> Optional[Union[Response, TestResponse]]:
+        return self.get("/health")
 
-    def patch(self, uri: str, data: Optional[dict] = None,
-              json: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if not self._vapp:
-            return requests.patch(self.url(uri), data=data, json=json, **self._kwargs(**kwargs))
-        return self._response(self._vapp.patch_json(self.url(uri), json or data, **self._kwargs(**kwargs)))
-
-    def post(self, uri: str, data: Optional[dict] = None, json: Optional[dict] = None,
-             files: Optional[dict] = None, **kwargs) -> Optional[Union[RequestResponse, TestResponse]]:
-        if not self._vapp:
-            return requests.post(self.url(uri), data=data, json=json, files=files, **self._kwargs(**kwargs))
-        if files:
-            response = self._vapp.post(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        else:
-            response = self._vapp.post_json(self.url(uri), json or data, upload_files=files, **self._kwargs(**kwargs))
-        return self._response(response)
+    def ping(self) -> bool:
+        try:
+            return self.get_health().status_code == 200
+        except Exception:
+            return False
 
     def get_schema(self, schema_name: str) -> Optional[dict]:
         return get_schema(self.schema_name(schema_name), portal_vapp=self._vapp, key=self._key)
@@ -283,20 +292,14 @@ def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
             super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
         return super_type_map_flattened
 
-    def ping(self) -> bool:
-        try:
-            return self.get("/health").status_code == 200
-        except Exception:
-            return False
-
-    def url(self, uri: str) -> str:
-        if not isinstance(uri, str) or not uri:
+    def url(self, url: str) -> str:
+        if not isinstance(url, str) or not url:
             return "/"
-        if (luri := uri.lower()).startswith("http://") or luri.startswith("https://"):
-            return uri
-        if not (uri := re.sub(r"/+", "/", uri)).startswith("/"):
-            uri = "/"
-        return self._server + uri if self._server else uri
+        if (lurl := url.lower()).startswith("http://") or lurl.startswith("https://"):
+            return url
+        if not (url := re.sub(r"/+", "/", url)).startswith("/"):
+            url = "/"
+        return self._server + url if self._server else url
 
     def _kwargs(self, **kwargs) -> dict:
         result_kwargs = {"headers":
@@ -317,9 +320,9 @@ def infer_app_from_env(env: str) -> Optional[str]:  # noqa
         if is_valid_app(app) or (app := infer_app_from_env(env)):
             return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
 
-    def _response(self, response: TestResponse) -> Optional[RequestResponse]:
+    def _response(self, response: TestResponse) -> Optional[Response]:
         if response and isinstance(getattr(response.__class__, "json"), property):
-            class RequestResponseWrapper:  # For consistency change json property to method.
+            class TestResponseWrapper(TestResponse):  # For consistency change json property to method.
                 def __init__(self, response, **kwargs):
                     super().__init__(**kwargs)
                     self._response = response
@@ -327,7 +330,7 @@ def __getattr__(self, attr):  # noqa
                     return getattr(self._response, attr)
                 def json(self):  # noqa
                     return self._response.json
-            response = RequestResponseWrapper(response)
+            response = TestResponseWrapper(response)
         return response
 
     @staticmethod

From 8dee26adf91960de91ffc3ca149363124a3cdef0 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 15:17:22 -0500
Subject: [PATCH 34/45] minor portal_utils refactoring

---
 dcicutils/portal_utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 3f84da3cd..d95d6ca19 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -268,7 +268,7 @@ def get_schemas_super_type_map(self) -> dict:
         This is a dictionary of all types which have (one or more) sub-types whose value is
         an array of all of those sub-types (direct and all descendents), in breadth first order.
         """
-        def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
+        def list_breadth_first(super_type_map: dict, super_type_name: str) -> dict:
             result = []
             queue = deque(super_type_map.get(super_type_name, []))
             while queue:
@@ -289,7 +289,7 @@ def breadth_first(super_type_map: dict, super_type_name: str) -> dict:
                         super_type_map[super_type_name].append(type_name)
         super_type_map_flattened = {}
         for super_type_name in super_type_map:
-            super_type_map_flattened[super_type_name] = breadth_first(super_type_map, super_type_name)
+            super_type_map_flattened[super_type_name] = list_breadth_first(super_type_map, super_type_name)
         return super_type_map_flattened
 
     def url(self, url: str) -> str:
@@ -320,7 +320,7 @@ def infer_app_from_env(env: str) -> Optional[str]:  # noqa
         if is_valid_app(app) or (app := infer_app_from_env(env)):
             return os.path.join(Portal.KEYS_FILE_DIRECTORY, f".{app.lower()}-keys.json")
 
-    def _response(self, response: TestResponse) -> Optional[Response]:
+    def _response(self, response: TestResponse) -> TestResponse:
         if response and isinstance(getattr(response.__class__, "json"), property):
             class TestResponseWrapper(TestResponse):  # For consistency change json property to method.
                 def __init__(self, response, **kwargs):

From ef65f7ce2d4ec73f5c1bc97f605af555806b1aeb Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 15:17:52 -0500
Subject: [PATCH 35/45] minor portal_utils refactoring

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 03c3f776a..f99d35066 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b9"  # TODO: To become 8.6.0
+version = "8.5.0.1b10"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From f22b955e810762e1da43fc01a0dd3c1dad787c26 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 19 Dec 2023 19:21:53 -0500
Subject: [PATCH 36/45] portal_utils refactoring

---
 dcicutils/portal_utils.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index d95d6ca19..c98a0b724 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -28,15 +28,15 @@ class Portal:
     This is meant to be an uber wrapper for Portal access. It can be created in a variety of ways:
     1. From a (Portal) .ini file (e.g. development.ini)
     2. From a key dictionary, containing "key" and "secret" property values.
-    3. From a key tuple, containing (in order) a key and secret values.
+    3. From a key pair tuple, containing (in order) a key and secret values.
     4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
        or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
        and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
        if an "app" value is not specified but the given "env" value begins with one of the app values then that value
-       will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
+       will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than app is assumed to be "smaht".
     5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
        is looked up via the given "server" name and the "server" key dictionary value in the key file.
-    6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
+    6. From a given "vapp" value (which may be a webtest/TestApp or VirtualApp or even a pyramid/Router).
     7. From another Portal object; or from a a pyramid Router object.
     """
     FILE_SCHEMA_NAME = "File"
@@ -237,7 +237,7 @@ def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
             return post_metadata(schema_name=object_type, post_item=data, key=self._key)
         return self.post(f"/{object_type}", data).json()
 
-    def get_health(self) -> Optional[Union[Response, TestResponse]]:
+    def get_health(self) -> OptionalResponse:
         return self.get("/health")
 
     def ping(self) -> bool:
@@ -254,7 +254,7 @@ def get_schemas(self) -> dict:
 
     @staticmethod
     def schema_name(name: str) -> str:
-        return to_camel_case(name)
+        return to_camel_case(name if not name.endswith(".json") else name[:-5]) if isinstance(name, str) else ""
 
     def is_file_schema(self, schema_name: str) -> bool:
         if super_type_map := self.get_schemas_super_type_map():
@@ -404,10 +404,10 @@ def endpoint_wrapper(request):  # noqa
                 return Portal._create_router_for_testing([])
             return config.make_wsgi_app()
 
-    def start_for_testing(self, port: int = 8080, asynchronous: bool = False) -> Optional[Thread]:
+    def start_for_testing(self, port: int = 7070, asynchronous: bool = False) -> Optional[Thread]:
         if isinstance(self._vapp, TestApp) and hasattr(self._vapp, "app") and isinstance(self._vapp.app, PyramidRouter):
             def start_server() -> None:  # noqa
-                with wsgi_make_server("0.0.0.0", port, self._vapp.app) as server:
+                with wsgi_make_server("0.0.0.0", port or 7070, self._vapp.app) as server:
                     server.serve_forever()
             if asynchronous:
                 server_thread = Thread(target=start_server)

From 81efca98b6e384d695fcdd65c937ec81e960c544 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Wed, 20 Dec 2023 06:26:21 -0500
Subject: [PATCH 37/45] minor fix to portal_utils for server

---
 dcicutils/portal_utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index c98a0b724..2d35cfdee 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -115,8 +115,8 @@ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str
                 init_from_key(key, server)
                 self._keys_file = keys_file
                 self._env = env
-            elif isinstance(server, str) and server and (key := [k for k in keys if keys[k].get("server") == server]):
-                init_from_key(key, server)
+            elif isinstance(server, str) and server and (key := [keys[k] for k in keys if keys[k].get("server") == server]):
+                init_from_key(key[0], server)
                 self._keys_file = keys_file
             elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:
                 init_from_key(key, server)
@@ -156,7 +156,7 @@ def normalize_server(server: str) -> Optional[str]:
             init_from_keys_file(arg, env, server, unspecified=[app])
         elif isinstance(arg, str) and arg:
             init_from_env_server_app(arg, server, app, unspecified=[env])
-        elif isinstance(env, str) and env:
+        elif (isinstance(env, str) and env) or (isinstance(server, str) and server):
             init_from_env_server_app(env, server, app, unspecified=[arg])
         else:
             raise Exception("Portal init error; invalid args.")

From 4c60b3e0a72a7e4c9445a21eb22dc85f5c29476f Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Wed, 20 Dec 2023 07:01:06 -0500
Subject: [PATCH 38/45] flake8

---
 dcicutils/portal_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index 2d35cfdee..e71039e57 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -115,7 +115,8 @@ def init_from_keys_file(keys_file: str, env: Optional[str], server: Optional[str
                 init_from_key(key, server)
                 self._keys_file = keys_file
                 self._env = env
-            elif isinstance(server, str) and server and (key := [keys[k] for k in keys if keys[k].get("server") == server]):
+            elif (isinstance(server, str) and (server := normalize_server(server)) and
+                  (key := [keys[k] for k in keys if normalize_server(keys[k].get("server")) == server])):
                 init_from_key(key[0], server)
                 self._keys_file = keys_file
             elif len(keys) == 1 and (env := next(iter(keys))) and isinstance(key := keys[env], dict) and key:

From ff33ae9ad93836033fe863f889987b63b3e8ef07 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Sat, 23 Dec 2023 12:42:51 -0500
Subject: [PATCH 39/45] Refactor out functions in zip_utils to tmpfile_utils

---
 dcicutils/misc_utils.py    | 14 ++++++++++++++
 dcicutils/portal_utils.py  |  2 +-
 dcicutils/tmpfile_utils.py | 36 ++++++++++++++++++++++++++++++++++++
 dcicutils/zip_utils.py     | 36 ++----------------------------------
 test/test_misc_utils.py    | 12 +++++++++++-
 5 files changed, 64 insertions(+), 36 deletions(-)
 create mode 100644 dcicutils/tmpfile_utils.py

diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py
index 307c66ca3..27d0e1cc7 100644
--- a/dcicutils/misc_utils.py
+++ b/dcicutils/misc_utils.py
@@ -2190,6 +2190,20 @@ def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]]
     return target
 
 
+def load_json_from_file_expanding_environment_variables(file: str) -> Union[dict, list]:
+    def expand_envronment_variables(data):  # noqa
+        if isinstance(data, dict):
+            return {key: expand_envronment_variables(value) for key, value in data.items()}
+        if isinstance(data, list):
+            return [expand_envronment_variables(element) for element in data]
+        if isinstance(data, str):
+            return re.sub(r"\$\{([^}]+)\}|\$([a-zA-Z_][a-zA-Z0-9_]*)",
+                          lambda match: os.environ.get(match.group(1) or match.group(2), match.group(0)), data)
+        return data
+    with open(file, "r") as file:
+        return expand_envronment_variables(json.load(file))
+
+
 # Stealing topological sort classes below from python's graphlib module introduced
 # in v3.9 with minor refactoring.
 # Source: https://github.com/python/cpython/blob/3.11/Lib/graphlib.py
diff --git a/dcicutils/portal_utils.py b/dcicutils/portal_utils.py
index e71039e57..2856eb889 100644
--- a/dcicutils/portal_utils.py
+++ b/dcicutils/portal_utils.py
@@ -17,7 +17,7 @@
 from dcicutils.common import OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
-from dcicutils.zip_utils import temporary_file
+from dcicutils.tmpfile_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
 OptionalResponse = Optional[Union[Response, TestResponse]]
diff --git a/dcicutils/tmpfile_utils.py b/dcicutils/tmpfile_utils.py
new file mode 100644
index 000000000..ba1652cb6
--- /dev/null
+++ b/dcicutils/tmpfile_utils.py
@@ -0,0 +1,36 @@
+from contextlib import contextmanager
+import os
+import shutil
+import tempfile
+from typing import List, Optional, Union
+
+
+@contextmanager
+def temporary_directory() -> str:
+    try:
+        with tempfile.TemporaryDirectory() as tmp_directory_name:
+            yield tmp_directory_name
+    finally:
+        remove_temporary_directory(tmp_directory_name)
+
+
+@contextmanager
+def temporary_file(name: Optional[str] = None, suffix: Optional[str] = None,
+                   content: Optional[Union[str, bytes, List[str]]] = None) -> str:
+    with temporary_directory() as tmp_directory_name:
+        tmp_file_name = os.path.join(tmp_directory_name, name or tempfile.mktemp(dir="")) + (suffix or "")
+        if content is not None:
+            with open(tmp_file_name, "wb" if isinstance(content, bytes) else "w") as tmp_file:
+                tmp_file.write("\n".join(content) if isinstance(content, list) else content)
+        yield tmp_file_name
+
+
+def remove_temporary_directory(tmp_directory_name: str) -> None:
+    def is_temporary_directory(path: str) -> bool:
+        try:
+            tmpdir = tempfile.gettempdir()
+            return os.path.commonpath([path, tmpdir]) == tmpdir and os.path.exists(path) and os.path.isdir(path)
+        except Exception:
+            return False
+    if is_temporary_directory(tmp_directory_name):  # Guard against errant deletion.
+        shutil.rmtree(tmp_directory_name)
diff --git a/dcicutils/zip_utils.py b/dcicutils/zip_utils.py
index e98e6ac70..8a65f8abe 100644
--- a/dcicutils/zip_utils.py
+++ b/dcicutils/zip_utils.py
@@ -1,10 +1,9 @@
 from contextlib import contextmanager
+from dcicutils.tmpfile_utils import temporary_directory, temporary_file
 import gzip
 import os
-import shutil
 import tarfile
-import tempfile
-from typing import List, Optional, Union
+from typing import List, Optional
 import zipfile
 
 
@@ -46,34 +45,3 @@ def unpack_gz_file_to_temporary_file(file: str, suffix: Optional[str] = None) ->
                     outputf.write(inputf.read())
                     outputf.close()
                     yield tmp_file_name
-
-
-@contextmanager
-def temporary_directory() -> str:
-    try:
-        with tempfile.TemporaryDirectory() as tmp_directory_name:
-            yield tmp_directory_name
-    finally:
-        remove_temporary_directory(tmp_directory_name)
-
-
-@contextmanager
-def temporary_file(name: Optional[str] = None, suffix: Optional[str] = None,
-                   content: Optional[Union[str, bytes, List[str]]] = None) -> str:
-    with temporary_directory() as tmp_directory_name:
-        tmp_file_name = os.path.join(tmp_directory_name, name or tempfile.mktemp(dir="")) + (suffix or "")
-        if content is not None:
-            with open(tmp_file_name, "wb" if isinstance(content, bytes) else "w") as tmp_file:
-                tmp_file.write("\n".join(content) if isinstance(content, list) else content)
-        yield tmp_file_name
-
-
-def remove_temporary_directory(tmp_directory_name: str) -> None:
-    def is_temporary_directory(path: str) -> bool:
-        try:
-            tmpdir = tempfile.gettempdir()
-            return os.path.commonpath([path, tmpdir]) == tmpdir and os.path.exists(path) and os.path.isdir(path)
-        except Exception:
-            return False
-    if is_temporary_directory(tmp_directory_name):  # Guard against errant deletion.
-        shutil.rmtree(tmp_directory_name)
diff --git a/test/test_misc_utils.py b/test/test_misc_utils.py
index fabf661f6..c3f5f0ba3 100644
--- a/test/test_misc_utils.py
+++ b/test/test_misc_utils.py
@@ -31,12 +31,14 @@
     ObsoleteError, CycleError, TopologicalSorter, keys_and_values_to_dict, dict_to_keys_and_values, is_c4_arn,
     deduplicate_list, chunked, parse_in_radix, format_in_radix, managed_property, future_datetime,
     MIN_DATETIME, MIN_DATETIME_UTC, INPUT, builtin_print, map_chunked, to_camel_case, json_file_contents,
-    pad_to, JsonLinesReader, split_string, merge_objects, to_integer
+    pad_to, JsonLinesReader, split_string, merge_objects, to_integer,
+    load_json_from_file_expanding_environment_variables
 )
 from dcicutils.qa_utils import (
     Occasionally, ControlledTime, override_environ as qa_override_environ, MockFileSystem, printed_output,
     raises_regexp, MockId, MockLog, input_series,
 )
+from dcicutils.tmpfile_utils import temporary_file
 from typing import Any, Dict, List
 from unittest import mock
 
@@ -3691,3 +3693,11 @@ def test_to_integer():
     assert to_integer("0.0") == 0
     assert to_integer("asdf") is None
     assert to_integer("asdf", "myfallback") == "myfallback"
+
+
+def test_load_json_from_file_expanding_environment_variables():
+    with mock.patch.object(os, "environ", {"Auth0Secret": "dgakjhdgretqobv", "SomeEnvVar": "xyzzy"}):
+        some_json = {"Auth0Secret": "${Auth0Secret}", "abc": "def", "someproperty": "$SomeEnvVar"}
+        with temporary_file(content=json.dumps(some_json), suffix=".json") as tmpfile:
+            expanded_json = load_json_from_file_expanding_environment_variables(tmpfile)
+            assert expanded_json == {"Auth0Secret": "dgakjhdgretqobv", "abc": "def", "someproperty": "xyzzy"}

From 24e55346776b9842b9b6d13dc62afc2cfb41a565 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Sat, 23 Dec 2023 12:43:26 -0500
Subject: [PATCH 40/45] Refactor out functions in zip_utils to tmpfile_utils

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index f99d35066..52f3ebe09 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b10"  # TODO: To become 8.6.0
+version = "8.5.0.1b11"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From d346e75d85a9117d886394d8baa1fa34a9e091e1 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Sat, 23 Dec 2023 12:44:57 -0500
Subject: [PATCH 41/45] Refactor out functions in zip_utils to tmpfile_utils

---
 docs/source/dcicutils.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/source/dcicutils.rst b/docs/source/dcicutils.rst
index 37da22490..470e11828 100644
--- a/docs/source/dcicutils.rst
+++ b/docs/source/dcicutils.rst
@@ -337,6 +337,13 @@ task_utils
   :members:
 
 
+tmpfile_utils
+^^^^^^^^^^^^^
+
+.. automodule:: dcicutils.tmpfile_utils
+   :members:
+
+
 trace_utils
 ^^^^^^^^^^^
 

From 09bad08f58f7bb70805661b105551c7e6c7a8fd6 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 2 Jan 2024 08:57:41 -0500
Subject: [PATCH 42/45] comment update

---
 dcicutils/misc_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py
index 27d0e1cc7..922e5de5d 100644
--- a/dcicutils/misc_utils.py
+++ b/dcicutils/misc_utils.py
@@ -1472,6 +1472,7 @@ def string_list(s):
 def split_string(value: str, delimiter: str, escape: Optional[str] = None, unique: bool = False) -> List[str]:
     """
     Splits the given string into an array of string based on the given delimiter, and an optional escape character.
+    If the given unique flag is True then duplicate values will not be included.
     """
     if not isinstance(value, str) or not (value := value.strip()):
         return []

From 0c26943c319cd5faf7baff4cfd3a85b682e31aa7 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 2 Jan 2024 08:58:30 -0500
Subject: [PATCH 43/45] version update

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 52f3ebe09..c63c2cfc9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b11"  # TODO: To become 8.6.0
+version = "8.5.0.1b12"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

From 13414d9ab1c5bc25da0dcd46454dff48b81bf291 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 2 Jan 2024 09:05:17 -0500
Subject: [PATCH 44/45] Copyright update to 2024

---
 LICENSE.txt                | 2 +-
 pyproject.toml             | 2 +-
 test/test_license_utils.py | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index dbd9eb7db..e7d2f54ef 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright 2017-2023 President and Fellows of Harvard College
+Copyright 2017-2024 President and Fellows of Harvard College
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/pyproject.toml b/pyproject.toml
index c63c2cfc9..a0214ed72 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b12"  # TODO: To become 8.6.0
+version = "8.5.0.1b13"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"
diff --git a/test/test_license_utils.py b/test/test_license_utils.py
index e0d0fdc25..80fa64a62 100644
--- a/test/test_license_utils.py
+++ b/test/test_license_utils.py
@@ -759,13 +759,13 @@ def mocked_license_logger(message):
 
             # Test that with no analysis argument, problems get sent out as warnings
             LicenseFileParser.validate_simple_license_file(filename='LICENSE.txt')
-            assert license_warnings == ["The copyright year, '2020', should have '2023' at the end."]
+            assert license_warnings == ["The copyright year, '2020', should have '2024' at the end."]
 
             # Test that with an analysis argument, problems get summarized to that object
             analysis = LicenseAnalysis()
             license_warnings = []
             LicenseFileParser.validate_simple_license_file(filename='LICENSE.txt', analysis=analysis)
-            assert analysis.miscellaneous == ["The copyright year, '2020', should have '2023' at the end."]
+            assert analysis.miscellaneous == ["The copyright year, '2020', should have '2024' at the end."]
             assert license_warnings == []
 
 

From a209bc43b62b93354546caf1c29544931af48a71 Mon Sep 17 00:00:00 2001
From: David Michaels <david_michaels@hms.harvard.edu>
Date: Tue, 2 Jan 2024 10:48:52 -0500
Subject: [PATCH 45/45] version bump - ready to merge PR-295 to master

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index a0214ed72..e7fa23cfc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b13"  # TODO: To become 8.6.0
+version = "8.6.0"
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"