Updated dcicutils to 8.6.0 (with minor fixes related to structured_da…

CHANGELOG.rst

@@ -6,6 +6,12 @@ snovault
 Change Log
 ----------
 
+
+11.7.0
+======
+* Updated dcicutils to 8.6.0 (with minor fixes related to structured_data and SMaHT ingestion).
+
+
 11.6.0
 ======
 * Updated dcicutils to 8.4.1 (with structured_data).

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicsnovault"
-version = "11.6.0"
+version = "11.7.0"
 description = "Storage support for 4DN Data Portals."
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -43,7 +43,7 @@ boto3 = ">=1.28.62"  # no particular version required, but this speeds up search
 botocore = ">=1.31.62"  # no particular version required, but this speeds up search
 elasticsearch = "7.13.4"  # versions >= 7.14.0 lock out AWS ES
 elasticsearch_dsl = "^7.4.0"
-dcicutils = "^8.5.0"
+dcicutils = "^8.6.0"
 future = ">=0.15.2,<1"
 html5lib = ">=1.1"  # experimental, should be OK now that we're not using moto server
 humanfriendly = "^1.44.9"

snovault/commands/generate_local_access_key.py

@@ -71,14 +71,14 @@
 import requests
 from typing import Optional, Tuple
 import uuid
-from webtest import TestApp
 from dcicutils.common import AnyJsonData
+from dcicutils.portal_utils import Portal
 from snovault.authentication import (
     generate_password as generate_access_key_secret,
     generate_user as generate_access_key
 )
 from snovault.edw_hash import EDWHash
-from snovault.loadxl import create_testapp, load_all
+from snovault.loadxl import load_all
 from ..project_app import app_project
 from .captured_output import captured_output
 
@@ -137,16 +137,16 @@ def main() -> None:
         # multiauth.policy.auth0.namespace = auth0
         # multiauth.policy.auth0.base = encoded.authentication.Auth0AuthenticationPolicy
         with captured_output(not args.debug):
-            app = create_testapp(args.ini)
+            portal = Portal(args.ini)
     else:
-        app = None
+        portal = None
 
     if args.list:
-        _print_all_access_keys(app, args.verbose)
+        _print_all_access_keys(portal, args.verbose)
         return
 
     print(f"Creating a new local portal access-key for {args.app} ... ", end="")
-    access_key_user_uuid = _generate_user_uuid(args.user, app)
+    access_key_user_uuid = _generate_user_uuid(args.user, portal)
     access_key_id, access_key_secret, access_key_secret_hash = _generate_access_key(args.ini)
     access_key_inserts_file_item = _generate_access_key_inserts_item(access_key_id, access_key_secret_hash, access_key_user_uuid)
     access_keys_file_item = _generate_access_keys_file_item(access_key_id, access_key_secret, args.port)
@@ -173,16 +173,16 @@ def main() -> None:
             _exit_without_action(f"Portal must be running locally ({_get_local_portal_url(args.port)}) to do an insert.")
         print(f"Writing new local portal access-key to locally running portal database ... ", end="")
         with captured_output(not args.debug):
-            _load_data(app, access_key_inserts_file_item, data_type="access_key")
+            _load_data(portal, access_key_inserts_file_item, data_type="access_key")
         print("Done.")
     if not args.update_database or args.verbose:
         print(f"New local portal access-key insert record suitable for: {_INSERTS_DIR}/access_key.json ...")
         print(json.dumps(access_key_inserts_file_item, indent=4))
 
 
-def _generate_user_uuid(user: Optional[str], app: TestApp = None) -> str:
+def _generate_user_uuid(user: Optional[str], portal: Optional[Portal] = None) -> str:
     if not user:
-        if app:
+        if portal:
             _exit_without_action(f"The --user option must specify a UUID or email in {_USER_INSERTS_FILE}")
         return "<your-user-uuid>"
     user_uuid = None
@@ -195,8 +195,8 @@ def _generate_user_uuid(user: Optional[str], app: TestApp = None) -> str:
                 _exit_without_action(f"The given user ({user}) was not found as an email"
                                      f" in: {_USER_INSERTS_FILE}; and it is not a UUID.")
             user_uuid = user_uuid_from_inserts[0]["uuid"]
-    if user_uuid and app:
-        user = app.get_with_follow(f"/{user_uuid}", raise_exception=False)
+    if user_uuid and portal:
+        user = portal.get(f"/{user_uuid}", raise_exception=False)
         if not user or user.status_code != 200:
             _exit_without_action(f"The given user ({user_uuid}) was not found in the locally running portal database.")
     return user_uuid
@@ -280,21 +280,21 @@ def _get_local_portal_url(port: int) -> None:
     return f"http://localhost:{port}"
 
 
-def _load_data(app: TestApp, data: AnyJsonData, data_type: str) -> bool:
+def _load_data(portal: Portal, data: AnyJsonData, data_type: str) -> bool:
     if isinstance(data, dict):
         data = [data]
     elif not isinstance(data, list):
         return False
     if not data_type:
         return False
     data = {data_type: data}
-    load_all(app, inserts=data, docsdir=None, overwrite=True, itype=[data_type], from_json=True)
+    load_all(portal.vapp, inserts=data, docsdir=None, overwrite=True, itype=[data_type], from_json=True)
     return True
 
 
-def _print_all_access_keys(app: TestApp, verbose: bool = False) -> None:
+def _print_all_access_keys(portal: Portal, verbose: bool = False) -> None:
     print("All access-keys defined for locally running portal:")
-    for item in _get_all_access_keys(app):
+    for item in _get_all_access_keys(portal):
         print(f"{item.id}", end="")
         if item.created:
             print(f" | Created: {item.created}", end="")
@@ -305,11 +305,11 @@ def _print_all_access_keys(app: TestApp, verbose: bool = False) -> None:
         print()
 
 
-def _get_all_access_keys(app: TestApp) -> list:
+def _get_all_access_keys(portal: Portal) -> list:
     response = []
     try:
         AccessKey = namedtuple("AccessKey", ["id", "uuid", "created", "expires"])
-        for access_key in app.get_with_follow(f"/access-keys").json["@graph"]:
+        for access_key in portal.get(f"/access-keys").json()["@graph"]:
             response.append(AccessKey(
                 id=access_key.get("access_key_id"),
                 uuid=access_key.get("uuid"),

snovault/commands/view_local_object.py

@@ -86,7 +86,7 @@
 from typing import Optional
 import yaml
 from dcicutils.misc_utils import get_error_message
-from snovault.loadxl import create_testapp
+from dcicutils.portal_utils import Portal
 from snovault.commands.captured_output import captured_output, uncaptured_output
 
 
@@ -98,33 +98,44 @@ def main():
     parser = argparse.ArgumentParser(description="Create local portal access-key for dev/testing purposes.")
     parser.add_argument("uuid", type=str,
                         help=f"The uuid (or path) of the object to fetch and view. ")
-    parser.add_argument("--ini", type=str, required=False, default=_DEFAULT_INI_FILE,
+    parser.add_argument("--ini", type=str, required=False, default=None,
                         help=f"Name of the application .ini file; default is: {_DEFAULT_INI_FILE}")
+    parser.add_argument("--env", type=str, required=False, default=None,
+                        help=f"Environment name (key from ~/.smaht-keys.json).")
+    parser.add_argument("--server", type=str, required=False, default=None,
+                        help=f"Environment server name (server from key in ~/.smaht-keys.json).")
+    parser.add_argument("--app", type=str, required=False, default=None,
+                        help=f"Application name (one of: smaht, cgap, fourfront).")
     parser.add_argument("--yaml", action="store_true", required=False, default=False, help="YAML output.")
     parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
     parser.add_argument("--debug", action="store_true", required=False, default=False, help="Debugging output.")
     args = parser.parse_args()
 
-    data = _get_local_object(uuid=args.uuid, ini=args.ini, verbose=args.verbose, debug=args.debug)
+    portal = _create_portal(ini=args.ini, env=args.env, server=args.server, app=args.app, debug=args.debug)
+    data = _get_local_object(portal=portal, uuid=args.uuid, verbose=args.verbose)
 
     if args.yaml:
         _print(yaml.dump(data))
     else:
         _print(json.dumps(data, default=str, indent=4))
 
 
-def _get_local_object(uuid: str, ini: str = _DEFAULT_INI_FILE, verbose: bool = False, debug: bool = False) -> dict:
+def _create_portal(ini: str = _DEFAULT_INI_FILE, env: Optional[str] = None,
+                   server: Optional[str] = None, app: Optional[str] = None, debug: bool = False) -> dict:
+    with captured_output(not debug):
+        return Portal(env, server=server, app=app) if env or app else Portal(ini or _DEFAULT_INI_FILE)
+
+
+def _get_local_object(portal: Portal, uuid: str, verbose: bool = False) -> dict:
     if verbose:
         _print(f"Getting object ({uuid}) from local portal ... ", end="")
     response = None
     try:
-        with captured_output(not debug):
-            app = create_testapp(ini)
-            if not uuid.startswith("/"):
-                path = f"/{uuid}"
-            else:
-                path = uuid
-            response = app.get_with_follow(path)
+        if not uuid.startswith("/"):
+            path = f"/{uuid}"
+        else:
+            path = uuid
+        response = portal.get(path)
     except Exception as e:
         if "404" in str(e) and "not found" in str(e).lower():
             if verbose:
@@ -141,7 +152,7 @@ def _get_local_object(uuid: str, ini: str = _DEFAULT_INI_FILE, verbose: bool = F
         _exit_without_action(f"Invalid JSON getting object {uuid}).")
     if verbose:
         _print("OK")
-    return response.json
+    return response.json()
 
 
 def _print(*args, **kwargs):