Merge pull request #106 from 4dn-dcic/ajs_upd_get_schema_fxns

Ajs upd get schema fxns

CHANGELOG.rst

@@ -6,6 +6,17 @@ dcicwrangling
 Change Log
 ----------
 
+
+2.3.0
+=====
+
+`PR:106 update get_schemas calls <https://github.com/4dn-dcic/dcicwrangling/pull/106>_`
+
+* update to use get_schemas function from dcicutils rather than having a 'broken' redundant copy
+* same for dump_json_data function
+* updated notebooks to call dcicutils versions of functions
+* simplify get_schemas_names_and_fields to use dcicutil function
+
 2.2.1
 =====
 

functions/notebook_functions.py

@@ -1,6 +1,7 @@
 from dcicutils import ff_utils
 from uuid import UUID
 import os
+import sys
 import json
 import openpyxl
 import warnings  # to suppress openpxl warning about headers
@@ -310,26 +311,13 @@ def find_uuids(val):
     return vals
 
 
-def get_schema_names(con_key):
-    schema_name = {}
-    profiles = ff_utils.get_metadata('/profiles/', key=con_key, add_on='frame=raw')
-    for key, value in profiles.items():
-        try:
-            schema_name[key] = value['id'].split('/')[-1][:-5]
-        except:
-            continue
-    return schema_name
-
-
 def get_schema_names_and_fields(con_key):
     '''Gets concrete item types from profiles and returns a dict of schema
     names, with properties and property type (including if array_linkTo)'''
     schemas = {}
-    profiles = ff_utils.get_metadata('/profiles/', key=con_key, add_on='frame=raw')
+    profiles = ff_utils.get_schemas(key=con_key, allow_abstract=False, require_id=True)
     for item in profiles.values():
-        if item['isAbstract'] is True:
-            continue
-        schema_name = item['id'].split('/')[-1][:-5]
+        schema_name = item['$id'].split('/')[-1][:-5]
         schemas[schema_name] = {}
         for field, content in item['properties'].items():
             field_type = content['type']
@@ -339,15 +327,6 @@ def get_schema_names_and_fields(con_key):
     return schemas
 
 
-def dump_results_to_json(store, folder):
-    if not os.path.exists(folder):
-        os.makedirs(folder)
-    for a_type in store:
-        filename = folder + '/' + a_type + '.json'
-        with open(filename, 'w') as outfile:
-            json.dump(store[a_type], outfile, indent=4)
-
-
 def printTable(myDict, colList=None):
     """ Pretty print a list of dictionaries Author: Thierry Husson"""
     if not colList:

notebooks/useful_notebooks/03_find_and_transfer_env.ipynb

@@ -26,7 +26,7 @@
     "# get key from keypairs.json\n",
     "my_env = 'data'\n",
     "my_key = get_key('koray_data')\n",
-    "schema_name = get_schema_names(my_key) \n",
+    "schema_name = ff_utils.get_schema_names(my_key) \n",
     "print('WORKING ON', my_key['server'], '\\n')\n",
     "\n",
     "##### COLLECT ITEMS TO Release #####\n",

notebooks/useful_notebooks/04_microscopy_template_submit4dn.ipynb

@@ -30,7 +30,7 @@
     "\n",
     "# get key from keypairs.json\n",
     "my_key = get_key('koray_data')\n",
-    "schema_name = get_schema_names(my_key) \n",
+    "schema_name = ff_utils.get_schema_names(my_key) \n",
     "\n",
     "# project folder\n",
     "my_folder = '/Users/koray/Desktop/wrangling/Templating/'\n",

notebooks/useful_notebooks/04_omics_template_submit4dn.ipynb

@@ -31,7 +31,7 @@
     "\n",
     "# get key from keypairs.json\n",
     "my_key = get_key('')\n",
-    "schema_name = get_schema_names(my_key) \n",
+    "schema_name = ff_utils.get_schema_names(my_key) \n",
     "\n",
     "# template excel from submit4dn\n",
     "excel_file = '/Users/user/Desktop/Templating/MetadataSheets.xls'\n",

notebooks/useful_notebooks/07_find_items_add_to_excel_or_create_inserts.ipynb

@@ -46,11 +46,11 @@
     "print(round((time2-time1), 1), 'sec for collection')\n",
     "\n",
     "if purpose == 'excel':\n",
-    "    schema_name = get_schema_names(my_auth) \n",
+    "    schema_name = ff_utils.get_schema_names(my_auth) \n",
     "    append_items_to_xlsx(excel_file, store, schema_name)\n",
     "\n",
     "elif purpose == 'inserts':\n",
-    "    dump_results_to_json(store, my_folder)"
+    "    ff_utils.dump_results_to_json(store, my_folder)"
    ]
   }
  ],

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicwrangling"
-version = "2.2.1"
+version = "2.3.0"
 description = "Scripts and Jupyter notebooks for 4DN wrangling"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -11,7 +11,7 @@ packages = [{ include="scripts", from="." }]
 
 [tool.poetry.dependencies]
 python = ">=3.7.1, <3.9"
-dcicutils = "^7.8.0"
+dcicutils = "^7.9.0"
 openpyxl = "^3.0.9"
 Biopython = "1.76"
 GEOparse = "^2.0.1"