Merge pull request #1057 from Sage-Bionetworks/develop

Schematic Release v23.1.1

.github/workflows/test.yml

@@ -25,11 +25,11 @@ concurrency:
   cancel-in-progress: true
 jobs:
   test:
-
     runs-on: ubuntu-latest
     env:
       POETRY_VERSION:  1.2.0rc1
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
 
@@ -109,16 +109,17 @@ jobs:
         run: >
           source .venv/bin/activate;
           pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
-          -m "not google_credentials_needed"
+          -m "not (google_credentials_needed or table_operations)"
           
       - name: Run tests
         env:
-          SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}          
+          SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
+          SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
         if: ${{ false == contains(github.event.head_commit.message, 'runcombos') }}
         run: >
           source .venv/bin/activate;
           pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
-          -m "not (google_credentials_needed or rule_combos or schematic_api)"
+          -m "not (google_credentials_needed or rule_combos or schematic_api or table_operations)"
     
       - name: Upload pytest test results
         uses: actions/upload-artifact@v2

.readthedocs.yml

@@ -0,0 +1,35 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+# Borrowed idea from: https://github.com/readthedocs/readthedocs.org/issues/4912
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+  jobs:
+    post_install:
+      - pip install poetry==1.2.0b1
+      - poetry config virtualenvs.create false
+      - poetry install --with doc
+    #Poetry will install my dependencies into the virtualenv created by readthedocs if I set virtualenvs.create=false
+    # You can also specify other tool versions:
+    # nodejs: "16"
+    # rust: "1.55"
+    # golang: "1.17"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+# formats:
+#    - pdf
+
+# Optionally declare the Python requirements required to build your docs
+# python:
+#    install:
+#    - requirements: docs/requirements.txt

Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.10.6
+# FROM python:3.10.6
+FROM python:3.10.8-slim-bullseye
 
 ENV PYTHONFAULTHANDLER=1 \
   PYTHONUNBUFFERED=1 \
@@ -12,8 +13,11 @@ WORKDIR /usr/src/app
 
 RUN apt-get update -qqy \
   && apt-get install -qqy \
-    libopenblas-dev \
-    gfortran
+  libopenblas-dev \
+  gfortran 
+
+# remove libtiff5 for security reasons
+RUN apt remove -y libtiff5
 
 RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
 

README.md

@@ -94,15 +94,12 @@ Description of `config.yml` attributes
 
     definitions:
         synapse_config: "~/path/to/.synapseConfig"
-        creds_path: "~/path/to/credentials.json"
-        token_pickle: "~/path/to/token.pickle"
         service_acct_creds: "~/path/to/service_account_creds.json"
 
     synapse:
         master_fileview: "syn23643253" # fileview of project with datasets on Synapse
         manifest_folder: "~/path/to/manifest_folder/" # manifests will be downloaded to this folder
         manifest_basename: "filename" # base name of the manifest file in the project dataset, without extension
-        token_creds: "syn23643259" # synapse ID of credentials.json file
         service_acct_creds: "syn25171627" # synapse ID of service_account_creds.json file
 
     manifest:
@@ -139,28 +136,12 @@ synapse login -u <synapse username> -p <synapse password> --rememberMe
 Please make sure that you run the command before running `schematic init` below
 
 7. Obtain Google credential Files
-
-To obtain ``credentials.json`` and ``token.pickle``, please run:
-
+To obtain  ``schematic_service_account_creds.json``, please run: 
 ```
 schematic init --config ~/path/to/config.yml
 ```
-This should prompt you with a URL that will take you through Google OAuth. Your `credentials.json` and `token.pickle` will get automatically downloaded the first time you run this command.
-
-*Note* : The ``credentials.json`` file is required when you are using
-[OAuth2](https://developers.google.com/identity/protocols/oauth2)
-to authenticate with the Google APIs.
-
-For details about the steps involved in the [OAuth2 authorization
-flow](https://github.com/Sage-Bionetworks/schematic/blob/develop/schematic/utils/google_api_utils.py#L18)
-refer to the ``Credentials`` section in the
-[docs/md/details](https://github.com/Sage-Bionetworks/schematic/blob/develop/docs/md/details.md#credentials)
-document.
+> As v22.12.1 version of schematic, using `token` mode of authentication (in other words, using `token.pickle` and `credentials.json`) is no longer supported due to Google's decision to move away from using OAuth out-of-band (OOB) flow. Click [here](https://developers.google.com/identity/protocols/oauth2/resources/oob-migration) to learn more. 
 
-To obtain  ``schematic_service_account_creds.json``, please run: 
-```
-schematic init --config ~/path/to/config.yml --auth service_account
-```
 *Notes*: Use the ``schematic_service_account_creds.json`` file for the service
 account mode of authentication (*for Google services/APIs*). Service accounts 
 are special Google accounts that can be used by applications to access Google APIs 
@@ -224,16 +205,19 @@ docker run --rm -p 3001:3001 \
   python /usr/src/app/run_api.py
 ``` 
 
-#### Use content of `config.yml` as an environment variable to run API endpoints: 
-1. save content of `config.yml` as to environment variable `SCHEMATIC_CONFIG_CONTENT` by doing: `export SCHEMATIC_CONFIG_CONTENT=$(cat config.yml)`
+#### Use content of `config.yml` and `schematic_service_account_creds.json`as an environment variable to run API endpoints: 
+1. save content of `config.yml` as to environment variable `SCHEMATIC_CONFIG_CONTENT` by doing: `export SCHEMATIC_CONFIG_CONTENT=$(cat /path/to/config.yml)`
+
+2. Similarly, save the content of `schematic_service_account_creds.json` as `SERVICE_ACCOUNT_CREDS` by doing: `export SERVICE_ACCOUNT_CREDS=$(cat /path/to/schematic_service_account_creds.json)`
 
-2. Pass `SCHEMATIC_CONFIG_CONTENT` as an environment variable by using `docker run`
+3. Pass `SCHEMATIC_CONFIG_CONTENT` and `schematic_service_account_creds` as environment variables by using `docker run`
 
 ```
 docker run --rm -p 3001:3001 \
   -v $(pwd):/schematic -w /schematic --name schematic \
   -e GE_HOME=/usr/src/app/great_expectations/ \
   -e SCHEMATIC_CONFIG_CONTENT=$SCHEMATIC_CONFIG_CONTENT \
+  -e SERVICE_ACCOUNT_CREDS=$SERVICE_ACCOUNT_CREDS \
   sagebionetworks/schematic \
   python /usr/src/app/run_api.py
 ``` 

api/openapi/api.yaml

@@ -47,13 +47,6 @@ paths:
             - Patient
             - Biospecimen
           required: true
-        - in: query
-          name: oauth
-          schema:
-            type: boolean
-            default: true
-          description: OAuth or Service Account
-          required: false
         - in: query
           name: use_annotations
           schema:
@@ -413,8 +406,8 @@ paths:
         - Manifest Operations
   /get/datatype/manifest:
     get:
-      summary: Retrieve asset view table as a dataframe.
-      description: Retrieve asset view table as a dataframe.
+      summary:  Get datatype of attributes in manifest
+      description: Get datatype of attributes in manifest
       operationId: api.routes.get_manifest_datatype
       parameters:
         - in: query

api/routes.py

@@ -197,12 +197,11 @@ def get_temp_jsonld(schema_url):
     return tmp_file.name
 
 # @before_request
-def get_manifest_route(schema_url: str, oauth: bool, use_annotations: bool, dataset_ids=None, asset_view = None, output_format=None, title=None):
+def get_manifest_route(schema_url: str, use_annotations: bool, dataset_ids=None, asset_view = None, output_format=None, title=None):
     """Get the immediate dependencies that are related to a given source node.
         Args:
             schema_url: link to data model in json ld format
             title: title of a given manifest. 
-            oauth: if user wants to use OAuth for Google authentication
             dataset_id: Synapse ID of the "dataset" entity on Synapse (for a given center/project).
             output_format: contains three option: "excel", "google_sheet", and "dataframe". if set to "excel", return an excel spreadsheet
             use_annotations: Whether to use existing annotations during manifest generation
@@ -258,7 +257,6 @@ def create_single_manifest(data_type, title, dataset_id=None, output_format=None
             path_to_json_ld=jsonld,
             title=title,
             root=data_type,
-            oauth=oauth,
             use_annotations=use_annotations,
             alphabetize_valid_values = 'ascending',
         )

config.yml

@@ -1,15 +1,12 @@
 # Do not change the 'definitions' section unless you know what you're doing
 definitions:
   synapse_config: ".synapseConfig"
-  creds_path: "credentials.json"
-  token_pickle: "token.pickle"
   service_acct_creds: "schematic_service_account_creds.json"
 
 synapse:
   master_fileview: 'syn23643253'
   manifest_folder: 'manifests'
   manifest_basename: 'synapse_storage_manifest' 
-  token_creds: 'syn23643259'
   service_acct_creds: 'syn25171627'
 
 manifest: