Merge pull request #1001 from Sage-Bionetworks/develop

Release 22.11.1

.github/workflows/pdoc.yml

@@ -0,0 +1,95 @@
+name: pdoc
+
+# build the documentation whenever there are new commits on main
+on:
+  push:
+    branches:
+      - develop
+  workflow_dispatch:  # Allow manually triggering the workflow
+
+# security: restrict permissions for CI jobs.
+permissions:
+  contents: read
+
+concurrency:
+  # cancel the current running workflow from the same branch, PR when a new workflow is triggered 
+  # when the trigger is not a PR but a push, it will use the commit sha to generate the concurrency group
+  # {{ github.workflow }}: the workflow name is used to generate the concurrency group. This allows you to have more than one workflows
+  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag 
+  # {{ github.event.pull_request.number}}: get PR number 
+  # {{ github.sha }}: full commit sha 
+  # credit: https://github.com/Sage-Bionetworks-Workflows/sagetasks/blob/main/.github/workflows/ci.yml
+  group: >-
+    ${{ github.workflow }}-${{ github.ref_type }}-
+    ${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      POETRY_VERSION:  1.2.0
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python     
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      #----------------------------------------------
+      #          install & configure poetry         
+      #----------------------------------------------
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org \
+            | python3 - --version ${{ env.POETRY_VERSION }};
+          poetry config virtualenvs.create true; 
+          poetry config virtualenvs.in-project true; 
+
+      #----------------------------------------------
+      #       load cached venv if cache exists      
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+
+      #----------------------------------------------
+      # install dependencies if cache does not exist 
+      #----------------------------------------------
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      # create documentation
+      - run: poetry show pdoc
+      - run: poetry run pdoc --docformat google -o docs/schematic schematic/manifest schematic/models schematic/schemas schematic/store schematic/utils schematic/visualization
+
+      - uses: actions/upload-pages-artifact@v1
+        with:
+          path: docs/schematic
+
+  # Deploy the artifact to GitHub pages.
+  # This is a separate job so that only actions/deploy-pages has the necessary permissions.
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v1

README.md

@@ -179,6 +179,72 @@ For new features, bugs, enhancements
 
 *Note*: Make sure you have the latest version of the `develop` branch on your local machine.
 
+## Installation Guide - Docker 
+
+1. Install docker from https://www.docker.com/ . <br>
+2.  Identify docker image of interest from [Schematic DockerHub](https://hub.docker.com/r/sagebionetworks/schematic/tags) <br>
+    Ex `docker pull sagebionetworks/schematic:latest` from the CLI or, run `docker compose up` after cloning the schematic github repo <br>
+    in this case, `sagebionetworks/schematic:latest` is the name of the image chosen
+3. Run Schematic Command with `docker run <flags> <schematic command and args>`. <br>
+<t> - For more information on flags for `docker run` and what they do, visit the [Docker Documentation](https://docs.docker.com/engine/reference/commandline/run/) <br>
+<t> - These example commands assume that you have navigated to the directory you want to run schematic from. To specify your working directory, use `$(pwd)` on MacOS/Linux or `%cd%` on Windows.  <br>
+<t> - If not using the latest image, then the full name should be specified: ie `sagebionetworks/schematic:commit-e611e4a` <br>
+<t> - If using local image created by `docker compose up`, then the docker image name should be changed: i.e. `schematic_schematic` <br>
+<t> - Using the `--name` flag sets the name of the container running locally on your machine <br>
+
+### Example For REST API <br>
+
+#### Use file path of `config.yml` to run API endpoints: 
+```
+docker run --rm -p 3001:3001 \
+  -v $(pwd):/schematic -w /schematic --name schematic \
+  -e SCHEMATIC_CONFIG=/schematic/config.yml \
+  -e GE_HOME=/usr/src/app/great_expectations/ \
+  sagebionetworks/schematic \
+  python /usr/src/app/run_api.py
+``` 
+
+#### Use content of `config.yml` as an environment variable to run API endpoints: 
+1. save content of `config.yml` as to environment variable `SCHEMATIC_CONFIG_CONTENT` by doing: `export SCHEMATIC_CONFIG_CONTENT=$(cat config.yml)`
+
+2. Pass `SCHEMATIC_CONFIG_CONTENT` as an environment variable by using `docker run`
+
+```
+docker run --rm -p 3001:3001 \
+  -v $(pwd):/schematic -w /schematic --name schematic \
+  -e GE_HOME=/usr/src/app/great_expectations/ \
+  -e SCHEMATIC_CONFIG_CONTENT=$SCHEMATIC_CONFIG_CONTENT \
+  sagebionetworks/schematic \
+  python /usr/src/app/run_api.py
+``` 
+
+
+### Example For Schematic on mac/linux <br>
+To run example below, first clone schematic into your home directory  `git clone https://github.com/sage-bionetworks/schematic ~/schematic` <br>
+Then update .synapseConfig with your credentials
+```
+docker run \
+  -v ~/schematic:/schematic \
+  -w /schematic \
+  -e SCHEMATIC_CONFIG=/schematic/config.yml \
+  -e GE_HOME=/usr/src/app/great_expectations/ \
+  sagebionetworks/schematic schematic model \
+  -c /schematic/config.yml validate \
+  -mp /schematic/tests/data/mock_manifests/Valid_Test_Manifest.csv \
+  -dt MockComponent \
+  -js /schematic/tests/data/example.model.jsonld
+``` 
+
+### Example For Schematic on Windows <br>
+```
+docker run -v %cd%:/schematic \
+  -w /schematic \
+  -e GE_HOME=/usr/src/app/great_expectations/ \
+  sagebionetworks/schematic \
+  schematic model \
+  -c config.yml validate -mp tests/data/mock_manifests/inValid_Test_Manifest.csv -dt MockComponent -js /schematic/data/example.model.jsonld
+```
+
 # Other Contribution Guidelines
 ## Updating readthedocs documentation
 1. `cd docs`

api/__init__.py

@@ -15,7 +15,10 @@ def create_app():
     # path to config.yml file saved as a Flask config variable
     default_config = os.path.abspath(os.path.join(__file__, "../../config.yml"))
     schematic_config = os.environ.get("SCHEMATIC_CONFIG", default_config)
+    schematic_config_content = os.environ.get("SCHEMATIC_CONFIG_CONTENT")
+
     app.config["SCHEMATIC_CONFIG"] = schematic_config
+    app.config["SCHEMATIC_CONFIG_CONTENT"] = schematic_config_content
 
     # Configure flask app
     # app.config[] = schematic[]

api/routes.py

@@ -37,7 +37,14 @@
 def config_handler(asset_view=None):
     path_to_config = app.config["SCHEMATIC_CONFIG"]
 
-    # check if file exists at the path created, i.e., app.config['SCHEMATIC_CONFIG']
+    # if content of the config file is provided: 
+    content_of_config = app.config["SCHEMATIC_CONFIG_CONTENT"]
+
+    # if the environment variable exists
+    if content_of_config:
+        CONFIG.load_config_content_from_env()
+
+    # check if path to config is provided
     if os.path.isfile(path_to_config):
         CONFIG.load_config(path_to_config, asset_view = asset_view)
 

docker-compose.yml

@@ -13,4 +13,5 @@ services:
       APP_HOST: "0.0.0.0"
       APP_PORT: "3001"
       SCHEMATIC_CONFIG: /schematic/config.yml
+      SCHEMATIC_CONFIG_CONTENT: "${SCHEMATIC_CONFIG_CONTENT}"
       GE_HOME: /usr/src/app/great_expectations/

pyproject.toml

@@ -66,6 +66,7 @@ Jinja2 = "2.11.3"
 openpyxl = "^3.0.9"
 "backports.zoneinfo" = {markers = "python_version < \"3.9\"", version = "^0.2.1"}
 Flask-Cors = "^3.0.10"
+pdoc = "^12.2.0"
 
 
 [tool.poetry.dev-dependencies]

schematic/configuration.py

@@ -9,12 +9,16 @@ def __init__(self):
         # entire configuration data
         self.DATA = None
 
+
     def __getattribute__(self, name):
         value = super().__getattribute__(name)
-        if value is None and "SCHEMATIC_CONFIG" in os.environ:
+        if value is None and "SCHEMATIC_CONFIG_CONTENT" in os.environ:
+            self.load_config_content_from_env()
+            value = super().__getattribute__(name)
+        elif value is None and "SCHEMATIC_CONFIG" in os.environ:
             self.load_config_from_env()
             value = super().__getattribute__(name)
-        elif value is None and "SCHEMATIC_CONFIG" not in os.environ:
+        elif value is None and "SCHEMATIC_CONFIG" not in os.environ and "SCHEMATIC_CONFIG_CONTENT" not in os.environ:
             raise AttributeError(
                 "The '%s' configuration field was accessed, but it hasn't been "
                 "set yet, presumably because the schematic.CONFIG.load_config() "
@@ -34,6 +38,14 @@ def get(self, key, default):
             value = default
         return value
 
+    def load_config_content(self, str_yaml: str) -> dict:
+        try: 
+            config_data = yaml.safe_load(str_yaml)
+        except yaml.YAMLError as exc:
+            print(exc)
+            return None
+        return config_data
+
     @staticmethod
     def load_yaml(file_path: str) -> dict:
         with open(file_path, "r") as stream:
@@ -45,9 +57,15 @@ def load_yaml(file_path: str) -> dict:
         return config_data
 
     def normalize_path(self, path):
-        # Retrieve parent directory of the config to decode relative paths
-        parent_dir = os.path.dirname(self.CONFIG_PATH)
-        # Ensure absolute file paths
+
+        if self.CONFIG_PATH:
+            # Retrieve parent directory of the config to decode relative paths
+            parent_dir = os.path.dirname(self.CONFIG_PATH)
+        else:
+            # assume the parent dir would be the current work dir
+            parent_dir = os.getcwd()
+
+            # Ensure absolute file paths
         if not os.path.isabs(path):
             path = os.path.join(parent_dir, path)
         # And lastly, normalize file paths
@@ -61,7 +79,19 @@ def load_config_from_env(self):
         )
         return self.load_config(schematic_config)
 
-    def load_config(self, config_path=None, asset_view=None):
+    def load_config_content_from_env(self):
+        schematic_config_content = os.environ["SCHEMATIC_CONFIG_CONTENT"]
+
+        print(
+            'Loading content of config file:  %s' % schematic_config_content
+        )
+
+        config_content_yaml = self.load_config_content(schematic_config_content)
+        self.DATA = config_content_yaml
+
+        return self.DATA
+
+    def load_config(self, config_path=None, asset_view=None): 
         # If config_path is None, try loading from environment
         if config_path is None and "SCHEMATIC_CONFIG" in os.environ:
             return self.load_config_from_env()