Merge pull request #1236 from Sage-Bionetworks/develop

Schematic Release v23.6.1

.github/workflows/api_test.yml

@@ -0,0 +1,88 @@
+# Built from:
+# https://github.com/Sage-Bionetworks/schematic/blob/develop/.github/workflows/test.yml
+
+name: Test schematic API
+
+on:
+  workflow_dispatch:
+    inputs:
+      perform_benchmarking:
+        required: true
+        type: boolean
+        description: perform benchmarking test (True) or skip (False)
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      POETRY_VERSION:  1.2.0
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10"]
+
+    steps:
+
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      #----------------------------------------------
+      #          install & configure poetry
+      #----------------------------------------------
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org \
+            | python3 - --version ${{ env.POETRY_VERSION }};
+          poetry config virtualenvs.create true;
+          poetry config virtualenvs.in-project true;
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root --all-extras
+
+      #----------------------------------------------
+      #    install your root project, if required
+      #----------------------------------------------
+      - name: Install library
+        run: poetry install --no-interaction
+
+      #----------------------------------------------
+      #              run API test suite
+      #----------------------------------------------
+      - name: Run all API tests
+        env:
+          SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
+          SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
+        if: ${{ inputs.perform_benchmarking }}
+        run: >
+          source .venv/bin/activate;
+          pytest -m "schematic_api"
+
+      - name: Run API tests + Exclude Benchmarks
+        env:
+          SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
+          SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
+        if: ${{ false == inputs.perform_benchmarking }}
+        run: >
+          source .venv/bin/activate;
+          pytest -m "schematic_api and not rule_benchmark"

.github/workflows/docker.yml

@@ -5,7 +5,7 @@ name: Build and publish container to Docker Hub
 
 on:
   push:
-    tags: ['v[0-9]*', '[0-9]+.[0-9]+*']  # Match tags that resemble a version
+    tags: ['^v\d+\.\d+\.\d+$']  # Match tags that resemble a version, but exclude minior releases. For example, v0.1.1-beta would not get included
   workflow_dispatch:
 
 jobs:

.github/workflows/docker_build.yml

@@ -0,0 +1,49 @@
+# Taken from https://github.com/Sage-Bionetworks/data_curator/blob/schematic-rest-api/.github/workflows/docker_build.yml
+# Workflow derived from https://github.com/r-lib/actions/tree/master/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    tags:
+      - '*beta*'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_PATH: ghcr.io/${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.IMAGE_PATH }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{raw}}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          file: schematic_api/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/pdoc.yml

@@ -72,6 +72,7 @@ jobs:
         run: poetry install --no-interaction --no-root
 
       # create documentation
+      - run: poetry add [email protected]
       - run: poetry show pdoc
       - run: poetry run pdoc --docformat google -o docs/schematic schematic/manifest schematic/models schematic/schemas schematic/store schematic/utils schematic/visualization
 

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish to PyPI
 on:
   push:
     tags:
-      - '*'
+      - 'v[1-9][0-9].[0-9]+.[0-9]+'
     branches: [main]
 
 jobs:

.github/workflows/test.yml

@@ -8,16 +8,16 @@ name: Test schematic
 on:
   push:
     branches: ['main']
-  pull_request: 
+  pull_request:
     branches: ['*']
   workflow_dispatch:  # Allow manually triggering the workflow
 concurrency:
-  # cancel the current running workflow from the same branch, PR when a new workflow is triggered 
+  # cancel the current running workflow from the same branch, PR when a new workflow is triggered
   # when the trigger is not a PR but a push, it will use the commit sha to generate the concurrency group
   # {{ github.workflow }}: the workflow name is used to generate the concurrency group. This allows you to have more than one workflows
-  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag 
-  # {{ github.event.pull_request.number}}: get PR number 
-  # {{ github.sha }}: full commit sha 
+  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag
+  # {{ github.event.pull_request.number}}: get PR number
+  # {{ github.sha }}: full commit sha
   # credit: https://github.com/Sage-Bionetworks-Workflows/sagetasks/blob/main/.github/workflows/ci.yml
   group: >-
     ${{ github.workflow }}-${{ github.ref_type }}-
@@ -27,16 +27,16 @@ jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      POETRY_VERSION:  1.2.0rc1
+      POETRY_VERSION:  1.2.0
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
 
     steps:
 
       #----------------------------------------------
-      #       check-out repo and set-up python     
+      #       check-out repo and set-up python
       #----------------------------------------------
       - name: Check out repository
         uses: actions/checkout@v2
@@ -45,19 +45,18 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      
+
       #----------------------------------------------
-      #          install & configure poetry         
+      #          install & configure poetry
       #----------------------------------------------
       - name: Install Poetry
         run: |
           curl -sSL https://install.python-poetry.org \
             | python3 - --version ${{ env.POETRY_VERSION }};
-          poetry config virtualenvs.create true; 
-          poetry config virtualenvs.in-project true; 
-
+          poetry config virtualenvs.create true;
+          poetry config virtualenvs.in-project true;
       #----------------------------------------------
-      #       load cached venv if cache exists      
+      #       load cached venv if cache exists
       #----------------------------------------------
       - name: Load cached venv
         id: cached-poetry-dependencies
@@ -67,21 +66,21 @@ jobs:
           key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
 
       #----------------------------------------------
-      # install dependencies if cache does not exist 
+      # install dependencies if cache does not exist
       #----------------------------------------------
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-
+        run: poetry install --no-interaction --no-root --all-extras
+
+      #----------------------------------------------
+      #    install your root project, if required
       #----------------------------------------------
-      #    install your root project, if required 
-      #----------------------------------------------      
       - name: Install library
         run: poetry install --no-interaction
-      
+
       #----------------------------------------------
       #             perform linting
-      #----------------------------------------------    
+      #----------------------------------------------
       # Disabled until we agree to turn it on
       # - name: Lint with flake8
       #   run: |
@@ -93,14 +92,34 @@ jobs:
       #----------------------------------------------
       #             check formatting
       #----------------------------------------------
-      # Disabled until we agree to turn it on
-      # - name: Code formatting with black
-      #   run: |
-      #     # run black in check mode
-      #     # if files are not formatted correctly, the build will not go through
-      #     black . --check
+      - name: Code formatting with black
+        run: |
+          # ran only on certain files for now
+          # add here when checked
+          poetry run black schematic/configuration.py schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py --check
+
       #----------------------------------------------
-      #              run test suite   
+      #             type checking/enforcement
+      #----------------------------------------------
+      - name: Type checking with mypy
+        run: |
+          # ran only on certain files for now
+          # add here when checked
+          poetry run mypy --install-types --non-interactive schematic/configuration.py
+          # add here when enforced
+          poetry run mypy --disallow-untyped-defs --install-types --non-interactive schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py
+
+      #----------------------------------------------
+      #             linting
+      #----------------------------------------------
+      - name: Lint with pylint
+        run: |
+          # ran only on certain files for now
+          # add here when checked
+          poetry run pylint schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py
+
+      #----------------------------------------------
+      #              run test suite
       #----------------------------------------------
       - name: Run regular tests and rule combination tests
         env:
@@ -109,8 +128,8 @@ jobs:
         run: >
           source .venv/bin/activate;
           pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
-          -m "not (google_credentials_needed or table_operations)"
-          
+          -m "not (google_credentials_needed or schematic_api or table_operations)"
+
       - name: Run tests
         env:
           SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
@@ -120,7 +139,7 @@ jobs:
           source .venv/bin/activate;
           pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
           -m "not (google_credentials_needed or rule_combos or schematic_api or table_operations)"
-    
+
       - name: Upload pytest test results
         uses: actions/upload-artifact@v2
         with:

.pylintrc

@@ -0,0 +1,2 @@
+[MAIN]
+load-plugins=pylint.extensions.docparams

.readthedocs.yml

@@ -12,7 +12,7 @@ build:
     python: "3.9"
   jobs:
     post_install:
-      - pip install poetry==1.2.0b1
+      - pip install poetry==1.2.0
       - poetry config virtualenvs.create false
       - poetry install --with doc
     #Poetry will install my dependencies into the virtualenv created by readthedocs if I set virtualenvs.create=false

Dockerfile

@@ -7,7 +7,7 @@ ENV PYTHONFAULTHANDLER=1 \
   PIP_NO_CACHE_DIR=off \
   PIP_DISABLE_PIP_VERSION_CHECK=on \
   PIP_DEFAULT_TIMEOUT=200 \
-  POETRY_VERSION=1.2.0rc1
+  POETRY_VERSION=1.2.0
 
 WORKDIR /usr/src/app
 
@@ -29,4 +29,4 @@ RUN poetry install --no-interaction --no-ansi --no-root
 
 COPY . ./
 
-RUN poetry install --no-interaction --no-ansi --only-root
+RUN poetry install --only-root

README.md

@@ -20,7 +20,7 @@ SCHEMATIC is an acronym for _Schema Engine for Manifest Ingress and Curation_. T
 
 # Installation
 ## Installation Requirements
-* Python 3.7.1 or higher
+* Python version 3.9.0≤x<3.11.0 
 
 Note: You need to be a registered and certified user on [`synapse.org`](https://www.synapse.org/), and also have the right permissions to download the Google credentials files from Synapse.
 

certificate.conf

@@ -0,0 +1,22 @@
+server {
+    listen 443 ssl http2 default_server;
+    listen [::]:443 ssl http2 default_server;
+    include     /etc/nginx/conf.d/self-signed.conf;
+    include     /etc/nginx/conf.d/ssl-params.conf;
+    server_name 127.0.0.1;
+
+    location / {
+        try_files $uri @app;
+    }
+    location @app {
+        include uwsgi_params;
+        uwsgi_pass unix:///tmp/uwsgi.sock;
+    }
+    location /static {
+        alias /app/static;
+    }
+    location /health {
+        return 200 'alive';
+        add_header Content-Type text/plain;
+    }
+}