Merge pull request #754 from dbmi-bgm/upgrade-python-20230925

Upgrade to Python 3.11

.github/workflows/main.yml

@@ -20,14 +20,14 @@ jobs:
     name: cgap-portal tests
 
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Build matrix
     strategy:
       matrix:
         test_type: ['UNIT', 'INDEX', 'DOCKER']
         # We are really not set up for these next two to be multiplicative, so be careful adding more.
-        python_version: ['3.8']
+        python_version: ['3.11']
         node_version: ['18']
 
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -108,7 +108,7 @@ jobs:
           poetry run wipe-test-indices $TEST_JOB_ID search-cgap-unit-testing-opensearch-tcs45cjpwgdzoi7pafr6oewq6u.us-east-1.es.amazonaws.com:443
 
       - name: Docker Build
-        if: ${{ matrix.test_type == 'DOCKER' && matrix.node_version == '18' && matrix.python_version == '3.8' }}
+        if: ${{ matrix.test_type == 'DOCKER' && matrix.node_version == '18' && matrix.python_version == '3.11' }}
         run: |
           touch deploy/docker/local/docker_development.ini  # cheap substitute for prepare-docker to make ignored file
           docker build .

CHANGELOG.rst

@@ -6,6 +6,19 @@ cgap-portal
 Change Log
 ----------
 
+15.0.0
+======
+* Upgrade to Python 3.11.
+* Changed pyyaml version to ^6.0.1.
+* Updated boto versions.
+* Removed types/access_key.py and schemas/access_key.json as the ones in snovault are sufficient.
+* Added generate-local-access-key script (from snovault) to pyproject.toml;
+  orignally created for smaht-portal since early in development no way to
+  create an access-key normally using the UI; but generall useful/convenient.
+* Added generate-local-access-key script (defined in snovault).
+* Added view-local-object script (defined in snovault).
+
+
 14.3.1
 ======
 `PR 753: Auth0 Symlink Bugfix <https://github.com/dbmi-bgm/cgap-portal/pull/753>`_

Dockerfile

@@ -1,7 +1,9 @@
 # CGAP-Portal (Production) Dockerfile
 
-# Debian Buster with Python 3.8.13
-FROM python:3.8.13-slim-buster
+# Bullseye with Python 3.11.5
+# 2023-09-28: Update docker image to a Python 3.11 version;
+# this was previously: FROM python:3.8.13-slim-buster
+FROM python:3.11.5-slim-bullseye
 
 MAINTAINER William Ronchetti "[email protected]"
 
@@ -32,7 +34,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 # Note that the ordering of these operations is intentional to minimize package footprint
 WORKDIR /home/nginx/.nvm
 ENV NVM_DIR=/home/nginx/.nvm
-COPY deploy/docker/production/install_nginx.sh /install_nginx.sh
+COPY deploy/docker/production/install_nginx_bullseye.sh /install_nginx.sh
 
 # Temporarily replacing
 #     curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/venv python - && \

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "cgap-portal".
 name = "encoded"
-version = "14.3.1"
+version = "15.0.0"
 description = "Computational Genome Analysis Platform"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -31,15 +31,17 @@ classifiers = [
     # Specify the Python versions you support here. In particular, ensure
     # that you indicate whether you support Python 2, Python 3 or both.
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<3.10"
-awscli = ">=1.27.79"
-boto3 = "^1.26.133"
-botocore = "^1.29.79"
+python = ">=3.8.1,<3.12"
+awscli = ">=1.29.62"
+boto3 = "^1.28.62"
+botocore = "^1.31.62"
 certifi = ">=2021.5.30"
 chardet = "3.0.4"
 codeguru-profiler-agent = "^1.2.4"
@@ -48,9 +50,9 @@ codeguru-profiler-agent = "^1.2.4"
 # of cryptography.hazmat.bindings._rust in cryptography package. 2023-04-21.
 cryptography = "39.0.2"
 colorama = "0.3.3"
-dcicpyvcf = "1.0.0.1b0"  # "^1.0.0"
-dcicsnovault = "^10.0.2"
-dcicutils = "^7.7.0"
+dcicpyvcf = "^3.0.0"
+dcicsnovault = "^11.0.0"
+dcicutils = "^8.0.0"
 elasticsearch = "7.13.4"
 execnet = "1.4.1"
 future = ">=0.18.2,<1"
@@ -88,7 +90,7 @@ python-dateutil = "^2.8.2"
 # python-magic is presently pinned to 0.4.15 in lockstep with dcicsnovault's requirements. See explanation there.
 python_magic = ">=0.4.24,<1"
 pytz = ">=2021.3"
-PyYAML = "5.3.1"
+PyYAML = "^6.0.1"
 rdflib = "^4.2.2"
 rdflib-jsonld = ">=0.5.0,<1.0.0"
 # repoze.debug is needed to use pyramid.pserve - Will Feb 17 2022
@@ -122,12 +124,11 @@ WSGIProxy2 = "0.4.2"
 "zope.deprecation" = "^4.4.0"
 "zope.interface" = ">=4.7.2,<6"
 "zope.sqlalchemy" = "1.6"
-pyyaml = "5.3.1"
 
 [tool.poetry.dev-dependencies]
 # PyCharm says boto3-stubs contains useful type hints
-boto3-stubs = "^1.26.79"
-botocore-stubs = "^1.29.79"
+boto3-stubs = "^1.28.62"
+botocore-stubs = "^1.31.62"
 coverage = ">=6.2"
 codacy-coverage = ">=1.3.11"
 coveralls = ">=3.3.1"
@@ -179,6 +180,7 @@ dis2pheno = "encoded.commands.parse_hpoa:main"
 es-index-data = "snovault.commands.es_index_data:main"
 export-data = "encoded.commands.export_data:main"
 extract-test-data = "encoded.commands.extract_test_data:main"
+generate-local-access-key = "snovault.commands.generate_local_access_key:main"
 import-data = "encoded.commands.import_data:main"
 jsonld-rdf = "encoded.commands.jsonld_rdf:main"
 load-access-keys = "encoded.commands.load_access_keys:main"
@@ -203,6 +205,7 @@ submission-test = "encoded.commands.submission_test:main"
 # submit-metadata-bundle = "encoded.commands.submit_metadata_bundle:main"
 update-inserts-from-server = "encoded.commands.update_inserts_from_server:main"
 verify-item = "encoded.commands.verify_item:main"
+view-local-object= "snovault.commands.view_local_object:main"
 # cgap-specific commands
 clear-variants-and-genes = "encoded.commands.clear_variants_and_genes:main"
 gene-table-intake = "encoded.commands.gene_table_intake:main"

src/encoded/ingestion/table_utils.py

@@ -3,7 +3,7 @@
 import io
 import json
 import logging
-from collections import Mapping
+from collections.abc import Mapping
 from ..util import resolve_file_path
 
 logger = logging.getLogger(__name__)

src/encoded/tests/test_schemas.py

@@ -249,7 +249,7 @@ def test_linkTo_saves_uuid(root, bgm_user, institution):
 
 
 def test_mixinProperties():
-    schema = load_schema('encoded:schemas/access_key.json')
+    schema = load_schema('snovault:schemas/access_key.json')
     assert schema['properties']['uuid']['type'] == 'string'