Merge pull request #298 from 4dn-dcic/python-3.12

Python 3.12

.github/workflows/main-publish.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install Python dependencies for publish
         run: python -m pip install dcicutils==7.5.0
       - name: Publish

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
     # Build matrix
     strategy:
       matrix:
-        python_version: [3.8, 3.11]
+        python_version: [3.9, 3.11, 3.12]
         test_type: ['UNIT', 'INDEXING']
 
     # Steps represent a sequence of tasks that will be executed as part of the job

.github/workflows/static-checks.yml

@@ -27,7 +27,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Python Package Setup
         run: |
           make build-for-ga

CHANGELOG.rst

@@ -6,6 +6,23 @@ snovault
 Change Log
 ----------
 
+11.18.0
+=======
+
+* Dropped support for Python 3.8.
+* Updates related to Python 3.12.
+  - Had to update venusian (from 1.2.0) to 3.1.0.
+  - Had to update pyramid (from 1.10.4) to 1.10.8 (for imp import not found).
+    - Had to add pmdarima (no module pyramid.compat).
+    - Had to define/update numpy (to 1.26.4) for this as it was implicitly,
+      due to something else, using 1.24.4 which failed to build with Python 3.12.
+      - And had to update lower bound of Python (from 3.8.1) to 3.9 for this.
+  - Had to update dcicutils (from 8.11.0) to 8.13.0  (for pyramid update for imp import not found).
+* Minor change to dev_servers.py to facilitate running a local ElasticSearch proxy
+  to observe traffic (resquests/responses) between the portal and ElasticSearch
+  with a tool like mitmproxy or mitmweb; see comments in dev_server.py.
+
+
 11.17.0
 =======
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicsnovault"
-version = "11.17.0"
+version = "11.18.0"
 description = "Storage support for 4DN Data Portals."
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -34,27 +34,34 @@ classifiers = [
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12'
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<3.12"
+python = ">=3.9,<3.13"
 aws_requests_auth = "^0.4.1"
-boto3 = ">=1.28.62"  # no particular version required, but this speeds up search
-botocore = ">=1.31.62"  # no particular version required, but this speeds up search
+boto3 = "^1.34.136"
+botocore = "^1.34.136"
 elasticsearch = "7.13.4"  # versions >= 7.14.0 lock out AWS ES
 elasticsearch_dsl = "^7.4.0"
-dcicutils = "^8.11.0"
-future = ">=0.15.2,<1"
+#python-3.12 elasticsearch = "^7.17.9"
+#python-3.12 elasticsearch_dsl = "^7.4.1"
+dcicutils = "^8.13.0"
+future = "^0.18.3"
 html5lib = ">=1.1"  # experimental, should be OK now that we're not using moto server
 humanfriendly = "^1.44.9"
 netaddr = ">=0.8.0,<1"
+numpy = "^1.26.4"
 passlib = "^1.7.4"
 pillow = "^9.5.0"
+pmdarima = "^2.0.4"
 psutil = "^5.9.0"
 psycopg2-binary = "^2.9.1"
 PyBrowserID = ">=0.10.0,<1"
 pyjwt = "^2.6.0"
-pyramid = "1.10.4"
+#pyramid = "^2.0.2"
+#pyramid = "1.10.4"
+pyramid = "1.10.8"
 pyramid-multiauth = ">=0.9.0,<1"
 pyramid-retry = "^1.0"
 pyramid-tm = "^2.5"
@@ -75,7 +82,7 @@ subprocess_middleware = ">=0.3,<1"
 # TODO: Investigate whether a major version upgrade is allowable for 'transaction'.
 transaction = "^3.0.1"
 # TODO: Investigate whether a major version upgrade is allowable for 'venusian'.
-venusian = "^1.2.0"
+venusian = "^3.1.0"
 WebOb = "^1.8.7"
 WebTest = "^2.0.35"
 WSGIProxy2 = "0.4.2"
@@ -86,8 +93,8 @@ xlrd = "^1.0.0"
 jsonschema = {extras = ["format-nongpl"], version = "^4.19.0"}
 
 [tool.poetry.dev-dependencies]
-boto3-stubs = ">=1.28.62"  # no particular version required, but this speeds up search
-botocore-stubs = ">=1.31.62"  # no particular version required, but this speeds up search
+boto3-stubs = "^1.34.136"
+botocore-stubs = "^1.34.136"
 coverage = ">=6.2"
 codacy-coverage = ">=1.3.11"
 # When we add coverage, this must be loaded manually in GA workflow for coverage because a dependency on 2to3

snovault/dev_servers.py

@@ -160,8 +160,20 @@ def cleanup_process():
     es_server_url = config.get('elasticsearch.server', "localhost")
 
     if '127.0.0.1' in es_server_url or 'localhost' in es_server_url:
-        # Bootup local ES server subprocess. Else assume connecting to remote ES cluster.
-        elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True)
+        # Bootup local ES server subprocess; otherwise assume we are connecting to a remote ES cluster.
+        # The elasticsearch.server.actual_port property is useful (only) for running a localhost ElasticSearch
+        # proxy in order to observe traffic (requests/responses) between portal and ElasticSearch with a tool like
+        # mitmweb; e.g. setting elasticsearch.server.actual_port to 9201 and elasticsearch.server to localhost:9200
+        # will cause ElasticSearch to actually run on port 9201 but will cause portal to talk to it via port 9200,
+        # and then we can run mitmweb --mode reverse:http://localhost:9201 -p 9200 --web-port 8081 which will
+        # allow us to browse to http://localhost:8081 locally to observe all of the ElasticSearch traffic.
+        if (es_port := config.get('elasticsearch.server.actual_port', None)) and es_port.isdigit():
+            es_port = int(es_port)
+        elif ((colon := es_server_url.rfind(":")) > 0) and (es_port := es_server_url[colon + 1:]).isdigit():
+            es_port = int(es_port)
+        else:
+            es_port = None
+        elasticsearch = elasticsearch_fixture.server_process(esdata, port=es_port, echo=True)
         processes.append(elasticsearch)
     elif not config.get('indexer.namespace'):
         raise Exception(

snovault/embed.py

@@ -1,14 +1,14 @@
 import logging
 from copy import deepcopy
 from posixpath import join
-from pyramid.compat import (
-    native_,
-    unquote_bytes_to_wsgi,
-)
 from pyramid.httpexceptions import HTTPNotFound, HTTPServerError
 import pyramid.request
 from .crud_views import collection_add as sno_collection_add
 from .interfaces import COLLECTIONS, CONNECTION
+from .pyramid_compat import (
+    native_,
+    unquote_bytes_to_wsgi,
+)
 from .resources import Collection
 from .schema_utils import validate_request
 from dcicutils.misc_utils import check_true

snovault/local_roles.py

@@ -1,6 +1,6 @@
 from dcicutils.misc_utils import environ_bool, PRINT
 from pyramid.authorization import ACLAuthorizationPolicy
-from pyramid.compat import is_nonstr_iter
+from pyramid.util import is_nonstr_iter
 from pyramid.interfaces import IAuthorizationPolicy
 from pyramid.location import lineage
 from zope.interface import implementer

snovault/pyramid_compat.py

@@ -0,0 +1,24 @@
+# Note that pyramid.compat disappeared with pyramid 2.0.2, during move to Python 3.12,
+# though went back to pyramid 1.10.8 subsequently, will run into this eventually.
+
+from urllib.parse import unquote_to_bytes
+
+text_type = str
+
+
+def ascii_native_(s):
+    if isinstance(s, text_type):
+        s = s.encode('ascii')
+    return str(s, 'ascii', 'strict')
+
+
+def native_(s, encoding='latin-1', errors='strict'):
+    """ If ``s`` is an instance of ``text_type``, return
+    ``s``, otherwise return ``str(s, encoding, errors)``"""
+    if isinstance(s, text_type):
+        return s
+    return str(s, encoding, errors)
+
+
+def unquote_bytes_to_wsgi(bytestring):
+    return unquote_to_bytes(bytestring).decode('latin-1')

snovault/tests/test_views.py

@@ -4,10 +4,10 @@
 
 from base64 import b64encode
 from jsonschema import Draft202012Validator
-from pyramid.compat import ascii_native_
 from uuid import uuid4
 
 from ..interfaces import TYPES
+from ..pyramid_compat import ascii_native_
 from ..util import mappings_use_nested
 
 from .testing_views import (