update lambdas to Python 3.11 (#4241)

.github/workflows/deploy-lambdas.yaml

@@ -38,7 +38,6 @@ jobs:
           docker run --rm \
             --entrypoint /build_zip.sh \
             -v "$PWD/lambdas/${{ matrix.path }}":/lambda/function:z \
-            -v "$PWD/lambdas/shared":/lambda/shared:z \
             -v "$PWD/out.zip":/out.zip:z \
             -v "$PWD/lambdas/scripts/build_zip.sh":/build_zip.sh:z \
             "$BUILDER_IMAGE"
@@ -75,7 +74,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Build Docker image
         working-directory: ./lambdas/${{ matrix.path }}
-        run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile ..
+        run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile .
       - name: Configure AWS credentials from Prod account
         uses: aws-actions/configure-aws-credentials@v4
         with:

.github/workflows/py-ci.yml

@@ -187,23 +187,15 @@ jobs:
             python -m pip install wheel 'numpy<2'
           fi
 
-          # XXX: something fishy is going on with this "if [ ] X then Y" syntax
-          if [ ${{ matrix.path }} == "shared" ]
-            python -m pip install -e lambdas/shared[tests]
-          then
-            python -m pip install -e lambdas/shared
-            python -m pip install -e lambdas/${{ matrix.path }}
-          fi
-
-          python -m pip install -r lambdas/${{ matrix.path }}/test-requirements.txt
-
           # Try to simulate the lambda .zip file:
           # - Use --no-deps to ensure that second-order dependencies are included in the requirements file
           # - Remove "tests" directories
           # - Run "strip" on shared libraries
-          python -m pip install -t deps --no-deps -r lambdas/${{ matrix.path }}/requirements.txt
+          python -m pip install -t deps --no-deps -r lambdas/${{ matrix.path }}/requirements.txt lambdas/${{ matrix.path }}
           find deps -name tests -type d -exec rm -r \{} \+
           find deps \( -name '*.so.*' -o -name '*.so' \) -type f -exec strip \{} \+
+
+          python -m pip install -r lambdas/${{ matrix.path }}/test-requirements.txt
       - name: Pytest
         run: |
           pytest --cov=lambdas lambdas/${{ matrix.path }}

lambdas/access_counts/.python-version

@@ -1 +1 @@
-3.8
+3.11

lambdas/access_counts/CHANGELOG.md

@@ -0,0 +1,21 @@
+<!-- markdownlint-disable line-length -->
+# Changelog
+
+Changes are listed in reverse chronological order (newer entries at the top).
+The entry format is
+
+```markdown
+- [Verb] Change description ([#<PR-number>](https://github.com/quiltdata/quilt/pull/<PR-number>))
+```
+
+where verb is one of
+
+- Removed
+- Added
+- Fixed
+- Changed
+
+## Changes
+
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
+- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241))

lambdas/access_counts/requirements.txt

@@ -0,0 +1,24 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile
+#
+attrs==24.2.0
+    # via
+    #   jsonschema
+    #   referencing
+jsonschema==4.23.0
+    # via t4-lambda-shared
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+rpds-py==0.21.0
+    # via
+    #   jsonschema
+    #   referencing
+t4-lambda-shared @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared
+    # via lambda_function (setup.py)

lambdas/access_counts/setup.py

@@ -4,4 +4,11 @@
     name='lambda_function',
     version='0.0.1',
     py_modules=['index'],
+    install_requires=[
+        (
+            "t4_lambda_shared @ https://github.com/quiltdata/quilt/archive/"
+            "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip"
+            "#subdirectory=lambdas/shared"
+        ),
+    ],
 )

lambdas/access_counts/test-requirements.in

@@ -0,0 +1,4 @@
+-c requirements.txt
+boto3 ~= 1.17
+pytest ~= 8.0
+pytest-cov ~= 6.0

lambdas/access_counts/test-requirements.txt

@@ -1,4 +1,38 @@
-boto3==1.17.100
-botocore==1.20.100
-pytest==4.3.0
-pytest-cov==2.6.1
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+boto3==1.35.67
+    # via -r test-requirements.in
+botocore==1.35.67
+    # via
+    #   boto3
+    #   s3transfer
+coverage[toml]==7.6.7
+    # via pytest-cov
+iniconfig==2.0.0
+    # via pytest
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+packaging==24.2
+    # via pytest
+pluggy==1.5.0
+    # via pytest
+pytest==8.3.3
+    # via
+    #   -r test-requirements.in
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via -r test-requirements.in
+python-dateutil==2.9.0.post0
+    # via botocore
+s3transfer==0.10.4
+    # via boto3
+six==1.16.0
+    # via python-dateutil
+urllib3==2.2.3
+    # via botocore

lambdas/indexer/.python-version

@@ -1 +1 @@
-3.8
+3.11

lambdas/indexer/CHANGELOG.md

@@ -17,5 +17,6 @@ where verb is one of
 
 ## Changes
 
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
 - [Changed] Stop using S3 select ([#4212](https://github.com/quiltdata/quilt/pull/4212))
 - [Added] Bootstrap the change log ([#4212](https://github.com/quiltdata/quilt/pull/4212))

lambdas/indexer/requirements.txt

@@ -1,48 +1,117 @@
-attrs==19.1.0
-aws-requests-auth==0.4.2
-boto3==1.34.41
-botocore==1.34.41
-certifi==2024.7.4
-cffi==1.15.1
-chardet==3.0.4
-charset-normalizer==2.0.12
-cryptography==43.0.1
-decorator==4.4.0
-docutils==0.15.2
-# This is the highest version that's less than our pinned ES version of 6.7
-elasticsearch==6.3.1
-et-xmlfile==1.1.0
-fcsparser==0.2.1
-idna==3.7
-importlib-metadata==6.6.0
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile
+#
+attrs==24.2.0
+    # via jsonschema
+aws-requests-auth==0.4.3
+    # via es_indexer (setup.py)
+boto3==1.35.70
+    # via es_indexer (setup.py)
+botocore==1.35.70
+    # via
+    #   boto3
+    #   s3transfer
+certifi==2024.8.30
+    # via requests
+cffi==1.17.1
+    # via cryptography
+charset-normalizer==3.4.0
+    # via
+    #   pdfminer-six
+    #   requests
+cryptography==44.0.0
+    # via pdfminer-six
+elasticsearch==6.8.2
+    # via es_indexer (setup.py)
+et-xmlfile==2.0.0
+    # via openpyxl
+fcsparser==0.2.8
+    # via t4-lambda-shared
+idna==3.10
+    # via requests
 ipython-genutils==0.2.0
-jmespath==0.9.4
-jsonschema==3.2.0
+    # via nbformat
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
 jsonpointer==2.4
-jupyter-core==4.11.2
-lxml==4.9.2
+    # via es_indexer (setup.py)
+jsonschema==3.2.0
+    # via
+    #   es_indexer (setup.py)
+    #   nbformat
+    #   t4-lambda-shared
+jupyter-core==5.7.2
+    # via nbformat
+lxml==5.3.0
+    # via python-pptx
 nbformat==5.1.3
-numpy==1.22.0
-openpyxl==3.0.7
-pandas==1.3.5
-pdfminer.six==20201018
-Pillow==10.3.0
-psutil==5.7.0
-pyarrow==14.0.1
-pycparser==2.21
-pyrsistent==0.15.4
-python-dateutil==2.8.0
-python-pptx==0.6.21
-pytz==2019.2
-requests==2.32.2
-s3transfer==0.10.0
-six==1.12.0
-sortedcontainers==2.4.0
+    # via es_indexer (setup.py)
+numpy==1.26.4
+    # via
+    #   fcsparser
+    #   pandas
+openpyxl==3.1.5
+    # via t4-lambda-shared
+pandas==2.2.3
+    # via
+    #   fcsparser
+    #   t4-lambda-shared
+pdfminer-six==20240706
+    # via es_indexer (setup.py)
+pillow==11.0.0
+    # via python-pptx
+platformdirs==4.3.6
+    # via jupyter-core
+psutil==6.1.0
+    # via t4-lambda-shared
+pyarrow==18.1.0
+    # via t4-lambda-shared
+pycparser==2.22
+    # via cffi
+pyrsistent==0.20.0
+    # via jsonschema
+python-dateutil==2.9.0.post0
+    # via
+    #   botocore
+    #   pandas
+python-pptx==0.6.23
+    # via es_indexer (setup.py)
+pytz==2024.2
+    # via pandas
+requests==2.32.3
+    # via aws-requests-auth
+s3transfer==0.10.4
+    # via boto3
+six==1.16.0
+    # via
+    #   jsonschema
+    #   python-dateutil
 strict-rfc3339==0.7
-tenacity==5.0.3
-traitlets==4.3.2
-typing_extensions==4.5.0
-urllib3==1.26.19
+    # via es_indexer (setup.py)
+t4-lambda-shared[mem,preview] @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared
+    # via es_indexer (setup.py)
+tenacity==9.0.0
+    # via es_indexer (setup.py)
+traitlets==5.14.3
+    # via
+    #   jupyter-core
+    #   nbformat
+tzdata==2024.2
+    # via pandas
+urllib3==2.2.3
+    # via
+    #   botocore
+    #   elasticsearch
+    #   requests
 xlrd==2.0.1
-XlsxWriter==3.1.0
-zipp==3.19.1
+    # via t4-lambda-shared
+xlsxwriter==3.2.0
+    # via python-pptx
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

lambdas/indexer/setup.py

@@ -4,4 +4,21 @@
     name='es_indexer',
     version='0.0.1',
     py_modules=['index', 'document_queue'],
+    install_requires=[
+        "aws-requests-auth ~= 0.4.2",
+        "boto3 ~= 1.34",
+        "elasticsearch ~= 6.3",
+        "jsonpointer ~= 2.4",
+        "jsonschema ~= 3.2",
+        "nbformat ~= 5.1.3",
+        "pdfminer.six == 20240706",
+        "python-pptx ~= 0.6.21",
+        "strict-rfc3339 ~= 0.7",  # for jsonschema format
+        "tenacity ~= 9.0",
+        (
+            "t4_lambda_shared[mem,preview] @ https://github.com/quiltdata/quilt/archive/"
+            "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip"
+            "#subdirectory=lambdas/shared"
+        ),
+    ],
 )

lambdas/indexer/test-requirements.in

@@ -0,0 +1,5 @@
+-c requirements.txt
+pytest ~= 8.0
+pytest-cov ~= 6.0
+pytest-env ~= 1.1
+responses ~= 0.25.3