diff --git a/.github/workflows/deploy-lambdas.yaml b/.github/workflows/deploy-lambdas.yaml index abac7e40cd4..e453ef18f5c 100644 --- a/.github/workflows/deploy-lambdas.yaml +++ b/.github/workflows/deploy-lambdas.yaml @@ -38,7 +38,6 @@ jobs: docker run --rm \ --entrypoint /build_zip.sh \ -v "$PWD/lambdas/${{ matrix.path }}":/lambda/function:z \ - -v "$PWD/lambdas/shared":/lambda/shared:z \ -v "$PWD/out.zip":/out.zip:z \ -v "$PWD/lambdas/scripts/build_zip.sh":/build_zip.sh:z \ "$BUILDER_IMAGE" @@ -75,7 +74,7 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker image working-directory: ./lambdas/${{ matrix.path }} - run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile .. + run: docker buildx build -t "quiltdata/lambdas/${{ matrix.path }}:${{ github.sha }}" -f Dockerfile . - name: Configure AWS credentials from Prod account uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index 2b8332b060a..f0b467ea918 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -187,23 +187,15 @@ jobs: python -m pip install wheel 'numpy<2' fi - # XXX: something fishy is going on with this "if [ ] X then Y" syntax - if [ ${{ matrix.path }} == "shared" ] - python -m pip install -e lambdas/shared[tests] - then - python -m pip install -e lambdas/shared - python -m pip install -e lambdas/${{ matrix.path }} - fi - - python -m pip install -r lambdas/${{ matrix.path }}/test-requirements.txt - # Try to simulate the lambda .zip file: # - Use --no-deps to ensure that second-order dependencies are included in the requirements file # - Remove "tests" directories # - Run "strip" on shared libraries - python -m pip install -t deps --no-deps -r lambdas/${{ matrix.path }}/requirements.txt + python -m pip install -t deps --no-deps -r lambdas/${{ matrix.path }}/requirements.txt lambdas/${{ matrix.path }} find deps -name tests -type d -exec rm -r \{} \+ find deps \( -name '*.so.*' -o -name '*.so' \) -type f -exec strip \{} \+ + + python -m pip install -r lambdas/${{ matrix.path }}/test-requirements.txt - name: Pytest run: | pytest --cov=lambdas lambdas/${{ matrix.path }} diff --git a/lambdas/access_counts/.python-version b/lambdas/access_counts/.python-version index cc1923a40b1..2c0733315e4 100644 --- a/lambdas/access_counts/.python-version +++ b/lambdas/access_counts/.python-version @@ -1 +1 @@ -3.8 +3.11 diff --git a/lambdas/access_counts/CHANGELOG.md b/lambdas/access_counts/CHANGELOG.md new file mode 100644 index 00000000000..637b3180c37 --- /dev/null +++ b/lambdas/access_counts/CHANGELOG.md @@ -0,0 +1,21 @@ + +# Changelog + +Changes are listed in reverse chronological order (newer entries at the top). +The entry format is + +```markdown +- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/)) +``` + +where verb is one of + +- Removed +- Added +- Fixed +- Changed + +## Changes + +- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241)) +- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241)) diff --git a/lambdas/access_counts/requirements.txt b/lambdas/access_counts/requirements.txt index e69de29bb2d..23b95f92116 100644 --- a/lambdas/access_counts/requirements.txt +++ b/lambdas/access_counts/requirements.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile +# +attrs==24.2.0 + # via + # jsonschema + # referencing +jsonschema==4.23.0 + # via t4-lambda-shared +jsonschema-specifications==2024.10.1 + # via jsonschema +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.21.0 + # via + # jsonschema + # referencing +t4-lambda-shared @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared + # via lambda_function (setup.py) diff --git a/lambdas/access_counts/setup.py b/lambdas/access_counts/setup.py index 7bb69f4d595..3310d6e6bc5 100644 --- a/lambdas/access_counts/setup.py +++ b/lambdas/access_counts/setup.py @@ -4,4 +4,11 @@ name='lambda_function', version='0.0.1', py_modules=['index'], + install_requires=[ + ( + "t4_lambda_shared @ https://github.com/quiltdata/quilt/archive/" + "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip" + "#subdirectory=lambdas/shared" + ), + ], ) diff --git a/lambdas/access_counts/test-requirements.in b/lambdas/access_counts/test-requirements.in new file mode 100644 index 00000000000..80340a137f0 --- /dev/null +++ b/lambdas/access_counts/test-requirements.in @@ -0,0 +1,4 @@ +-c requirements.txt +boto3 ~= 1.17 +pytest ~= 8.0 +pytest-cov ~= 6.0 diff --git a/lambdas/access_counts/test-requirements.txt b/lambdas/access_counts/test-requirements.txt index b0aa661633c..eab1aa082e3 100644 --- a/lambdas/access_counts/test-requirements.txt +++ b/lambdas/access_counts/test-requirements.txt @@ -1,4 +1,38 @@ -boto3==1.17.100 -botocore==1.20.100 -pytest==4.3.0 -pytest-cov==2.6.1 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile test-requirements.in +# +boto3==1.35.67 + # via -r test-requirements.in +botocore==1.35.67 + # via + # boto3 + # s3transfer +coverage[toml]==7.6.7 + # via pytest-cov +iniconfig==2.0.0 + # via pytest +jmespath==1.0.1 + # via + # boto3 + # botocore +packaging==24.2 + # via pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.3 + # via + # -r test-requirements.in + # pytest-cov +pytest-cov==6.0.0 + # via -r test-requirements.in +python-dateutil==2.9.0.post0 + # via botocore +s3transfer==0.10.4 + # via boto3 +six==1.16.0 + # via python-dateutil +urllib3==2.2.3 + # via botocore diff --git a/lambdas/indexer/.python-version b/lambdas/indexer/.python-version index cc1923a40b1..2c0733315e4 100644 --- a/lambdas/indexer/.python-version +++ b/lambdas/indexer/.python-version @@ -1 +1 @@ -3.8 +3.11 diff --git a/lambdas/indexer/CHANGELOG.md b/lambdas/indexer/CHANGELOG.md index c7ea99597d5..c8fb9be45e8 100644 --- a/lambdas/indexer/CHANGELOG.md +++ b/lambdas/indexer/CHANGELOG.md @@ -17,5 +17,6 @@ where verb is one of ## Changes +- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241)) - [Changed] Stop using S3 select ([#4212](https://github.com/quiltdata/quilt/pull/4212)) - [Added] Bootstrap the change log ([#4212](https://github.com/quiltdata/quilt/pull/4212)) diff --git a/lambdas/indexer/requirements.txt b/lambdas/indexer/requirements.txt index f8875d3d424..5e739b69913 100644 --- a/lambdas/indexer/requirements.txt +++ b/lambdas/indexer/requirements.txt @@ -1,48 +1,117 @@ -attrs==19.1.0 -aws-requests-auth==0.4.2 -boto3==1.34.41 -botocore==1.34.41 -certifi==2024.7.4 -cffi==1.15.1 -chardet==3.0.4 -charset-normalizer==2.0.12 -cryptography==43.0.1 -decorator==4.4.0 -docutils==0.15.2 -# This is the highest version that's less than our pinned ES version of 6.7 -elasticsearch==6.3.1 -et-xmlfile==1.1.0 -fcsparser==0.2.1 -idna==3.7 -importlib-metadata==6.6.0 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile +# +attrs==24.2.0 + # via jsonschema +aws-requests-auth==0.4.3 + # via es_indexer (setup.py) +boto3==1.35.70 + # via es_indexer (setup.py) +botocore==1.35.70 + # via + # boto3 + # s3transfer +certifi==2024.8.30 + # via requests +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.0 + # via + # pdfminer-six + # requests +cryptography==44.0.0 + # via pdfminer-six +elasticsearch==6.8.2 + # via es_indexer (setup.py) +et-xmlfile==2.0.0 + # via openpyxl +fcsparser==0.2.8 + # via t4-lambda-shared +idna==3.10 + # via requests ipython-genutils==0.2.0 -jmespath==0.9.4 -jsonschema==3.2.0 + # via nbformat +jmespath==1.0.1 + # via + # boto3 + # botocore jsonpointer==2.4 -jupyter-core==4.11.2 -lxml==4.9.2 + # via es_indexer (setup.py) +jsonschema==3.2.0 + # via + # es_indexer (setup.py) + # nbformat + # t4-lambda-shared +jupyter-core==5.7.2 + # via nbformat +lxml==5.3.0 + # via python-pptx nbformat==5.1.3 -numpy==1.22.0 -openpyxl==3.0.7 -pandas==1.3.5 -pdfminer.six==20201018 -Pillow==10.3.0 -psutil==5.7.0 -pyarrow==14.0.1 -pycparser==2.21 -pyrsistent==0.15.4 -python-dateutil==2.8.0 -python-pptx==0.6.21 -pytz==2019.2 -requests==2.32.2 -s3transfer==0.10.0 -six==1.12.0 -sortedcontainers==2.4.0 + # via es_indexer (setup.py) +numpy==1.26.4 + # via + # fcsparser + # pandas +openpyxl==3.1.5 + # via t4-lambda-shared +pandas==2.2.3 + # via + # fcsparser + # t4-lambda-shared +pdfminer-six==20240706 + # via es_indexer (setup.py) +pillow==11.0.0 + # via python-pptx +platformdirs==4.3.6 + # via jupyter-core +psutil==6.1.0 + # via t4-lambda-shared +pyarrow==18.1.0 + # via t4-lambda-shared +pycparser==2.22 + # via cffi +pyrsistent==0.20.0 + # via jsonschema +python-dateutil==2.9.0.post0 + # via + # botocore + # pandas +python-pptx==0.6.23 + # via es_indexer (setup.py) +pytz==2024.2 + # via pandas +requests==2.32.3 + # via aws-requests-auth +s3transfer==0.10.4 + # via boto3 +six==1.16.0 + # via + # jsonschema + # python-dateutil strict-rfc3339==0.7 -tenacity==5.0.3 -traitlets==4.3.2 -typing_extensions==4.5.0 -urllib3==1.26.19 + # via es_indexer (setup.py) +t4-lambda-shared[mem,preview] @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared + # via es_indexer (setup.py) +tenacity==9.0.0 + # via es_indexer (setup.py) +traitlets==5.14.3 + # via + # jupyter-core + # nbformat +tzdata==2024.2 + # via pandas +urllib3==2.2.3 + # via + # botocore + # elasticsearch + # requests xlrd==2.0.1 -XlsxWriter==3.1.0 -zipp==3.19.1 + # via t4-lambda-shared +xlsxwriter==3.2.0 + # via python-pptx + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/lambdas/indexer/setup.py b/lambdas/indexer/setup.py index 116eea2939e..f910e72a81f 100644 --- a/lambdas/indexer/setup.py +++ b/lambdas/indexer/setup.py @@ -4,4 +4,21 @@ name='es_indexer', version='0.0.1', py_modules=['index', 'document_queue'], + install_requires=[ + "aws-requests-auth ~= 0.4.2", + "boto3 ~= 1.34", + "elasticsearch ~= 6.3", + "jsonpointer ~= 2.4", + "jsonschema ~= 3.2", + "nbformat ~= 5.1.3", + "pdfminer.six == 20240706", + "python-pptx ~= 0.6.21", + "strict-rfc3339 ~= 0.7", # for jsonschema format + "tenacity ~= 9.0", + ( + "t4_lambda_shared[mem,preview] @ https://github.com/quiltdata/quilt/archive/" + "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip" + "#subdirectory=lambdas/shared" + ), + ], ) diff --git a/lambdas/indexer/test-requirements.in b/lambdas/indexer/test-requirements.in new file mode 100644 index 00000000000..53716f20fb1 --- /dev/null +++ b/lambdas/indexer/test-requirements.in @@ -0,0 +1,5 @@ +-c requirements.txt +pytest ~= 8.0 +pytest-cov ~= 6.0 +pytest-env ~= 1.1 +responses ~= 0.25.3 diff --git a/lambdas/indexer/test-requirements.txt b/lambdas/indexer/test-requirements.txt index b8fc13134ea..dfb62e2d5a7 100644 --- a/lambdas/indexer/test-requirements.txt +++ b/lambdas/indexer/test-requirements.txt @@ -1,9 +1,48 @@ -atomicwrites==1.3.0 -coverage==5.5 -more-itertools==6.0.0 -pluggy==0.9 -py==1.10.0 -pytest==4.4.0 -pytest-cov==2.6.1 -pytest-env==0.6.2 -responses==0.10.14 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile test-requirements.in +# +certifi==2024.8.30 + # via + # -c requirements.txt + # requests +charset-normalizer==3.4.0 + # via + # -c requirements.txt + # requests +coverage[toml]==7.6.8 + # via pytest-cov +idna==3.10 + # via + # -c requirements.txt + # requests +iniconfig==2.0.0 + # via pytest +packaging==24.2 + # via pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.3 + # via + # -r test-requirements.in + # pytest-cov + # pytest-env +pytest-cov==6.0.0 + # via -r test-requirements.in +pytest-env==1.1.5 + # via -r test-requirements.in +pyyaml==6.0.2 + # via responses +requests==2.32.3 + # via + # -c requirements.txt + # responses +responses==0.25.3 + # via -r test-requirements.in +urllib3==2.2.3 + # via + # -c requirements.txt + # requests + # responses diff --git a/lambdas/pkgevents/.python-version b/lambdas/pkgevents/.python-version index cc1923a40b1..2c0733315e4 100644 --- a/lambdas/pkgevents/.python-version +++ b/lambdas/pkgevents/.python-version @@ -1 +1 @@ -3.8 +3.11 diff --git a/lambdas/pkgevents/CHANGELOG.md b/lambdas/pkgevents/CHANGELOG.md new file mode 100644 index 00000000000..637b3180c37 --- /dev/null +++ b/lambdas/pkgevents/CHANGELOG.md @@ -0,0 +1,21 @@ + +# Changelog + +Changes are listed in reverse chronological order (newer entries at the top). +The entry format is + +```markdown +- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/)) +``` + +where verb is one of + +- Removed +- Added +- Fixed +- Changed + +## Changes + +- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241)) +- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241)) diff --git a/lambdas/pkgevents/requirements.txt b/lambdas/pkgevents/requirements.txt index 5e9065f0043..93afd03f3f3 100644 --- a/lambdas/pkgevents/requirements.txt +++ b/lambdas/pkgevents/requirements.txt @@ -1,7 +1,42 @@ -boto3==1.17.100 -botocore==1.20.100 -jmespath==0.10.0 -python-dateutil==2.8.2 -s3transfer==0.4.2 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile +# +attrs==24.2.0 + # via + # jsonschema + # referencing +boto3==1.35.70 + # via pkgevents (setup.py) +botocore==1.35.70 + # via + # boto3 + # s3transfer +jmespath==1.0.1 + # via + # boto3 + # botocore +jsonschema==4.23.0 + # via t4-lambda-shared +jsonschema-specifications==2024.10.1 + # via jsonschema +python-dateutil==2.9.0.post0 + # via botocore +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.21.0 + # via + # jsonschema + # referencing +s3transfer==0.10.4 + # via boto3 six==1.16.0 -urllib3==1.26.19 + # via python-dateutil +t4-lambda-shared @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared + # via pkgevents (setup.py) +urllib3==2.2.3 + # via botocore diff --git a/lambdas/pkgevents/setup.py b/lambdas/pkgevents/setup.py index 299031e0e67..b8e91100d8d 100644 --- a/lambdas/pkgevents/setup.py +++ b/lambdas/pkgevents/setup.py @@ -4,4 +4,12 @@ name='pkgevents', version='0.0.1', py_modules=['index'], + install_requires=[ + "boto3 ~= 1.34", + ( + "t4_lambda_shared @ https://github.com/quiltdata/quilt/archive/" + "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip" + "#subdirectory=lambdas/shared" + ), + ], ) diff --git a/lambdas/pkgevents/test-requirements.in b/lambdas/pkgevents/test-requirements.in new file mode 100644 index 00000000000..46c6cbd16d0 --- /dev/null +++ b/lambdas/pkgevents/test-requirements.in @@ -0,0 +1,3 @@ +-c requirements.txt +pytest ~= 8.0 +pytest-cov ~= 6.0 diff --git a/lambdas/pkgevents/test-requirements.txt b/lambdas/pkgevents/test-requirements.txt index e69de29bb2d..04ebefd290c 100644 --- a/lambdas/pkgevents/test-requirements.txt +++ b/lambdas/pkgevents/test-requirements.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile test-requirements.in +# +coverage[toml]==7.6.8 + # via pytest-cov +iniconfig==2.0.0 + # via pytest +packaging==24.2 + # via pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.3 + # via + # -r test-requirements.in + # pytest-cov +pytest-cov==6.0.0 + # via -r test-requirements.in diff --git a/lambdas/pkgpush/.python-version b/lambdas/pkgpush/.python-version index cc1923a40b1..2c0733315e4 100644 --- a/lambdas/pkgpush/.python-version +++ b/lambdas/pkgpush/.python-version @@ -1 +1 @@ -3.8 +3.11 diff --git a/lambdas/pkgpush/CHANGELOG.md b/lambdas/pkgpush/CHANGELOG.md index 1251e5c50cf..ef05f7f9e09 100644 --- a/lambdas/pkgpush/CHANGELOG.md +++ b/lambdas/pkgpush/CHANGELOG.md @@ -16,6 +16,7 @@ where verb is one of ## Changes +- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241)) - [Changed] Use per-region scratch buckets ([#3923](https://github.com/quiltdata/quilt/pull/3923)) - [Changed] Speed-up copying of large files during promotion ([#3884](https://github.com/quiltdata/quilt/pull/3884)) - [Changed] Bump quilt3 to set max_pool_connections, this improves performance ([#3870](https://github.com/quiltdata/quilt/pull/3870)) diff --git a/lambdas/pkgpush/requirements.txt b/lambdas/pkgpush/requirements.txt index adb8cd10b8a..b8751ff1a77 100644 --- a/lambdas/pkgpush/requirements.txt +++ b/lambdas/pkgpush/requirements.txt @@ -1,65 +1,59 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile # -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing aws-requests-auth==0.4.3 # via quilt3 -boto3==1.34.31 +boto3==1.35.70 # via # quilt-shared # quilt3 # t4_lambda_pkgpush (setup.py) -boto3-stubs[s3,sts]==1.34.31 +boto3-stubs[s3,sts]==1.35.70 # via quilt-shared -botocore==1.34.31 +botocore==1.35.70 # via # boto3 # s3transfer -botocore-stubs==1.34.31 +botocore-stubs==1.35.70 # via # boto3-stubs # types-aiobotocore -certifi==2024.7.4 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests -idna==3.7 +idna==3.10 # via requests -importlib-resources==6.1.1 - # via - # jsonschema - # jsonschema-specifications jmespath==1.0.1 # via # boto3 # botocore jsonlines==1.2.0 # via quilt3 -jsonschema==4.21.1 +jsonschema==4.23.0 # via quilt3 -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -mypy-boto3-s3==1.34.14 +mypy-boto3-s3==1.35.69 # via boto3-stubs -mypy-boto3-sts==1.34.0 +mypy-boto3-sts==1.35.61 # via boto3-stubs -pkgutil-resolve-name==1.3.10 - # via jsonschema -platformdirs==4.2.0 +platformdirs==4.3.6 # via quilt3 -pydantic==1.10.14 +pydantic==1.10.19 # via # quilt-shared # t4_lambda_pkgpush (setup.py) -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via botocore -pyyaml==6.0.1 +pyyaml==6.0.2 # via quilt3 quilt-shared[boto,pydantic,quilt] @ https://github.com/quiltdata/quilt/archive/7c6edd14fbe8a26613bc26b1bbdc0b956132ef8c.zip#subdirectory=py-shared # via t4_lambda_pkgpush (setup.py) @@ -67,52 +61,49 @@ quilt3 @ https://github.com/quiltdata/quilt/archive/5c2b79128fe4d5d1e6093ff6a7d1 # via # quilt-shared # t4_lambda_pkgpush (setup.py) -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.32.2 +requests==2.32.3 # via # aws-requests-auth # quilt3 # requests-futures requests-futures==1.0.0 # via quilt3 -rpds-py==0.17.1 +rpds-py==0.21.0 # via # jsonschema # referencing -s3transfer==0.10.0 +s3transfer==0.10.4 # via boto3 six==1.16.0 # via # jsonlines # python-dateutil -tenacity==8.2.3 +tenacity==9.0.0 # via quilt3 -tqdm==4.66.3 +tqdm==4.67.1 # via quilt3 -types-aiobotocore[s3]==2.11.1 +types-aiobotocore[s3]==2.15.2.post1 # via quilt-shared -types-aiobotocore-s3==2.11.1 +types-aiobotocore-s3==2.15.2 # via types-aiobotocore -types-awscrt==0.20.3 +types-awscrt==0.23.1 # via botocore-stubs -types-s3transfer==0.10.0 +types-s3transfer==0.10.4 # via boto3-stubs -typing-extensions==4.9.0 +typing-extensions==4.12.2 # via # boto3-stubs - # botocore-stubs # mypy-boto3-s3 # mypy-boto3-sts # pydantic # quilt-shared # types-aiobotocore # types-aiobotocore-s3 -urllib3==1.26.19 +urllib3==2.2.3 # via # botocore # requests -zipp==3.19.1 - # via importlib-resources diff --git a/lambdas/pkgpush/test-requirements.in b/lambdas/pkgpush/test-requirements.in index 797cfec8179..4356e6434b9 100644 --- a/lambdas/pkgpush/test-requirements.in +++ b/lambdas/pkgpush/test-requirements.in @@ -1,4 +1,5 @@ -c requirements.txt pytest ~= 8.0 +pytest-cov ~= 6.0 pytest-mock ~= 3.14 pytest-subtests ~= 0.11 diff --git a/lambdas/pkgpush/test-requirements.txt b/lambdas/pkgpush/test-requirements.txt index a2a1f43d6c9..be283e5a6d4 100644 --- a/lambdas/pkgpush/test-requirements.txt +++ b/lambdas/pkgpush/test-requirements.txt @@ -1,29 +1,30 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile test-requirements.in # -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements.txt # pytest-subtests -exceptiongroup==1.2.0 - # via pytest +coverage[toml]==7.6.8 + # via pytest-cov iniconfig==2.0.0 # via pytest -packaging==23.2 +packaging==24.2 # via pytest -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -pytest==8.0.0 +pytest==8.3.3 # via # -r test-requirements.in + # pytest-cov # pytest-mock # pytest-subtests +pytest-cov==6.0.0 + # via -r test-requirements.in pytest-mock==3.14.0 # via -r test-requirements.in -pytest-subtests==0.11.0 +pytest-subtests==0.13.1 # via -r test-requirements.in -tomli==2.0.1 - # via pytest diff --git a/lambdas/preview/.python-version b/lambdas/preview/.python-version index cc1923a40b1..2c0733315e4 100644 --- a/lambdas/preview/.python-version +++ b/lambdas/preview/.python-version @@ -1 +1 @@ -3.8 +3.11 diff --git a/lambdas/preview/CHANGELOG.md b/lambdas/preview/CHANGELOG.md new file mode 100644 index 00000000000..637b3180c37 --- /dev/null +++ b/lambdas/preview/CHANGELOG.md @@ -0,0 +1,21 @@ + +# Changelog + +Changes are listed in reverse chronological order (newer entries at the top). +The entry format is + +```markdown +- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/)) +``` + +where verb is one of + +- Removed +- Added +- Fixed +- Changed + +## Changes + +- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241)) +- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241)) diff --git a/lambdas/preview/requirements.txt b/lambdas/preview/requirements.txt index 02e1773e9d6..ac0c5adbaae 100644 --- a/lambdas/preview/requirements.txt +++ b/lambdas/preview/requirements.txt @@ -1,50 +1,129 @@ -attrs==19.1.0 -beautifulsoup4==4.12.2 -bleach==3.3.0 -certifi==2024.7.4 -chardet==3.0.4 -charset-normalizer==2.0.12 -decorator==4.4.0 -defusedxml==0.6.0 -entrypoints==0.3 -et-xmlfile==1.1.0 -fastjsonschema==2.16.3 -fcsparser==0.2.1 -idna==3.7 -importlib-metadata==6.6.0 -ipython-genutils==0.2.0 -Jinja2==3.1.4 -jsonschema==3.2.0 -jupyter_client==7.4.9 -jupyter_core==4.11.2 -jupyterlab-pygments==0.2.2 -MarkupSafe==2.1.1 -mistune==2.0.5 -nbclient==0.7.0 -nbconvert==7.2.9 -nbformat==5.4.0 -nest-asyncio==1.5.6 -numpy==1.22.0 -openpyxl==3.0.7 -packaging==23.1 -pandas==1.1.5 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile +# +attrs==24.2.0 + # via + # jsonschema + # referencing +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.2.0 + # via nbconvert +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests +defusedxml==0.7.1 + # via nbconvert +et-xmlfile==2.0.0 + # via openpyxl +fastjsonschema==2.20.0 + # via nbformat +fcsparser==0.2.8 + # via t4-lambda-shared +idna==3.10 + # via requests +jinja2==3.1.4 + # via nbconvert +jsonschema==4.23.0 + # via + # nbformat + # t4-lambda-shared +jsonschema-specifications==2024.10.1 + # via jsonschema +jupyter-client==8.6.3 + # via nbclient +jupyter-core==5.7.2 + # via + # jupyter-client + # nbclient + # nbconvert + # nbformat +jupyterlab-pygments==0.3.0 + # via nbconvert +markupsafe==3.0.2 + # via + # jinja2 + # nbconvert +mistune==3.0.2 + # via nbconvert +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via t4_lambda_preview (setup.py) +nbformat==5.10.4 + # via + # nbclient + # nbconvert + # t4_lambda_preview (setup.py) +numpy==1.26.4 + # via + # fcsparser + # pandas +openpyxl==3.1.5 + # via t4-lambda-shared +packaging==24.2 + # via nbconvert +pandas==2.2.3 + # via + # fcsparser + # t4-lambda-shared + # t4_lambda_preview (setup.py) pandocfilters==1.5.1 -psutil==5.7.0 -pyarrow==14.0.1 -Pygments==2.15.0 -pyrsistent==0.15.2 -python-dateutil==2.8.2 -pytz==2019.1 -pyzmq==25.0.2 -requests==2.32.2 -six==1.12.0 -soupsieve==2.4.1 -testpath==0.4.2 -tinycss2==1.2.1 -tornado==6.4.1 -traitlets==5.3.0 -typing_extensions==4.5.0 -urllib3==1.26.19 + # via nbconvert +platformdirs==4.3.6 + # via jupyter-core +psutil==6.1.0 + # via t4-lambda-shared +pyarrow==18.1.0 + # via t4-lambda-shared +pygments==2.18.0 + # via nbconvert +python-dateutil==2.9.0.post0 + # via + # jupyter-client + # pandas +pytz==2024.2 + # via pandas +pyzmq==26.2.0 + # via jupyter-client +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via t4_lambda_preview (setup.py) +rpds-py==0.21.0 + # via + # jsonschema + # referencing +six==1.16.0 + # via python-dateutil +soupsieve==2.6 + # via beautifulsoup4 +t4-lambda-shared[mem,preview] @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared + # via t4_lambda_preview (setup.py) +tinycss2==1.4.0 + # via nbconvert +tornado==6.4.2 + # via jupyter-client +traitlets==5.14.3 + # via + # jupyter-client + # jupyter-core + # nbclient + # nbconvert + # nbformat +tzdata==2024.2 + # via pandas +urllib3==2.2.3 + # via requests webencodings==0.5.1 + # via + # bleach + # tinycss2 xlrd==2.0.1 -zipp==3.19.1 + # via t4-lambda-shared diff --git a/lambdas/preview/setup.py b/lambdas/preview/setup.py index 696706f94f5..5fa60ccf127 100644 --- a/lambdas/preview/setup.py +++ b/lambdas/preview/setup.py @@ -5,4 +5,15 @@ version='0.0.1', packages=find_packages(where="src"), package_dir={"": "src"}, + install_requires=[ + "nbconvert ~= 7.16", + "nbformat ~= 5.10", + "pandas ~= 2.2", + "requests ~= 2.32", + ( + "t4_lambda_shared[mem,preview] @ https://github.com/quiltdata/quilt/archive/" + "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip" + "#subdirectory=lambdas/shared" + ), + ], ) diff --git a/lambdas/preview/src/t4_lambda_preview/__init__.py b/lambdas/preview/src/t4_lambda_preview/__init__.py index 6b604d239bd..ebf6c34f1fd 100644 --- a/lambdas/preview/src/t4_lambda_preview/__init__.py +++ b/lambdas/preview/src/t4_lambda_preview/__init__.py @@ -6,7 +6,7 @@ """ import io import os -from contextlib import redirect_stderr +import warnings from urllib.parse import urlparse import pandas @@ -178,7 +178,7 @@ def extract_csv(head, separator): html - html version of *first sheet only* in workbook info - metadata """ - warnings_ = io.StringIO() + warnings_ = [] # this shouldn't balloon memory because head is limited in size by get_preview_lines try: data = pandas.read_csv( @@ -187,12 +187,10 @@ def extract_csv(head, separator): ) except pandas.errors.ParserError: - # temporarily redirect stderr to capture warnings (usually errors) - with redirect_stderr(warnings_): + with warnings.catch_warnings(record=True, category=pandas.errors.ParserWarning) as warnings_: data = pandas.read_csv( io.StringIO('\n'.join(head)), - error_bad_lines=False, - warn_bad_lines=True, + on_bad_lines="warn", # sep=None is slower (doesn't use C), deduces the separator sep=None ) @@ -201,7 +199,7 @@ def extract_csv(head, separator): return html, { 'note': TRUNCATED, - 'warnings': warnings_.getvalue() + 'warnings': "\n".join(map(str, warnings_)) } diff --git a/lambdas/preview/test-requirements.in b/lambdas/preview/test-requirements.in new file mode 100644 index 00000000000..0564ff484d6 --- /dev/null +++ b/lambdas/preview/test-requirements.in @@ -0,0 +1,4 @@ +-c requirements.txt +pytest ~= 8.0 +pytest-cov ~= 6.0 +responses ~= 0.25.3 diff --git a/lambdas/preview/test-requirements.txt b/lambdas/preview/test-requirements.txt index 0a1cab4bb17..d5fa24088e1 100644 --- a/lambdas/preview/test-requirements.txt +++ b/lambdas/preview/test-requirements.txt @@ -1,8 +1,47 @@ -atomicwrites==1.3.0 -coverage==5.5 -more-itertools==6.0.0 -pluggy==0.13.1 -py==1.10.0 -pytest==4.6.11 -pytest-cov==2.6.1 -responses==0.10.5 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile test-requirements.in +# +certifi==2024.8.30 + # via + # -c requirements.txt + # requests +charset-normalizer==3.4.0 + # via + # -c requirements.txt + # requests +coverage[toml]==7.6.8 + # via pytest-cov +idna==3.10 + # via + # -c requirements.txt + # requests +iniconfig==2.0.0 + # via pytest +packaging==24.2 + # via + # -c requirements.txt + # pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.3 + # via + # -r test-requirements.in + # pytest-cov +pytest-cov==6.0.0 + # via -r test-requirements.in +pyyaml==6.0.2 + # via responses +requests==2.32.3 + # via + # -c requirements.txt + # responses +responses==0.25.3 + # via -r test-requirements.in +urllib3==2.2.3 + # via + # -c requirements.txt + # requests + # responses diff --git a/lambdas/preview/test/data/fcs/BD - FACS Aria II - Compensation Controls_G710 Stained Control.fcs b/lambdas/preview/test/data/fcs/BD - FACS Aria II - Compensation Controls_G710 Stained Control.fcs new file mode 100644 index 00000000000..941a90c733b Binary files /dev/null and b/lambdas/preview/test/data/fcs/BD - FACS Aria II - Compensation Controls_G710 Stained Control.fcs differ diff --git a/lambdas/preview/test/test_index.py b/lambdas/preview/test/test_index.py index 995fdfe4a14..fec1390351f 100644 --- a/lambdas/preview/test/test_index.py +++ b/lambdas/preview/test/test_index.py @@ -68,7 +68,12 @@ def test_fcs(self): extended = False if ( set(os.path.split(f)[1] for f in fcs_files) - != set(['accuri-ao1.fcs', 'bad.fcs', '3215apc 100004.fcs']) + != { + 'accuri-ao1.fcs', + 'bad.fcs', + '3215apc 100004.fcs', + 'BD - FACS Aria II - Compensation Controls_G710 Stained Control.fcs', + } ): extended = True first = True @@ -98,7 +103,7 @@ def test_fcs(self): assert 'info' in body if 'warnings' not in body['info']: if not extended: - assert name == 'accuri-ao1.fcs' + assert name in ('accuri-ao1.fcs', '3215apc 100004.fcs') assert body['html'].startswith('
') assert body['html'].endswith('
') assert body['info']['metadata'].keys() @@ -110,7 +115,7 @@ def test_fcs(self): assert name == 'bad.fcs' else: if not extended: - assert name == '3215apc 100004.fcs' + assert name == 'BD - FACS Aria II - Compensation Controls_G710 Stained Control.fcs' def test_bad(self): """send a known bad event (no input query parameter)""" @@ -234,7 +239,7 @@ def test_ipynb(self): # check for some strings we know should be in there assert 'SVD of Minute-Market-Data' in body_html, 'missing expected contents' assert 'Preprocessing' in body_html, 'missing expected contents' - assert '
['SEE', 'SE', 'SHW', 'SIG',' in body_html, \
+        assert "
['SEE', 'SE', 'SHW', 'SIG'," in body_html, \
             'Cell 3 output seems off'
         assert (
             'batch_size=100'
@@ -258,7 +263,7 @@ def test_ipynb_chop(self):
         assert resp['statusCode'] == 200, 'preview failed on nb_1200727.ipynb'
         body_html = body['html']
         # isclose bc string sizes differ, e.g. on Linux
-        assert math.isclose(len(body_html), 18084, abs_tol=200), "Hmm, didn't chop nb_1200727.ipynb"
+        assert math.isclose(len(body_html), 18084, abs_tol=300), "Hmm, didn't chop nb_1200727.ipynb"
 
     @responses.activate
     def test_ipynb_exclude(self):
@@ -287,7 +292,7 @@ def test_ipynb_exclude(self):
         # check for some strings we know should be in there
         assert 'SVD of Minute-Market-Data' in body_html, 'missing expected contents'
         assert 'Preprocessing' in body_html, 'missing expected contents'
-        assert '
['SEE', 'SE', 'SHW', 'SIG',' not in body_html, \
+        assert "
['SEE', 'SE', 'SHW', 'SIG'," not in body_html, \
             'Unexpected output cell; exclude_output:true was given'
         assert (
             'batch_size=100'
diff --git a/lambdas/s3hash/.python-version b/lambdas/s3hash/.python-version
index cc1923a40b1..2c0733315e4 100644
--- a/lambdas/s3hash/.python-version
+++ b/lambdas/s3hash/.python-version
@@ -1 +1 @@
-3.8
+3.11
diff --git a/lambdas/s3hash/CHANGELOG.md b/lambdas/s3hash/CHANGELOG.md
index 551f5123164..a9a46c23d0a 100644
--- a/lambdas/s3hash/CHANGELOG.md
+++ b/lambdas/s3hash/CHANGELOG.md
@@ -17,6 +17,7 @@ where verb is one of
 
 ## Changes
 
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
 - [Fixed] Fix invalid checksum for some non-canonical objects with existing checksum ([#4062](https://github.com/quiltdata/quilt/pull/4062))
 - [Changed] Use per-region scratch buckets ([#3923](https://github.com/quiltdata/quilt/pull/3923))
 - [Changed] Always stream bytes in legacy mode ([#3903](https://github.com/quiltdata/quilt/pull/3903))
diff --git a/lambdas/s3hash/requirements.txt b/lambdas/s3hash/requirements.txt
index 277946fa793..de8d735c2b0 100644
--- a/lambdas/s3hash/requirements.txt
+++ b/lambdas/s3hash/requirements.txt
@@ -1,145 +1,139 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile
 #
-aiobotocore==2.11.1
+aiobotocore==2.15.2
     # via t4_lambda_s3hash (setup.py)
-aiohttp==3.10.2
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.11.7
     # via aiobotocore
-aioitertools==0.11.0
+aioitertools==0.12.0
     # via aiobotocore
 aiosignal==1.3.1
     # via aiohttp
-async-timeout==4.0.3
-    # via aiohttp
-attrs==23.2.0
+attrs==24.2.0
     # via
     #   aiohttp
     #   jsonschema
     #   referencing
 aws-requests-auth==0.4.3
     # via quilt3
-boto3==1.34.27
+boto3==1.35.36
     # via
     #   quilt-shared
     #   quilt3
-boto3-stubs[s3,sts]==1.34.31
+boto3-stubs[s3,sts]==1.35.70
     # via quilt-shared
-botocore==1.34.27
+botocore==1.35.36
     # via
     #   aiobotocore
     #   boto3
     #   s3transfer
     #   t4_lambda_s3hash (setup.py)
-botocore-stubs==1.34.31
+botocore-stubs==1.35.70
     # via
     #   boto3-stubs
     #   types-aiobotocore
-certifi==2024.7.4
+certifi==2024.8.30
     # via requests
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
-frozenlist==1.4.1
+frozenlist==1.5.0
     # via
     #   aiohttp
     #   aiosignal
-idna==3.7
+idna==3.10
     # via
     #   requests
     #   yarl
-importlib-resources==6.1.1
-    # via
-    #   jsonschema
-    #   jsonschema-specifications
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
 jsonlines==1.2.0
     # via quilt3
-jsonschema==4.21.1
+jsonschema==4.23.0
     # via quilt3
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
     # via jsonschema
-multidict==6.0.4
+multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
-mypy-boto3-s3==1.34.14
+mypy-boto3-s3==1.35.69
     # via boto3-stubs
-mypy-boto3-sts==1.34.0
+mypy-boto3-sts==1.35.61
     # via boto3-stubs
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
-platformdirs==4.2.0
+platformdirs==4.3.6
     # via quilt3
-pydantic==1.10.14
+propcache==0.2.0
+    # via
+    #   aiohttp
+    #   yarl
+pydantic==1.10.19
     # via
     #   quilt-shared
     #   t4_lambda_s3hash (setup.py)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via botocore
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via quilt3
 quilt-shared[boto,pydantic,quilt] @ https://github.com/quiltdata/quilt/archive/7c6edd14fbe8a26613bc26b1bbdc0b956132ef8c.zip#subdirectory=py-shared
     # via t4_lambda_s3hash (setup.py)
 quilt3==5.4.0
     # via quilt-shared
-referencing==0.33.0
+referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-requests==2.32.2
+requests==2.32.3
     # via
     #   aws-requests-auth
     #   quilt3
     #   requests-futures
 requests-futures==1.0.0
     # via quilt3
-rpds-py==0.17.1
+rpds-py==0.21.0
     # via
     #   jsonschema
     #   referencing
-s3transfer==0.10.0
+s3transfer==0.10.4
     # via boto3
 six==1.16.0
     # via
     #   jsonlines
     #   python-dateutil
-tenacity==8.2.3
+tenacity==9.0.0
     # via quilt3
-tqdm==4.66.3
+tqdm==4.67.1
     # via quilt3
-types-aiobotocore[s3]==2.11.1
+types-aiobotocore[s3]==2.15.2.post1
     # via
     #   quilt-shared
     #   t4_lambda_s3hash (setup.py)
-types-aiobotocore-s3==2.11.1
+types-aiobotocore-s3==2.15.2
     # via types-aiobotocore
-types-awscrt==0.20.3
+types-awscrt==0.23.1
     # via botocore-stubs
-types-s3transfer==0.10.0
+types-s3transfer==0.10.4
     # via boto3-stubs
-typing-extensions==4.9.0
+typing-extensions==4.12.2
     # via
-    #   aioitertools
     #   boto3-stubs
-    #   botocore-stubs
     #   mypy-boto3-s3
     #   mypy-boto3-sts
     #   pydantic
     #   quilt-shared
     #   types-aiobotocore
     #   types-aiobotocore-s3
-urllib3==1.26.19
+urllib3==2.2.3
     # via
     #   botocore
     #   requests
-wrapt==1.16.0
+wrapt==1.17.0
     # via aiobotocore
-yarl==1.9.4
+yarl==1.18.0
     # via aiohttp
-zipp==3.19.1
-    # via importlib-resources
diff --git a/lambdas/s3hash/test-requirements.in b/lambdas/s3hash/test-requirements.in
index 42497209bbd..d46cd837b0a 100644
--- a/lambdas/s3hash/test-requirements.in
+++ b/lambdas/s3hash/test-requirements.in
@@ -1,5 +1,6 @@
 -c requirements.txt
-pytest ~= 7.4
+pytest ~= 8.0
 pytest-asyncio ~= 0.23
+pytest-cov ~= 6.0
 pytest-env ~= 1.1
 pytest-mock ~= 3.12
diff --git a/lambdas/s3hash/test-requirements.txt b/lambdas/s3hash/test-requirements.txt
index 4f6513e93b9..db94f3e19e5 100644
--- a/lambdas/s3hash/test-requirements.txt
+++ b/lambdas/s3hash/test-requirements.txt
@@ -1,30 +1,29 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile test-requirements.in
 #
-exceptiongroup==1.2.0
-    # via pytest
+coverage[toml]==7.6.8
+    # via pytest-cov
 iniconfig==2.0.0
     # via pytest
-packaging==23.2
+packaging==24.2
     # via pytest
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pytest==7.4.4
+pytest==8.3.3
     # via
     #   -r test-requirements.in
     #   pytest-asyncio
+    #   pytest-cov
     #   pytest-env
     #   pytest-mock
-pytest-asyncio==0.23.4
+pytest-asyncio==0.24.0
     # via -r test-requirements.in
-pytest-env==1.1.3
+pytest-cov==6.0.0
     # via -r test-requirements.in
-pytest-mock==3.12.0
+pytest-env==1.1.5
+    # via -r test-requirements.in
+pytest-mock==3.14.0
     # via -r test-requirements.in
-tomli==2.0.1
-    # via
-    #   pytest
-    #   pytest-env
diff --git a/lambdas/scripts/build_zip.sh b/lambdas/scripts/build_zip.sh
index 5a44a5989d3..20c3e8faa99 100755
--- a/lambdas/scripts/build_zip.sh
+++ b/lambdas/scripts/build_zip.sh
@@ -18,7 +18,7 @@ cd out
 pip3 install -U pip setuptools
 
 # install everything into a temporary directory
-pip3 install --no-compile --no-deps -t . /lambda/shared/ -r /lambda/function/requirements.txt /lambda/function/
+pip3 install --no-compile --no-deps -t . -r /lambda/function/requirements.txt /lambda/function/
 python3 -m compileall -b .
 
 # add binaries
diff --git a/lambdas/shared/.python-version b/lambdas/shared/.python-version
index cc1923a40b1..2c0733315e4 100644
--- a/lambdas/shared/.python-version
+++ b/lambdas/shared/.python-version
@@ -1 +1 @@
-3.8
+3.11
diff --git a/lambdas/shared/CHANGELOG.md b/lambdas/shared/CHANGELOG.md
new file mode 100644
index 00000000000..61047b86a71
--- /dev/null
+++ b/lambdas/shared/CHANGELOG.md
@@ -0,0 +1,22 @@
+
+# Changelog
+
+Changes are listed in reverse chronological order (newer entries at the top).
+The entry format is
+
+```markdown
+- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/))
+```
+
+where verb is one of
+
+- Removed
+- Added
+- Fixed
+- Changed
+
+## Changes
+
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
+- [Changed] Specify dependencies explicitly ([#4241](https://github.com/quiltdata/quilt/pull/4241))
+- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241))
diff --git a/lambdas/shared/requirements.txt b/lambdas/shared/requirements.txt
index 207cbc46967..fc5f8bd4a20 100644
--- a/lambdas/shared/requirements.txt
+++ b/lambdas/shared/requirements.txt
@@ -1,11 +1,50 @@
-et-xmlfile==1.1.0
-fcsparser==0.2.1
-numpy==1.22.0
-openpyxl==3.0.7
-pandas==1.1.5
-psutil==5.7.0
-pyarrow==14.0.1
-python-dateutil==2.8.2
-pytz==2023.3
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --extra=mem --extra=preview
+#
+attrs==24.2.0
+    # via
+    #   jsonschema
+    #   referencing
+et-xmlfile==2.0.0
+    # via openpyxl
+fcsparser==0.2.8
+    # via t4-lambda-shared (setup.py)
+jsonschema==4.23.0
+    # via t4-lambda-shared (setup.py)
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+numpy==1.26.4
+    # via
+    #   fcsparser
+    #   pandas
+openpyxl==3.1.5
+    # via t4-lambda-shared (setup.py)
+pandas==2.2.3
+    # via
+    #   fcsparser
+    #   t4-lambda-shared (setup.py)
+psutil==6.1.0
+    # via t4-lambda-shared (setup.py)
+pyarrow==18.1.0
+    # via t4-lambda-shared (setup.py)
+python-dateutil==2.9.0.post0
+    # via pandas
+pytz==2024.2
+    # via pandas
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+rpds-py==0.21.0
+    # via
+    #   jsonschema
+    #   referencing
 six==1.16.0
+    # via python-dateutil
+tzdata==2024.2
+    # via pandas
 xlrd==2.0.1
+    # via t4-lambda-shared (setup.py)
diff --git a/lambdas/shared/setup.py b/lambdas/shared/setup.py
index f1d2fc773f3..29c512dab7e 100644
--- a/lambdas/shared/setup.py
+++ b/lambdas/shared/setup.py
@@ -1,9 +1,14 @@
 from setuptools import setup
 
+mem_deps = [
+    "psutil ~= 6.1",
+]
+
 setup(
     name="t4-lambda-shared",
     version="0.0.3",
     packages=["t4_lambda_shared"],
+    python_requires=">=3.9",
     install_requires=[
         "jsonschema>=2.6.0",
     ],
@@ -12,6 +17,16 @@
             "pytest",
             "pytest-cov",
         ],
+        "mem": mem_deps,  # for t4_lambda_shared.utils.get_available_memory()
+        # for t4_lambda_shared.preview
+        "preview": [
+            *mem_deps,
+            "fcsparser ~= 0.2.1",
+            "openpyxl ~= 3.1",
+            "pandas ~= 2.2",
+            "pyarrow ~= 18.0",
+            "xlrd ~= 2.0",
+        ],
         "lambda": [
             "awslambdaric>=2,<3",
         ],
diff --git a/lambdas/shared/test-requirements.in b/lambdas/shared/test-requirements.in
new file mode 100644
index 00000000000..f515e9af231
--- /dev/null
+++ b/lambdas/shared/test-requirements.in
@@ -0,0 +1,5 @@
+-c requirements.txt
+pytest ~= 8.0
+pytest-cov ~= 6.0
+py-w3c ~= 0.3.1
+testfixtures ~= 6.14.1
diff --git a/lambdas/shared/test-requirements.txt b/lambdas/shared/test-requirements.txt
index a18d82b2d25..0d39fdaf617 100644
--- a/lambdas/shared/test-requirements.txt
+++ b/lambdas/shared/test-requirements.txt
@@ -1,4 +1,24 @@
-boto3==1.17.100
-botocore==1.20.100
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+coverage[toml]==7.6.8
+    # via pytest-cov
+iniconfig==2.0.0
+    # via pytest
+packaging==24.2
+    # via pytest
+pluggy==1.5.0
+    # via pytest
 py-w3c==0.3.1
+    # via -r test-requirements.in
+pytest==8.3.3
+    # via
+    #   -r test-requirements.in
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via -r test-requirements.in
 testfixtures==6.14.1
+    # via -r test-requirements.in
diff --git a/lambdas/status_reports/.python-version b/lambdas/status_reports/.python-version
index cc1923a40b1..2c0733315e4 100644
--- a/lambdas/status_reports/.python-version
+++ b/lambdas/status_reports/.python-version
@@ -1 +1 @@
-3.8
+3.11
diff --git a/lambdas/status_reports/CHANGELOG.md b/lambdas/status_reports/CHANGELOG.md
new file mode 100644
index 00000000000..637b3180c37
--- /dev/null
+++ b/lambdas/status_reports/CHANGELOG.md
@@ -0,0 +1,21 @@
+
+# Changelog
+
+Changes are listed in reverse chronological order (newer entries at the top).
+The entry format is
+
+```markdown
+- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/))
+```
+
+where verb is one of
+
+- Removed
+- Added
+- Fixed
+- Changed
+
+## Changes
+
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
+- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241))
diff --git a/lambdas/status_reports/requirements.txt b/lambdas/status_reports/requirements.txt
index 0b0d0958922..95e42344fe6 100644
--- a/lambdas/status_reports/requirements.txt
+++ b/lambdas/status_reports/requirements.txt
@@ -1,27 +1,25 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile
 #
 aiobotocore==2.3.4
     # via t4_lambda_status_reports (setup.py)
+aiohappyeyeballs==2.4.3
+    # via aiohttp
 aiohttp==3.10.2
     # via aiobotocore
 aioitertools==0.10.0
     # via aiobotocore
 aiosignal==1.2.0
     # via aiohttp
-async-timeout==4.0.2
-    # via aiohttp
 attrs==22.1.0
     # via aiohttp
 botocore==1.24.21
     # via
     #   aiobotocore
     #   t4_lambda_status_reports (setup.py)
-charset-normalizer==2.1.1
-    # via aiohttp
 frozenlist==1.3.1
     # via
     #   aiohttp
@@ -42,8 +40,6 @@ python-dateutil==2.8.2
     # via botocore
 six==1.16.0
     # via python-dateutil
-typing-extensions==4.3.0
-    # via aioitertools
 urllib3==1.26.19
     # via botocore
 wrapt==1.14.1
diff --git a/lambdas/status_reports/test-requirements.in b/lambdas/status_reports/test-requirements.in
new file mode 100644
index 00000000000..46c6cbd16d0
--- /dev/null
+++ b/lambdas/status_reports/test-requirements.in
@@ -0,0 +1,3 @@
+-c requirements.txt
+pytest ~= 8.0
+pytest-cov ~= 6.0
diff --git a/lambdas/status_reports/test-requirements.txt b/lambdas/status_reports/test-requirements.txt
index 6d2e3fee441..04ebefd290c 100644
--- a/lambdas/status_reports/test-requirements.txt
+++ b/lambdas/status_reports/test-requirements.txt
@@ -1,3 +1,20 @@
-coverage==5.5
-pytest==4.3.0
-pytest-cov==2.6.1
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+coverage[toml]==7.6.8
+    # via pytest-cov
+iniconfig==2.0.0
+    # via pytest
+packaging==24.2
+    # via pytest
+pluggy==1.5.0
+    # via pytest
+pytest==8.3.3
+    # via
+    #   -r test-requirements.in
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via -r test-requirements.in
diff --git a/lambdas/tabular_preview/.python-version b/lambdas/tabular_preview/.python-version
index cc1923a40b1..2c0733315e4 100644
--- a/lambdas/tabular_preview/.python-version
+++ b/lambdas/tabular_preview/.python-version
@@ -1 +1 @@
-3.8
+3.11
diff --git a/lambdas/tabular_preview/CHANGELOG.md b/lambdas/tabular_preview/CHANGELOG.md
new file mode 100644
index 00000000000..637b3180c37
--- /dev/null
+++ b/lambdas/tabular_preview/CHANGELOG.md
@@ -0,0 +1,21 @@
+
+# Changelog
+
+Changes are listed in reverse chronological order (newer entries at the top).
+The entry format is
+
+```markdown
+- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/))
+```
+
+where verb is one of
+
+- Removed
+- Added
+- Fixed
+- Changed
+
+## Changes
+
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
+- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241))
diff --git a/lambdas/tabular_preview/requirements.txt b/lambdas/tabular_preview/requirements.txt
index b10ea779083..f628b1031cd 100644
--- a/lambdas/tabular_preview/requirements.txt
+++ b/lambdas/tabular_preview/requirements.txt
@@ -1,74 +1,71 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --output-file=requirements.txt ../shared/setup.py setup.py
+#    pip-compile
 #
-aiohttp==3.10.11
-    # via fsspec
-aiosignal==1.2.0
+aiohappyeyeballs==2.4.3
     # via aiohttp
-async-timeout==4.0.2
+aiohttp==3.11.7
+    # via fsspec
+aiosignal==1.3.1
     # via aiohttp
-attrs==21.4.0
+attrs==24.2.0
     # via
     #   aiohttp
     #   jsonschema
-certifi==2024.7.4
-    # via requests
-charset-normalizer==2.0.12
-    # via
-    #   aiohttp
-    #   requests
-et-xmlfile==1.1.0
+    #   referencing
+et-xmlfile==2.0.0
     # via openpyxl
-frozenlist==1.3.0
+frozenlist==1.5.0
     # via
     #   aiohttp
     #   aiosignal
-fsspec[http]==2022.1.0
-    # via
-    #   fsspec
-    #   t4_lambda_tabular_preview (setup.py)
-idna==3.7
-    # via
-    #   requests
-    #   yarl
-importlib-resources==6.1.1
+fsspec[http]==2024.10.0
+    # via t4_lambda_tabular_preview (setup.py)
+idna==3.10
+    # via yarl
+jsonschema==4.23.0
+    # via t4-lambda-shared
+jsonschema-specifications==2024.10.1
     # via jsonschema
-jsonschema==4.17.3
-    # via t4-lambda-shared (../shared/setup.py)
-multidict==6.0.2
+multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
-numpy==1.22.0
+numpy==1.26.4
     # via
     #   pandas
-    #   pyarrow
-openpyxl==3.0.9
+    #   t4_lambda_tabular_preview (setup.py)
+openpyxl==3.1.5
     # via t4_lambda_tabular_preview (setup.py)
-pandas==1.3.5
+pandas==2.2.3
     # via t4_lambda_tabular_preview (setup.py)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
-pyarrow==14.0.1
+propcache==0.2.0
+    # via
+    #   aiohttp
+    #   yarl
+pyarrow==18.1.0
     # via t4_lambda_tabular_preview (setup.py)
-pyrsistent==0.20.0
-    # via jsonschema
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via pandas
-pytz==2021.3
+pytz==2024.2
     # via pandas
-requests==2.32.2
-    # via fsspec
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+rpds-py==0.21.0
+    # via
+    #   jsonschema
+    #   referencing
 six==1.16.0
     # via python-dateutil
-urllib3==1.26.19
-    # via requests
+t4-lambda-shared @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared
+    # via t4_lambda_tabular_preview (setup.py)
+tzdata==2024.2
+    # via pandas
 xlrd==2.0.1
     # via t4_lambda_tabular_preview (setup.py)
-yarl==1.7.2
+yarl==1.18.0
     # via aiohttp
-zipp==3.19.1
-    # via importlib-resources
diff --git a/lambdas/tabular_preview/setup.py b/lambdas/tabular_preview/setup.py
index 645756f3c3c..c7333b3ce2b 100644
--- a/lambdas/tabular_preview/setup.py
+++ b/lambdas/tabular_preview/setup.py
@@ -6,10 +6,20 @@
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     install_requires=[
-        "pyarrow>=7,<15",
-        "pandas>=1.3,<1.4",
-        "xlrd>=2,<3",
-        "openpyxl>=3,<4",
-        "fsspec[http]>=2022.1.0",
+        "pyarrow ~= 18.0",
+        "pandas ~= 2.2",
+        "xlrd >=2,< 3",
+        "openpyxl >=3,<4 ",
+        "fsspec[http] >= 2022.1.0",
+        # Stripping numpy.libs in numpy 2 results in
+        # libscipy_openblas64_-ff651d7f.so: ELF load command address/offset not properly aligned
+        # which is probably caused by some bug in glibc (it happens on Ubuntu 22.04, Amazon Linux 2/2023,
+        # but not on Ubuntu 24.04).
+        "numpy < 2",
+        (
+            "t4_lambda_shared @ https://github.com/quiltdata/quilt/archive/"
+            "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip"
+            "#subdirectory=lambdas/shared"
+        ),
     ],
 )
diff --git a/lambdas/tabular_preview/test-requirements.in b/lambdas/tabular_preview/test-requirements.in
new file mode 100644
index 00000000000..46c6cbd16d0
--- /dev/null
+++ b/lambdas/tabular_preview/test-requirements.in
@@ -0,0 +1,3 @@
+-c requirements.txt
+pytest ~= 8.0
+pytest-cov ~= 6.0
diff --git a/lambdas/tabular_preview/test-requirements.txt b/lambdas/tabular_preview/test-requirements.txt
index 927094516e6..04ebefd290c 100644
--- a/lambdas/tabular_preview/test-requirements.txt
+++ b/lambdas/tabular_preview/test-requirements.txt
@@ -1 +1,20 @@
-pytest==6.2.5
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+coverage[toml]==7.6.8
+    # via pytest-cov
+iniconfig==2.0.0
+    # via pytest
+packaging==24.2
+    # via pytest
+pluggy==1.5.0
+    # via pytest
+pytest==8.3.3
+    # via
+    #   -r test-requirements.in
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via -r test-requirements.in
diff --git a/lambdas/thumbnail/CHANGELOG.md b/lambdas/thumbnail/CHANGELOG.md
index 04ddfb6cd77..eaeb145f7d1 100644
--- a/lambdas/thumbnail/CHANGELOG.md
+++ b/lambdas/thumbnail/CHANGELOG.md
@@ -16,4 +16,5 @@ where verb is one of
 
 ## Changes
 
+- [Changed] Regenerate requirements.txt ([#4241](https://github.com/quiltdata/quilt/pull/4241))
 - [Added] Allow overriding PIL.Image.MAX_IMAGE_PIXELS using env var ([#4100](https://github.com/quiltdata/quilt/pull/4100))
diff --git a/lambdas/thumbnail/Dockerfile b/lambdas/thumbnail/Dockerfile
index 6745394fef2..439a2f65e80 100644
--- a/lambdas/thumbnail/Dockerfile
+++ b/lambdas/thumbnail/Dockerfile
@@ -16,16 +16,15 @@ RUN apt-get update && \
         python3-dev \
         python3-pip
 
-COPY thumbnail/requirements.txt /requirements/thumbnail.txt
+COPY requirements.txt /requirements/thumbnail.txt
 # HACK: Pre-install numpy v1 as a build dependency for tifffile to prevent it from using v2 and failing to build
 RUN pip install -U pip setuptools 'numpy<2'
 RUN pip install --target /deps -r /requirements/thumbnail.txt
 RUN curl --output /deps/unoconv \
     https://raw.githubusercontent.com/unoconv/unoconv/be5301a757552f4ecac5d73187ce4d8e18341306/unoconv
 
-COPY shared/ /src/shared/
-COPY thumbnail/ /src/thumbnail/
-RUN pip install --target /lambda --no-deps /src/shared/ /src/thumbnail/
+COPY . /src/thumbnail/
+RUN pip install --target /lambda --no-deps /src/thumbnail/
 
 FROM base-image
 
diff --git a/lambdas/thumbnail/pyproject.toml b/lambdas/thumbnail/pyproject.toml
index 94a6ac47933..9727edf94a4 100644
--- a/lambdas/thumbnail/pyproject.toml
+++ b/lambdas/thumbnail/pyproject.toml
@@ -1,14 +1,15 @@
 [project]
 name = "t4_lambda_thumbnail"
 version = "0.0.1"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 
 dependencies = [
   "aicsimageio>=3,<3.1",
-  "imageio>=2.10.0,<3",
+  "imageio>=2.10.0,<=2.25.0",  # newer versions fail because of missing tifffile.tifffile.TiffFileError in tifffile==0.15.1
   "numpy>=1.21.5,<2",
   "pdf2image>=1.13.1,<2",
   "Pillow>=9.0.1,<11",
   "python-pptx>=0.6.21,<0.7",
   "requests>=2.26.0,<3",
+  "t4_lambda_shared[lambda] @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared",
 ]
diff --git a/lambdas/thumbnail/requirements.txt b/lambdas/thumbnail/requirements.txt
index 45032664a0d..c825e57a82e 100644
--- a/lambdas/thumbnail/requirements.txt
+++ b/lambdas/thumbnail/requirements.txt
@@ -2,56 +2,68 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --extra=lambda --output-file=requirements.txt ../shared/setup.py pyproject.toml
+#    pip-compile
 #
 aicsimageio==3.0.7
-    # via t4-lambda-thumbnail (pyproject.toml)
-attrs==22.2.0
-    # via jsonschema
-awslambdaric==2.0.4
-    # via t4-lambda-shared (../shared/setup.py)
-certifi==2024.7.4
+    # via t4_lambda_thumbnail (pyproject.toml)
+attrs==24.2.0
+    # via
+    #   jsonschema
+    #   referencing
+awslambdaric==2.2.1
+    # via t4-lambda-shared
+certifi==2024.8.30
     # via requests
-charset-normalizer==3.0.1
+charset-normalizer==3.4.0
     # via requests
-idna==3.7
+idna==3.10
     # via requests
 imageio==2.25.0
     # via
     #   aicsimageio
-    #   t4-lambda-thumbnail (pyproject.toml)
-jsonschema==4.17.3
-    # via t4-lambda-shared (../shared/setup.py)
-lxml==4.9.2
+    #   t4_lambda_thumbnail (pyproject.toml)
+jsonschema==4.23.0
+    # via t4-lambda-shared
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+lxml==5.3.0
     # via python-pptx
-numpy==1.24.1
+numpy==1.26.4
     # via
     #   aicsimageio
     #   imageio
     #   scipy
-    #   t4-lambda-thumbnail (pyproject.toml)
+    #   t4_lambda_thumbnail (pyproject.toml)
     #   tifffile
-pdf2image==1.16.2
-    # via t4-lambda-thumbnail (pyproject.toml)
-pillow==10.3.0
+pdf2image==1.17.0
+    # via t4_lambda_thumbnail (pyproject.toml)
+pillow==10.4.0
     # via
     #   imageio
     #   pdf2image
     #   python-pptx
-    #   t4-lambda-thumbnail (pyproject.toml)
-pyrsistent==0.19.3
-    # via jsonschema
-python-pptx==0.6.21
-    # via t4-lambda-thumbnail (pyproject.toml)
-requests==2.32.2
-    # via t4-lambda-thumbnail (pyproject.toml)
-scipy==1.10.0
+    #   t4_lambda_thumbnail (pyproject.toml)
+python-pptx==0.6.23
+    # via t4_lambda_thumbnail (pyproject.toml)
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.32.3
+    # via t4_lambda_thumbnail (pyproject.toml)
+rpds-py==0.21.0
+    # via
+    #   jsonschema
+    #   referencing
+scipy==1.13.1
     # via aicsimageio
-simplejson==3.17.2
+simplejson==3.19.3
     # via awslambdaric
+t4-lambda-shared[lambda] @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared
+    # via t4_lambda_thumbnail (pyproject.toml)
 tifffile==0.15.1
     # via aicsimageio
-urllib3==1.26.19
+urllib3==2.2.3
     # via requests
-xlsxwriter==3.0.7
+xlsxwriter==3.2.0
     # via python-pptx
diff --git a/lambdas/thumbnail/test-requirements.in b/lambdas/thumbnail/test-requirements.in
new file mode 100644
index 00000000000..0564ff484d6
--- /dev/null
+++ b/lambdas/thumbnail/test-requirements.in
@@ -0,0 +1,4 @@
+-c requirements.txt
+pytest ~= 8.0
+pytest-cov ~= 6.0
+responses ~= 0.25.3
diff --git a/lambdas/thumbnail/test-requirements.txt b/lambdas/thumbnail/test-requirements.txt
index 61551db17ab..b138e55bb14 100644
--- a/lambdas/thumbnail/test-requirements.txt
+++ b/lambdas/thumbnail/test-requirements.txt
@@ -1,8 +1,51 @@
-atomicwrites==1.3.0
-more-itertools==6.0.0
-pluggy==0.9.0
-py==1.10.0
-pytest==4.3.0
-pytest-cov==2.6.1
-pytest-raises==0.10
-responses==0.12.0
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+certifi==2024.8.30
+    # via
+    #   -c requirements.txt
+    #   requests
+charset-normalizer==3.4.0
+    # via
+    #   -c requirements.txt
+    #   requests
+coverage[toml]==7.6.8
+    # via pytest-cov
+exceptiongroup==1.2.2
+    # via pytest
+idna==3.10
+    # via
+    #   -c requirements.txt
+    #   requests
+iniconfig==2.0.0
+    # via pytest
+packaging==24.2
+    # via pytest
+pluggy==1.5.0
+    # via pytest
+pytest==8.3.3
+    # via
+    #   -r test-requirements.in
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via -r test-requirements.in
+pyyaml==6.0.2
+    # via responses
+requests==2.32.3
+    # via
+    #   -c requirements.txt
+    #   responses
+responses==0.25.3
+    # via -r test-requirements.in
+tomli==2.1.0
+    # via
+    #   coverage
+    #   pytest
+urllib3==2.2.3
+    # via
+    #   -c requirements.txt
+    #   requests
+    #   responses
diff --git a/lambdas/transcode/.python-version b/lambdas/transcode/.python-version
index cc1923a40b1..2c0733315e4 100644
--- a/lambdas/transcode/.python-version
+++ b/lambdas/transcode/.python-version
@@ -1 +1 @@
-3.8
+3.11
diff --git a/lambdas/transcode/CHANGELOG.md b/lambdas/transcode/CHANGELOG.md
new file mode 100644
index 00000000000..637b3180c37
--- /dev/null
+++ b/lambdas/transcode/CHANGELOG.md
@@ -0,0 +1,21 @@
+
+# Changelog
+
+Changes are listed in reverse chronological order (newer entries at the top).
+The entry format is
+
+```markdown
+- [Verb] Change description ([#](https://github.com/quiltdata/quilt/pull/))
+```
+
+where verb is one of
+
+- Removed
+- Added
+- Fixed
+- Changed
+
+## Changes
+
+- [Changed] Upgrade to Python 3.11 ([#4241](https://github.com/quiltdata/quilt/pull/4241))
+- [Added] Bootstrap the change log ([#4241](https://github.com/quiltdata/quilt/pull/4241))
diff --git a/lambdas/transcode/requirements.txt b/lambdas/transcode/requirements.txt
index d8a4e21246c..66440e4b671 100644
--- a/lambdas/transcode/requirements.txt
+++ b/lambdas/transcode/requirements.txt
@@ -1,4 +1,24 @@
-attrs==21.2.0
-jsonschema==3.2.0
-pyrsistent==0.18.0
-six==1.16.0
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile
+#
+attrs==24.2.0
+    # via
+    #   jsonschema
+    #   referencing
+jsonschema==4.23.0
+    # via t4-lambda-shared
+jsonschema-specifications==2024.10.1
+    # via jsonschema
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+rpds-py==0.21.0
+    # via
+    #   jsonschema
+    #   referencing
+t4-lambda-shared @ https://github.com/quiltdata/quilt/archive/f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip#subdirectory=lambdas/shared
+    # via t4_lambda_transcode (setup.py)
diff --git a/lambdas/transcode/setup.py b/lambdas/transcode/setup.py
index ba2d05a4ca8..df4033d1708 100644
--- a/lambdas/transcode/setup.py
+++ b/lambdas/transcode/setup.py
@@ -4,4 +4,11 @@
     name='t4_lambda_transcode',
     version='0.0.1',
     py_modules=['index'],
+    install_requires=[
+        (
+            "t4_lambda_shared @ https://github.com/quiltdata/quilt/archive/"
+            "f45d8ab51f2d60e98efda7510322f94d822e4eb4.zip"
+            "#subdirectory=lambdas/shared"
+        ),
+    ],
 )
diff --git a/lambdas/transcode/test-requirements.in b/lambdas/transcode/test-requirements.in
new file mode 100644
index 00000000000..46c6cbd16d0
--- /dev/null
+++ b/lambdas/transcode/test-requirements.in
@@ -0,0 +1,3 @@
+-c requirements.txt
+pytest ~= 8.0
+pytest-cov ~= 6.0
diff --git a/lambdas/transcode/test-requirements.txt b/lambdas/transcode/test-requirements.txt
index 28115a45291..04ebefd290c 100644
--- a/lambdas/transcode/test-requirements.txt
+++ b/lambdas/transcode/test-requirements.txt
@@ -1,7 +1,20 @@
-atomicwrites==1.3.0
-more-itertools==6.0.0
-pluggy==0.9.0
-py==1.10.0
-pytest==4.3.0
-pytest-cov==2.6.1
-pytest-raises==0.10
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile test-requirements.in
+#
+coverage[toml]==7.6.8
+    # via pytest-cov
+iniconfig==2.0.0
+    # via pytest
+packaging==24.2
+    # via pytest
+pluggy==1.5.0
+    # via pytest
+pytest==8.3.3
+    # via
+    #   -r test-requirements.in
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via -r test-requirements.in