From ffe7d351e3b97b8c211f065cb06b6b4a9888e908 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 21 Feb 2024 15:00:21 +0000 Subject: [PATCH 01/54] Add initial workers --- .github/workflows/deploy.functions.yml | 111 +++++++++++++++++++++++++ .github/workflows/deploy.webapp.yml | 51 ++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 .github/workflows/deploy.functions.yml create mode 100644 .github/workflows/deploy.webapp.yml diff --git a/.github/workflows/deploy.functions.yml b/.github/workflows/deploy.functions.yml new file mode 100644 index 000000000..57b9da633 --- /dev/null +++ b/.github/workflows/deploy.functions.yml @@ -0,0 +1,111 @@ +# Docs for the Azure Web Apps Deploy action: https://github.com/azure/functions-action +# More GitHub Actions for Azure: https://github.com/Azure/actions + +name: Build and deploy Python project to Azure Function App - ccom-funcs-dev + +on: + push: + branches: + - master + workflow_dispatch: + +env: + AZURE_FUNCTIONAPP_PACKAGE_PATH: './app/workers' + PYTHON_VERSION: '3.8' # set this to the python version to use (supports 3.6, 3.7, 3.8) + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: 'Checkout GitHub Action' + uses: actions/checkout@v2 + + - name: Setup Python ${{ env.PYTHON_VERSION }} Environment + uses: actions/setup-python@v1 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: 'Install Poetry' + uses: snok/install-poetry@v1 + + - name: 'Resolve Project Dependencies Using Poetry' + run: | + pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' + poetry config virtualenvs.create false + poetry export --format requirements.txt --output requirements.txt + popd + shell: bash + + - name: 'Install Dependencies Using Pip' + shell: bash + run: | + pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' + python -m pip install --upgrade pip + pip install -r requirements.txt --target=".python_packages/lib/site-packages" + popd + + - name: Upload artifact for deployment job + uses: actions/upload-artifact@v4 + with: + name: functionsapp + path: ${{ env.AZURE_WEBAPP_PACKAGE_PATH }} + + # Deploy Dev + deploy-dev: + runs-on: ubuntu-latest + needs: build + environment: dev + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure webapp deploy action using publish profile credentials' + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + package: ${{ github.workspace }}/functionsapp + + # Deploy Test + deploy-test: + runs-on: ubuntu-latest + needs: deploy-dev + environment: test + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure webapp deploy action using publish profile credentials' + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + package: ${{ github.workspace }}/functionsapp + + # Deploy Production + deploy-prod: + runs-on: ubuntu-latest + needs: deploy-test + environment: production + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure webapp deploy action using publish profile credentials' + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + package: ${{ github.workspace }}/functionsapp diff --git a/.github/workflows/deploy.webapp.yml b/.github/workflows/deploy.webapp.yml new file mode 100644 index 000000000..c8ff80ea1 --- /dev/null +++ b/.github/workflows/deploy.webapp.yml @@ -0,0 +1,51 @@ +# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy +# More GitHub Actions for Azure: https://github.com/Azure/actions + +name: Build and deploy container app to Azure Web App - ccom + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + build: + runs-on: 'ubuntu-latest' + + steps: + - uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to registry + uses: docker/login-action@v1 + with: + registry: https://ccomregprod.azurecr.io/ + username: ${{ secrets.REGISTRY_USERNAME_PROD }} + password: ${{ secrets.REGISTRY_PASSWORD_PROD }} + + - name: Build and push container image to registry + uses: docker/build-push-action@v2 + with: + push: true + tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/prod:${{ github.sha }} + file: ./app/Dockerfile.deploy + + deploy: + runs-on: ubuntu-latest + needs: build + environment: + name: 'production' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom' + slot-name: 'production' + publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_PROD }} + images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/prod:${{ github.sha }}' From 0e9f509ba4010c7d28127fd17f2344c11c3095cb Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:12:27 +0000 Subject: [PATCH 02/54] Add labeller --- .github/labeler.yml | 22 ++++++++++++++++++++++ .github/workflows/label.yml | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/label.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..c78b3ed1e --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,22 @@ +"Area: Migrations": + - 'app/api/mapping/migrations/*' + +dependencies: + - 'package.json' + - 'package-lock.json' + - 'poetry.lock' + - 'project.toml' + +"Area: CI": + - '.github/**/*' + +"Area: Backend": + - 'app/api/**' + - 'lib/Data/**' + - 'lib/Monitor.Shared/**' + +"Area: Frontend": + - 'app/react-client-app/**' + +"Area: Workers": + - 'app/workers/**' diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml new file mode 100644 index 000000000..a8a1bd725 --- /dev/null +++ b/.github/workflows/label.yml @@ -0,0 +1,22 @@ +# This workflow will triage pull requests and apply a label based on the +# paths that are modified in the pull request. +# +# To use this workflow, you will need to set up a .github/labeler.yml +# file with configuration. For more information, see: +# https://github.com/actions/labeler + +name: Labeler +on: [pull_request] + +jobs: + label: + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + + steps: + - uses: actions/labeler@v4 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" From c80d018f17178285067bff9ba354a5b95bbf4fd7 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:12:55 +0000 Subject: [PATCH 03/54] Add pytest CI --- .github/workflows/test.yml | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..0eba066ea --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: test + +on: [pull_request] + +jobs: + test-workers: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.11'] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Test with pytest Poetry + run: | + poetry run pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=sources app/worker/test | tee pytest-coverage.txt + + - name: Pytest coverage comment + uses: MishaKav/pytest-coverage-comment@main + with: + pytest-coverage-path: pytest-coverage.txt + junitxml-path: pytest.xml From c045f057d0b13d66ea229e8a7faa98beac8fa416 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:30:55 +0000 Subject: [PATCH 04/54] Add django test run --- .github/workflows/test.yml | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0eba066ea..8bf5afff7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,3 +41,42 @@ jobs: with: pytest-coverage-path: pytest-coverage.txt junitxml-path: pytest.xml + + test-django: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.11'] + django-version: [ "3.2", "4.2", "5" ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install django ${{ matrix.django-version }} + run: | + source .venv/bin/activate + pip install "Django==${{ matrix.django-version }}" + + - name: Test with Django + run: | + poetry run python manage.py test mapper -v2 From fee1a4ed6ffc149ac8f41552f9c60ae6973bf5d9 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:33:46 +0000 Subject: [PATCH 05/54] Update labeller --- .github/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index c78b3ed1e..62722dd48 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,7 +5,7 @@ dependencies: - 'package.json' - 'package-lock.json' - 'poetry.lock' - - 'project.toml' + - 'pyproject.toml' "Area: CI": - '.github/**/*' From 49091a6164e513a6257c795d47c53b8ce774b26b Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:39:34 +0000 Subject: [PATCH 06/54] Add test directories --- .github/workflows/test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8bf5afff7..5f1a3e473 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,11 +30,13 @@ jobs: - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + working-directory: app/workers run: poetry install --no-interaction --no-root - name: Test with pytest Poetry + working-directory: app/workers run: | - poetry run pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=sources app/worker/test | tee pytest-coverage.txt + poetry run pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=sources test | tee pytest-coverage.txt - name: Pytest coverage comment uses: MishaKav/pytest-coverage-comment@main @@ -70,6 +72,7 @@ jobs: - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + working-directory: app/api run: poetry install --no-interaction --no-root - name: Install django ${{ matrix.django-version }} @@ -78,5 +81,6 @@ jobs: pip install "Django==${{ matrix.django-version }}" - name: Test with Django + working-directory: app/api run: | - poetry run python manage.py test mapper -v2 + poetry run python manage.py test mapping -v2 From cfc16d1d8e7536e6e5063c59a397debebed36692 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:43:35 +0000 Subject: [PATCH 07/54] Add a sample test --- app/workers/poetry.lock | 168 +++++++++++++++++++++++++++++++++- app/workers/pyproject.toml | 6 ++ app/workers/test/test_pass.py | 2 + 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 app/workers/test/test_pass.py diff --git a/app/workers/poetry.lock b/app/workers/poetry.lock index eab36e34a..b37da5d66 100644 --- a/app/workers/poetry.lock +++ b/app/workers/poetry.lock @@ -275,6 +275,84 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.4.3" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8580b827d4746d47294c0e0b92854c85a92c2227927433998f0d3320ae8a71b6"}, + {file = "coverage-7.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:718187eeb9849fc6cc23e0d9b092bc2348821c5e1a901c9f8975df0bc785bfd4"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:767b35c3a246bcb55b8044fd3a43b8cd553dd1f9f2c1eeb87a302b1f8daa0524"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae7f19afe0cce50039e2c782bff379c7e347cba335429678450b8fe81c4ef96d"}, + {file = "coverage-7.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba3a8aaed13770e970b3df46980cb068d1c24af1a1968b7818b69af8c4347efb"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ee866acc0861caebb4f2ab79f0b94dbfbdbfadc19f82e6e9c93930f74e11d7a0"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:506edb1dd49e13a2d4cac6a5173317b82a23c9d6e8df63efb4f0380de0fbccbc"}, + {file = "coverage-7.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd6545d97c98a192c5ac995d21c894b581f1fd14cf389be90724d21808b657e2"}, + {file = "coverage-7.4.3-cp310-cp310-win32.whl", hash = "sha256:f6a09b360d67e589236a44f0c39218a8efba2593b6abdccc300a8862cffc2f94"}, + {file = "coverage-7.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:18d90523ce7553dd0b7e23cbb28865db23cddfd683a38fb224115f7826de78d0"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbbe5e739d45a52f3200a771c6d2c7acf89eb2524890a4a3aa1a7fa0695d2a47"}, + {file = "coverage-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:489763b2d037b164846ebac0cbd368b8a4ca56385c4090807ff9fad817de4113"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:451f433ad901b3bb00184d83fd83d135fb682d780b38af7944c9faeecb1e0bfe"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcc66e222cf4c719fe7722a403888b1f5e1682d1679bd780e2b26c18bb648cdc"}, + {file = "coverage-7.4.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ec74cfef2d985e145baae90d9b1b32f85e1741b04cd967aaf9cfa84c1334f3"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:abbbd8093c5229c72d4c2926afaee0e6e3140de69d5dcd918b2921f2f0c8baba"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:35eb581efdacf7b7422af677b92170da4ef34500467381e805944a3201df2079"}, + {file = "coverage-7.4.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8249b1c7334be8f8c3abcaaa996e1e4927b0e5a23b65f5bf6cfe3180d8ca7840"}, + {file = "coverage-7.4.3-cp311-cp311-win32.whl", hash = "sha256:cf30900aa1ba595312ae41978b95e256e419d8a823af79ce670835409fc02ad3"}, + {file = "coverage-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:18c7320695c949de11a351742ee001849912fd57e62a706d83dfc1581897fa2e"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b51bfc348925e92a9bd9b2e48dad13431b57011fd1038f08316e6bf1df107d10"}, + {file = "coverage-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d6cdecaedea1ea9e033d8adf6a0ab11107b49571bbb9737175444cea6eb72328"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b2eccb883368f9e972e216c7b4c7c06cabda925b5f06dde0650281cb7666a30"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c00cdc8fa4e50e1cc1f941a7f2e3e0f26cb2a1233c9696f26963ff58445bac7"}, + {file = "coverage-7.4.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9a4a8dd3dcf4cbd3165737358e4d7dfbd9d59902ad11e3b15eebb6393b0446e"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:062b0a75d9261e2f9c6d071753f7eef0fc9caf3a2c82d36d76667ba7b6470003"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ebe7c9e67a2d15fa97b77ea6571ce5e1e1f6b0db71d1d5e96f8d2bf134303c1d"}, + {file = "coverage-7.4.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0a120238dd71c68484f02562f6d446d736adcc6ca0993712289b102705a9a3a"}, + {file = "coverage-7.4.3-cp312-cp312-win32.whl", hash = "sha256:37389611ba54fd6d278fde86eb2c013c8e50232e38f5c68235d09d0a3f8aa352"}, + {file = "coverage-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:d25b937a5d9ffa857d41be042b4238dd61db888533b53bc76dc082cb5a15e914"}, + {file = "coverage-7.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:28ca2098939eabab044ad68850aac8f8db6bf0b29bc7f2887d05889b17346454"}, + {file = "coverage-7.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:280459f0a03cecbe8800786cdc23067a8fc64c0bd51dc614008d9c36e1659d7e"}, + {file = "coverage-7.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c0cdedd3500e0511eac1517bf560149764b7d8e65cb800d8bf1c63ebf39edd2"}, + {file = "coverage-7.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a9babb9466fe1da12417a4aed923e90124a534736de6201794a3aea9d98484e"}, + {file = "coverage-7.4.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dec9de46a33cf2dd87a5254af095a409ea3bf952d85ad339751e7de6d962cde6"}, + {file = "coverage-7.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:16bae383a9cc5abab9bb05c10a3e5a52e0a788325dc9ba8499e821885928968c"}, + {file = "coverage-7.4.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2c854ce44e1ee31bda4e318af1dbcfc929026d12c5ed030095ad98197eeeaed0"}, + {file = "coverage-7.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ce8c50520f57ec57aa21a63ea4f325c7b657386b3f02ccaedeccf9ebe27686e1"}, + {file = "coverage-7.4.3-cp38-cp38-win32.whl", hash = "sha256:708a3369dcf055c00ddeeaa2b20f0dd1ce664eeabde6623e516c5228b753654f"}, + {file = "coverage-7.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:1bf25fbca0c8d121a3e92a2a0555c7e5bc981aee5c3fdaf4bb7809f410f696b9"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3b253094dbe1b431d3a4ac2f053b6d7ede2664ac559705a704f621742e034f1f"}, + {file = "coverage-7.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77fbfc5720cceac9c200054b9fab50cb2a7d79660609200ab83f5db96162d20c"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6679060424faa9c11808598504c3ab472de4531c571ab2befa32f4971835788e"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4af154d617c875b52651dd8dd17a31270c495082f3d55f6128e7629658d63765"}, + {file = "coverage-7.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8640f1fde5e1b8e3439fe482cdc2b0bb6c329f4bb161927c28d2e8879c6029ee"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:69b9f6f66c0af29642e73a520b6fed25ff9fd69a25975ebe6acb297234eda501"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0842571634f39016a6c03e9d4aba502be652a6e4455fadb73cd3a3a49173e38f"}, + {file = "coverage-7.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a78ed23b08e8ab524551f52953a8a05d61c3a760781762aac49f8de6eede8c45"}, + {file = "coverage-7.4.3-cp39-cp39-win32.whl", hash = "sha256:c0524de3ff096e15fcbfe8f056fdb4ea0bf497d584454f344d59fce069d3e6e9"}, + {file = "coverage-7.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0209a6369ccce576b43bb227dc8322d8ef9e323d089c6f3f26a597b09cb4d2aa"}, + {file = "coverage-7.4.3-pp38.pp39.pp310-none-any.whl", hash = "sha256:7cbde573904625509a3f37b6fecea974e363460b556a627c60dc2f47e2fffa51"}, + {file = "coverage-7.4.3.tar.gz", hash = "sha256:276f6077a5c61447a48d133ed13e759c09e62aff0dc84274a68dc18660104d52"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + [[package]] name = "cryptography" version = "42.0.3" @@ -439,6 +517,17 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff", "zipp (>=3.17)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isodate" version = "0.6.1" @@ -581,6 +670,17 @@ files = [ [package.dependencies] et-xmlfile = "*" +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + [[package]] name = "pkgutil-resolve-name" version = "1.3.10" @@ -592,6 +692,21 @@ files = [ {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, ] +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "psutil" version = "5.9.8" @@ -631,6 +746,46 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +[[package]] +name = "pytest" +version = "8.0.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.2-py3-none-any.whl", hash = "sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096"}, + {file = "pytest-8.0.2.tar.gz", hash = "sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "4.1.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + [[package]] name = "python-easyconfig" version = "0.1.7" @@ -895,6 +1050,17 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -941,4 +1107,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "cc4ff74325beeab1ab79aa21b5b6055ee393c2dc5f408620900b495c7a34e0d4" +content-hash = "53dd5480be518ca031a2523481f1c7effacd6c62e3e99947c6d280052a408926" diff --git a/app/workers/pyproject.toml b/app/workers/pyproject.toml index 3c953cd6a..05b9fecb9 100644 --- a/app/workers/pyproject.toml +++ b/app/workers/pyproject.toml @@ -18,6 +18,12 @@ httpx = "^0.26.0" [tool.poetry.group.dev.dependencies] memory-profiler = "^0.61.0" + +[tool.poetry.group.test.dependencies] +pytest = "^8.0.2" +coverage = "^7.4.3" +pytest-cov = "^4.1.0" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/app/workers/test/test_pass.py b/app/workers/test/test_pass.py new file mode 100644 index 000000000..269ad912e --- /dev/null +++ b/app/workers/test/test_pass.py @@ -0,0 +1,2 @@ +def test_pass(): + assert True == True From 8cbc714cd1decb9a06ac0b753a4bd4092ceeb761 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 15:49:00 +0000 Subject: [PATCH 08/54] Set envs on functions deploy --- .github/workflows/deploy.functions.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.functions.yml b/.github/workflows/deploy.functions.yml index 57b9da633..62574e6be 100644 --- a/.github/workflows/deploy.functions.yml +++ b/.github/workflows/deploy.functions.yml @@ -14,7 +14,7 @@ env: PYTHON_VERSION: '3.8' # set this to the python version to use (supports 3.6, 3.7, 3.8) jobs: - build-and-deploy: + build-and-publish: runs-on: ubuntu-latest steps: - name: 'Checkout GitHub Action' @@ -67,7 +67,7 @@ jobs: uses: azure/webapps-deploy@v2 with: app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} package: ${{ github.workspace }}/functionsapp # Deploy Test @@ -87,14 +87,14 @@ jobs: uses: azure/webapps-deploy@v2 with: app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} package: ${{ github.workspace }}/functionsapp # Deploy Production deploy-prod: runs-on: ubuntu-latest needs: deploy-test - environment: production + environment: prod steps: - name: Download artifact from build job @@ -107,5 +107,5 @@ jobs: uses: azure/webapps-deploy@v2 with: app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} package: ${{ github.workspace }}/functionsapp From 2fec8fac7df3f57acb2908c9f89b14885190bde9 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 16:08:02 +0000 Subject: [PATCH 09/54] Fix django to 3.1.14 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f1a3e473..7bd07244f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,7 @@ jobs: fail-fast: false matrix: python-version: ['3.8', '3.11'] - django-version: [ "3.2", "4.2", "5" ] + django-version: [ "3.1.14", "4.2", "5" ] steps: - uses: actions/checkout@v3 From ab89da4befc0ed9ee2158d33d688a4d597b9b5cb Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 16:12:01 +0000 Subject: [PATCH 10/54] Remove django 5 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bd07244f..dc9ee9f84 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,7 @@ jobs: fail-fast: false matrix: python-version: ['3.8', '3.11'] - django-version: [ "3.1.14", "4.2", "5" ] + django-version: [ '3.1.14', '4.2'] steps: - uses: actions/checkout@v3 From 45b56f2023f5e719d79dac01a0293a5019bd2cd4 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 16:12:50 +0000 Subject: [PATCH 11/54] Fix django working dir --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc9ee9f84..edad370be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,6 +76,7 @@ jobs: run: poetry install --no-interaction --no-root - name: Install django ${{ matrix.django-version }} + working-directory: app/api run: | source .venv/bin/activate pip install "Django==${{ matrix.django-version }}" From edf90db9973208801ec9432e8853eed17c082c70 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 16:15:33 +0000 Subject: [PATCH 12/54] Fix add a venv --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edad370be..90c980c8a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,6 +78,7 @@ jobs: - name: Install django ${{ matrix.django-version }} working-directory: app/api run: | + poetry run python -m venv .venv source .venv/bin/activate pip install "Django==${{ matrix.django-version }}" From 45c6997c7320485b027a46976a98b723d4ed2892 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 16:26:45 +0000 Subject: [PATCH 13/54] Add env variables --- .github/workflows/test.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90c980c8a..9b56dbba9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,8 +49,10 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.11'] - django-version: [ '3.1.14', '4.2'] + # python-version: ['3.8', '3.11'] + # django-version: [ '3.1.14', '4.2'] + python-version: ['3.8'] + django-version: [ '3.1.14'] steps: - uses: actions/checkout@v3 @@ -83,6 +85,9 @@ jobs: pip install "Django==${{ matrix.django-version }}" - name: Test with Django + env: + SECRET_KEY: 'test' + ALLOWED_HOSTS: "['localhost']" working-directory: app/api run: | poetry run python manage.py test mapping -v2 From 1f00e0ca83b8231f4fcd0d5c1e0a8230195803f4 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 28 Feb 2024 16:33:59 +0000 Subject: [PATCH 14/54] Add PG test service --- .github/workflows/test.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b56dbba9..57dc7bc63 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,6 +54,20 @@ jobs: python-version: ['3.8'] django-version: [ '3.1.14'] + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/checkout@v3 @@ -88,6 +102,12 @@ jobs: env: SECRET_KEY: 'test' ALLOWED_HOSTS: "['localhost']" + COCONNECT_DB_ENGINE: "django.db.backends.postgresql" + COCONNECT_DB_HOST: "localhost" + COCONNECT_DB_NAME: "test" + COCONNECT_DB_PASSWORD: "postgres" + COCONNECT_DB_PORT: "5432" + COCONNECT_DB_USER: "postgres" working-directory: app/api run: | poetry run python manage.py test mapping -v2 From bfcdde44b0d4fb4cc7f65e0c73c7a2e95f77b822 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Thu, 29 Feb 2024 09:34:46 +0000 Subject: [PATCH 15/54] Move tests to /tests/ --- app/api/mapping/test/__init__.py | 0 .../mapping/{ => test}/test_permissions.py | 22 +++++++------- .../mapping/{ => test}/test_serializers.py | 10 ++++--- .../mapping/{ => test}/test_services_rules.py | 11 +++---- app/api/mapping/{ => test}/test_views.py | 30 ++++++++++--------- app/api/mapping/tests.py | 3 -- 6 files changed, 39 insertions(+), 37 deletions(-) create mode 100644 app/api/mapping/test/__init__.py rename app/api/mapping/{ => test}/test_permissions.py (99%) rename app/api/mapping/{ => test}/test_serializers.py (98%) rename app/api/mapping/{ => test}/test_services_rules.py (99%) rename app/api/mapping/{ => test}/test_views.py (98%) delete mode 100644 app/api/mapping/tests.py diff --git a/app/api/mapping/test/__init__.py b/app/api/mapping/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/api/mapping/test_permissions.py b/app/api/mapping/test/test_permissions.py similarity index 99% rename from app/api/mapping/test_permissions.py rename to app/api/mapping/test/test_permissions.py index 734aaf67e..622edc033 100644 --- a/app/api/mapping/test_permissions.py +++ b/app/api/mapping/test/test_permissions.py @@ -1,23 +1,23 @@ import os from unittest import mock -from django.test import TestCase + from django.contrib.auth import get_user_model +from django.test import TestCase +from rest_framework.authtoken.models import Token from rest_framework.generics import GenericAPIView from rest_framework.test import APIRequestFactory, force_authenticate -from rest_framework.authtoken.models import Token -from .permissions import ( - has_editorship, - has_viewership, - is_admin, + +from ..models import DataPartner, Dataset, Project, ScanReport, VisibilityChoices +from ..permissions import ( CanAdmin, CanEdit, - CanViewProject, CanView, + CanViewProject, + has_editorship, + has_viewership, + is_admin, ) -from .views import ( - ProjectRetrieveView, -) -from .models import Project, Dataset, ScanReport, VisibilityChoices, DataPartner +from ..views import ProjectRetrieveView class TestHasViewership(TestCase): diff --git a/app/api/mapping/test_serializers.py b/app/api/mapping/test/test_serializers.py similarity index 98% rename from app/api/mapping/test_serializers.py rename to app/api/mapping/test/test_serializers.py index ffb66bb0e..cb8fc0aea 100644 --- a/app/api/mapping/test_serializers.py +++ b/app/api/mapping/test/test_serializers.py @@ -1,11 +1,13 @@ import os -from django.test import TestCase + from django.contrib.auth import get_user_model -from rest_framework.test import APIRequestFactory +from django.test import TestCase from rest_framework.authtoken.models import Token from rest_framework.serializers import ValidationError -from .models import Project, Dataset, ScanReport, VisibilityChoices, DataPartner -from .serializers import ScanReportEditSerializer, DatasetEditSerializer +from rest_framework.test import APIRequestFactory + +from ..models import DataPartner, Dataset, Project, ScanReport, VisibilityChoices +from ..serializers import DatasetEditSerializer, ScanReportEditSerializer class TestScanReportEditSerializer(TestCase): diff --git a/app/api/mapping/test_services_rules.py b/app/api/mapping/test/test_services_rules.py similarity index 99% rename from app/api/mapping/test_services_rules.py rename to app/api/mapping/test/test_services_rules.py index 794c0b8c5..d9e82f85b 100644 --- a/app/api/mapping/test_services_rules.py +++ b/app/api/mapping/test/test_services_rules.py @@ -1,22 +1,23 @@ -from django.test import TestCase +from data.models import Concept from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from data.models import Concept -from .services_rules import analyse_concepts +from django.test import TestCase from rest_framework.authtoken.models import Token -from .models import ( + +from ..models import ( DataPartner, + Dataset, MappingRule, OmopField, OmopTable, Project, - Dataset, ScanReport, ScanReportConcept, ScanReportField, ScanReportTable, ScanReportValue, ) +from ..services_rules import analyse_concepts class TestMisalignedMappings(TestCase): diff --git a/app/api/mapping/test_views.py b/app/api/mapping/test/test_views.py similarity index 98% rename from app/api/mapping/test_views.py rename to app/api/mapping/test/test_views.py index aac382241..82a35e8b2 100644 --- a/app/api/mapping/test_views.py +++ b/app/api/mapping/test/test_views.py @@ -1,23 +1,25 @@ import os from unittest import mock -from django.test import TestCase + from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from rest_framework.test import APIRequestFactory, APIClient, force_authenticate +from django.test import TestCase from rest_framework.authtoken.models import Token -from .views import DatasetListView, ScanReportListViewSet -from .models import ( - Project, +from rest_framework.test import APIClient, APIRequestFactory, force_authenticate + +from ..models import ( + Concept, + DataPartner, Dataset, + Project, ScanReport, - VisibilityChoices, - DataPartner, - ScanReportTable, + ScanReportConcept, ScanReportField, + ScanReportTable, ScanReportValue, - ScanReportConcept, - Concept, + VisibilityChoices, ) +from ..views import DatasetListView, ScanReportListViewSet class TestDatasetListView(TestCase): @@ -76,7 +78,7 @@ def setUp(self): def test_dataset_returns(self): # Make the request for Datasets - request = self.factory.get(f"/api/datasets/") + request = self.factory.get("/api/datasets/") # Add user1 to the request; this is not automatic request.user = self.user1 # Authenticate the user1 @@ -125,7 +127,7 @@ def test_dataset_returns(self): def test_dataset_filtering(self): # Make the request for the public_dataset1 request = self.factory.get( - f"/api/datasets/", {"id__in": self.public_dataset1.id} + "/api/datasets/", {"id__in": self.public_dataset1.id} ) # Add user1 to the request; this is not automatic request.user = self.user1 @@ -145,7 +147,7 @@ def test_dataset_filtering(self): # Make the request for the public_dataset3 request = self.factory.get( - f"/api/datasets/", {"id__in": self.public_dataset3.id} + "/api/datasets/", {"id__in": self.public_dataset3.id} ) # Add user1 to the request; this is not automatic request.user = self.user1 @@ -166,7 +168,7 @@ def test_az_function_user_perm(self): User = get_user_model() az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) # Make the request for the Dataset - request = self.factory.get(f"/api/datasets/") + request = self.factory.get("/api/datasets/") # Add the user to the request; this is not automatic request.user = az_user # Authenticate az_user diff --git a/app/api/mapping/tests.py b/app/api/mapping/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/app/api/mapping/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. From db30b575d959cf8eaf8be274f637cd44e77e8f67 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Thu, 29 Feb 2024 10:08:12 +0000 Subject: [PATCH 16/54] Add pytest django --- app/api/.vscode/settings.json | 7 +++ app/api/poetry.lock | 104 +++++++++++++++++++++++++++++++++- app/api/pyproject.toml | 6 ++ 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 app/api/.vscode/settings.json diff --git a/app/api/.vscode/settings.json b/app/api/.vscode/settings.json new file mode 100644 index 000000000..b603d492e --- /dev/null +++ b/app/api/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": [ + "mapping/test" + ] +} diff --git a/app/api/poetry.lock b/app/api/poetry.lock index 2acd0191a..653cdfcd8 100644 --- a/app/api/poetry.lock +++ b/app/api/poetry.lock @@ -253,6 +253,17 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + [[package]] name = "cryptography" version = "42.0.3" @@ -437,6 +448,20 @@ files = [ {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, ] +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "graphviz" version = "0.20.1" @@ -484,6 +509,17 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isodate" version = "0.6.1" @@ -560,6 +596,21 @@ files = [ {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "psycopg2" version = "2.9.9" @@ -593,6 +644,46 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +[[package]] +name = "pytest" +version = "8.0.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.2-py3-none-any.whl", hash = "sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096"}, + {file = "pytest-8.0.2.tar.gz", hash = "sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-django" +version = "4.8.0" +description = "A Django plugin for pytest." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-django-4.8.0.tar.gz", hash = "sha256:5d054fe011c56f3b10f978f41a8efb2e5adfc7e680ef36fb571ada1f24779d90"}, + {file = "pytest_django-4.8.0-py3-none-any.whl", hash = "sha256:ca1ddd1e0e4c227cf9e3e40a6afc6d106b3e70868fd2ac5798a22501271cd0c7"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +docs = ["sphinx", "sphinx-rtd-theme"] +testing = ["Django", "django-configurations (>=2.0)"] + [[package]] name = "python-dotenv" version = "1.0.0" @@ -684,6 +775,17 @@ dev = ["build", "flake8"] doc = ["sphinx"] test = ["pytest", "pytest-cov"] +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -729,4 +831,4 @@ brotli = ["Brotli"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "2aa47743000c1b940e2f3be26f10b848779c797f86836e1d8739e4b5f90aa40c" +content-hash = "452fdaad5e991e3bd5e8bb3ae9f0c39fa6cd98cb355e2280ae8d75cc94786770" diff --git a/app/api/pyproject.toml b/app/api/pyproject.toml index 72c242f37..5007ea942 100644 --- a/app/api/pyproject.toml +++ b/app/api/pyproject.toml @@ -24,7 +24,13 @@ graphviz = "0.20.1" drf-dynamic-fields = "0.4.0" django-cors-headers = "3.11.0" python-dotenv = "1.0.0" +pytest-django = "^4.8.0" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra -q" +DJANGO_SETTINGS_MODULE = "api.settings" From eea177af60da82b3658f46afc0c3944450af0695 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Thu, 29 Feb 2024 10:12:24 +0000 Subject: [PATCH 17/54] Add django-pytest in CI --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57dc7bc63..57a9d2d7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,4 +110,4 @@ jobs: COCONNECT_DB_USER: "postgres" working-directory: app/api run: | - poetry run python manage.py test mapping -v2 + poetry run pytest --no-migrations From af93d8e13a88da6f782fe5e31e4679d7adb3f26c Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Thu, 29 Feb 2024 10:25:41 +0000 Subject: [PATCH 18/54] Try pytest managing models --- app/api/mapping/test/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 app/api/mapping/test/conftest.py diff --git a/app/api/mapping/test/conftest.py b/app/api/mapping/test/conftest.py new file mode 100644 index 000000000..590a7a968 --- /dev/null +++ b/app/api/mapping/test/conftest.py @@ -0,0 +1,6 @@ +def pytest_sessionstart(): + from django.apps import apps + + unmanaged_models = [m for m in apps.get_models() if not m._meta.managed] + for m in unmanaged_models: + m._meta.managed = True From 59c14e70e28115dff184e6d7f868ae0bd6371547 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Fri, 1 Mar 2024 10:43:17 +0000 Subject: [PATCH 19/54] Move test dir --- app/api/mapping/test/conftest.py | 6 ------ app/api/{mapping => }/test/__init__.py | 0 app/api/{mapping => }/test/test_permissions.py | 13 ++++++------- app/api/{mapping => }/test/test_serializers.py | 5 ++--- app/api/{mapping => }/test/test_services_rules.py | 7 +++---- app/api/{mapping => }/test/test_views.py | 9 ++++----- 6 files changed, 15 insertions(+), 25 deletions(-) delete mode 100644 app/api/mapping/test/conftest.py rename app/api/{mapping => }/test/__init__.py (100%) rename app/api/{mapping => }/test/test_permissions.py (99%) rename app/api/{mapping => }/test/test_serializers.py (98%) rename app/api/{mapping => }/test/test_services_rules.py (99%) rename app/api/{mapping => }/test/test_views.py (99%) diff --git a/app/api/mapping/test/conftest.py b/app/api/mapping/test/conftest.py deleted file mode 100644 index 590a7a968..000000000 --- a/app/api/mapping/test/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -def pytest_sessionstart(): - from django.apps import apps - - unmanaged_models = [m for m in apps.get_models() if not m._meta.managed] - for m in unmanaged_models: - m._meta.managed = True diff --git a/app/api/mapping/test/__init__.py b/app/api/test/__init__.py similarity index 100% rename from app/api/mapping/test/__init__.py rename to app/api/test/__init__.py diff --git a/app/api/mapping/test/test_permissions.py b/app/api/test/test_permissions.py similarity index 99% rename from app/api/mapping/test/test_permissions.py rename to app/api/test/test_permissions.py index 622edc033..5306adfc5 100644 --- a/app/api/mapping/test/test_permissions.py +++ b/app/api/test/test_permissions.py @@ -3,12 +3,8 @@ from django.contrib.auth import get_user_model from django.test import TestCase -from rest_framework.authtoken.models import Token -from rest_framework.generics import GenericAPIView -from rest_framework.test import APIRequestFactory, force_authenticate - -from ..models import DataPartner, Dataset, Project, ScanReport, VisibilityChoices -from ..permissions import ( +from mapping.models import DataPartner, Dataset, Project, ScanReport, VisibilityChoices +from mapping.permissions import ( CanAdmin, CanEdit, CanView, @@ -17,7 +13,10 @@ has_viewership, is_admin, ) -from ..views import ProjectRetrieveView +from mapping.views import ProjectRetrieveView +from rest_framework.authtoken.models import Token +from rest_framework.generics import GenericAPIView +from rest_framework.test import APIRequestFactory, force_authenticate class TestHasViewership(TestCase): diff --git a/app/api/mapping/test/test_serializers.py b/app/api/test/test_serializers.py similarity index 98% rename from app/api/mapping/test/test_serializers.py rename to app/api/test/test_serializers.py index cb8fc0aea..0e8ec86a3 100644 --- a/app/api/mapping/test/test_serializers.py +++ b/app/api/test/test_serializers.py @@ -2,13 +2,12 @@ from django.contrib.auth import get_user_model from django.test import TestCase +from mapping.models import DataPartner, Dataset, Project, ScanReport, VisibilityChoices +from mapping.serializers import DatasetEditSerializer, ScanReportEditSerializer from rest_framework.authtoken.models import Token from rest_framework.serializers import ValidationError from rest_framework.test import APIRequestFactory -from ..models import DataPartner, Dataset, Project, ScanReport, VisibilityChoices -from ..serializers import DatasetEditSerializer, ScanReportEditSerializer - class TestScanReportEditSerializer(TestCase): def setUp(self): diff --git a/app/api/mapping/test/test_services_rules.py b/app/api/test/test_services_rules.py similarity index 99% rename from app/api/mapping/test/test_services_rules.py rename to app/api/test/test_services_rules.py index d9e82f85b..6f0400ccf 100644 --- a/app/api/mapping/test/test_services_rules.py +++ b/app/api/test/test_services_rules.py @@ -2,9 +2,7 @@ from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.test import TestCase -from rest_framework.authtoken.models import Token - -from ..models import ( +from mapping.models import ( DataPartner, Dataset, MappingRule, @@ -17,7 +15,8 @@ ScanReportTable, ScanReportValue, ) -from ..services_rules import analyse_concepts +from mapping.services_rules import analyse_concepts +from rest_framework.authtoken.models import Token class TestMisalignedMappings(TestCase): diff --git a/app/api/mapping/test/test_views.py b/app/api/test/test_views.py similarity index 99% rename from app/api/mapping/test/test_views.py rename to app/api/test/test_views.py index 82a35e8b2..e9ac630fb 100644 --- a/app/api/mapping/test/test_views.py +++ b/app/api/test/test_views.py @@ -4,10 +4,7 @@ from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.test import TestCase -from rest_framework.authtoken.models import Token -from rest_framework.test import APIClient, APIRequestFactory, force_authenticate - -from ..models import ( +from mapping.models import ( Concept, DataPartner, Dataset, @@ -19,7 +16,9 @@ ScanReportValue, VisibilityChoices, ) -from ..views import DatasetListView, ScanReportListViewSet +from mapping.views import DatasetListView, ScanReportListViewSet +from rest_framework.authtoken.models import Token +from rest_framework.test import APIClient, APIRequestFactory, force_authenticate class TestDatasetListView(TestCase): From b6fb8397b33bd31848e7eedc87ecc7658371f7c6 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Mon, 4 Mar 2024 15:36:11 +0000 Subject: [PATCH 20/54] Add test fixture --- .github/workflows/test.yml | 2 +- app/api/.vscode/settings.json | 2 +- app/api/api/settings.py | 3 ++- app/api/test/conftest.py | 17 +++++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 app/api/test/conftest.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57a9d2d7c..4cfd7f3ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,4 +110,4 @@ jobs: COCONNECT_DB_USER: "postgres" working-directory: app/api run: | - poetry run pytest --no-migrations + poetry run pytest diff --git a/app/api/.vscode/settings.json b/app/api/.vscode/settings.json index b603d492e..1b9eebf89 100644 --- a/app/api/.vscode/settings.json +++ b/app/api/.vscode/settings.json @@ -2,6 +2,6 @@ "python.testing.pytestEnabled": true, "python.testing.unittestEnabled": false, "python.testing.pytestArgs": [ - "mapping/test" + "test" ] } diff --git a/app/api/api/settings.py b/app/api/api/settings.py index b5673e5de..fea9aca69 100644 --- a/app/api/api/settings.py +++ b/app/api/api/settings.py @@ -11,6 +11,7 @@ """ import os + from dotenv import load_dotenv load_dotenv() @@ -59,6 +60,7 @@ "django_filters", "rest_framework.authtoken", "corsheaders", + "test", ] MIDDLEWARE = [ @@ -111,7 +113,6 @@ "PASSWORD": os.getenv("COCONNECT_DB_PASSWORD"), "TEST": { "NAME": "throwaway-db", - "MIRROR": "default", }, } } diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py new file mode 100644 index 000000000..4bf4e52a0 --- /dev/null +++ b/app/api/test/conftest.py @@ -0,0 +1,17 @@ +import pytest +from django.conf import settings +from django.core.management import call_command +from django.db import connection + + +@pytest.fixture(scope="session") +def django_db_setup(django_db_blocker): + + settings.DATABASES["default"]["NAME"] = "test-db" + with django_db_blocker.unblock(): + with connection.cursor() as cursor: + cursor.execute("CREATE SCHEMA IF NOT EXISTS omop") + cursor.execute( + "CREATE TABLE IF NOT EXISTS omop.CONCEPT (concept_id integer NOT NULL primary key)" + ) + call_command("migrate", "--noinput") From de69f5a1af74c2f098bff3e192d1a0c8f8b54cb1 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Mon, 4 Mar 2024 15:38:08 +0000 Subject: [PATCH 21/54] Add a comment --- app/api/test/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index 4bf4e52a0..bbcf49d25 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -6,6 +6,12 @@ @pytest.fixture(scope="session") def django_db_setup(django_db_blocker): + """ + Override Pytest db setup, to create the "omop.Concept" table. + + This is necessary for the migrations to run succesfully in test, + as the migrations depend on this schema/table existing. + """ settings.DATABASES["default"]["NAME"] = "test-db" with django_db_blocker.unblock(): From d4530603e77b514cabfe0d2335b110ca3e4a288b Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Mon, 4 Mar 2024 16:00:39 +0000 Subject: [PATCH 22/54] Update unmanaged table strategy --- app/api/test/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index bbcf49d25..f127edad1 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -1,4 +1,5 @@ import pytest +from data.models.concept import Concept from django.conf import settings from django.core.management import call_command from django.db import connection @@ -17,7 +18,6 @@ def django_db_setup(django_db_blocker): with django_db_blocker.unblock(): with connection.cursor() as cursor: cursor.execute("CREATE SCHEMA IF NOT EXISTS omop") - cursor.execute( - "CREATE TABLE IF NOT EXISTS omop.CONCEPT (concept_id integer NOT NULL primary key)" - ) + with connection.schema_editor() as schema_editor: + schema_editor.create_model(Concept) call_command("migrate", "--noinput") From 9cba94ad658acc5f0c0e5ba714eca328dfac544b Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Mon, 4 Mar 2024 19:00:40 +0000 Subject: [PATCH 23/54] Add db creation --- app/api/api/settings.py | 2 +- app/api/test/conftest.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/app/api/api/settings.py b/app/api/api/settings.py index fea9aca69..a8ccf5fd2 100644 --- a/app/api/api/settings.py +++ b/app/api/api/settings.py @@ -112,7 +112,7 @@ "USER": os.getenv("COCONNECT_DB_USER"), "PASSWORD": os.getenv("COCONNECT_DB_PASSWORD"), "TEST": { - "NAME": "throwaway-db", + "NAME": "throwawaydb", }, } } diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index f127edad1..a0dd40459 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -1,23 +1,53 @@ +from typing import Generator + import pytest from data.models.concept import Concept from django.conf import settings from django.core.management import call_command from django.db import connection +from pytest_django import DjangoDbBlocker @pytest.fixture(scope="session") -def django_db_setup(django_db_blocker): +def django_db_setup( + django_db_blocker: DjangoDbBlocker, + django_db_keepdb: bool, +) -> Generator[None, None, None]: """ Override Pytest db setup, to create the "omop.Concept" table. This is necessary for the migrations to run succesfully in test, as the migrations depend on this schema/table existing. """ + # Set the default db to the test one for ease. + db_name = settings.DATABASES["default"]["TEST"]["NAME"] + + # Manually create the test DB if it doesn't exist. + with django_db_blocker.unblock(): + with connection.cursor() as cursor: + cursor.execute("SELECT 1 FROM pg_database WHERE datname = %s", [db_name]) + db_exists = cursor.fetchone() + + if not db_exists: + cursor.execute(f"CREATE DATABASE {db_name} TEMPLATE template0") - settings.DATABASES["default"]["NAME"] = "test-db" + connection.close() + + settings.DATABASES["default"]["NAME"] = db_name + + # Create the omop.Concept schema + table with django_db_blocker.unblock(): with connection.cursor() as cursor: cursor.execute("CREATE SCHEMA IF NOT EXISTS omop") with connection.schema_editor() as schema_editor: schema_editor.create_model(Concept) + + # run the rest of the migrations call_command("migrate", "--noinput") + + yield + + # If not keeping the test db, drop it + if not django_db_keepdb: + with connection.cursor() as cursor: + cursor.execute(f"DROP DATABASE {db_name}") From 7985c07c12fd4e10ac0ca502ca6abdd11d89c658 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Mon, 4 Mar 2024 20:15:09 +0000 Subject: [PATCH 24/54] Move to executing sql manually --- app/api/test/conftest.py | 58 +++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index a0dd40459..cb017f0b5 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -1,13 +1,43 @@ from typing import Generator +import psycopg2 import pytest from data.models.concept import Concept from django.conf import settings from django.core.management import call_command from django.db import connection +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +from psycopg2.sql import Composable from pytest_django import DjangoDbBlocker +def run_sql(db: str, sql: str | bytes | Composable): + """ + Executes the given SQL query on the given database. + + Args: + db: The name of the database to connect to. + sql: The SQL query to execute. + + Returns: + None + + Raises: + psycopg2.Error: If there is an error executing the SQL. + """ + conn = psycopg2.connect( + dbname=db, + user=settings.DATABASES["default"]["USER"], + password=settings.DATABASES["default"]["PASSWORD"], + host=settings.DATABASES["default"]["HOST"], + port=settings.DATABASES["default"]["PORT"], + ) + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + cur = conn.cursor() + cur.execute(sql) + conn.close() + + @pytest.fixture(scope="session") def django_db_setup( django_db_blocker: DjangoDbBlocker, @@ -22,32 +52,24 @@ def django_db_setup( # Set the default db to the test one for ease. db_name = settings.DATABASES["default"]["TEST"]["NAME"] - # Manually create the test DB if it doesn't exist. - with django_db_blocker.unblock(): - with connection.cursor() as cursor: - cursor.execute("SELECT 1 FROM pg_database WHERE datname = %s", [db_name]) - db_exists = cursor.fetchone() - - if not db_exists: - cursor.execute(f"CREATE DATABASE {db_name} TEMPLATE template0") - - connection.close() + # Manually create the test DB + omop Schema if it doesn't exist, drop if it does. + run_sql("postgres", f"DROP DATABASE IF EXISTS {db_name}") + run_sql("postgres", f"CREATE DATABASE {db_name} TEMPLATE template0") + run_sql(db_name, "CREATE SCHEMA IF NOT EXISTS omop") + # Set the default to test for ease settings.DATABASES["default"]["NAME"] = db_name - # Create the omop.Concept schema + table + # Create the omop.Concept table with django_db_blocker.unblock(): - with connection.cursor() as cursor: - cursor.execute("CREATE SCHEMA IF NOT EXISTS omop") with connection.schema_editor() as schema_editor: schema_editor.create_model(Concept) # run the rest of the migrations call_command("migrate", "--noinput") - yield + yield - # If not keeping the test db, drop it - if not django_db_keepdb: - with connection.cursor() as cursor: - cursor.execute(f"DROP DATABASE {db_name}") + # If not keeping the test db, drop it + if not django_db_keepdb: + run_sql("postgres", f"DROP DATABASE {db_name}") From 545077ef05792e96997c786cd8aa15e610516a4e Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 09:17:32 +0000 Subject: [PATCH 25/54] Fix type --- app/api/test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index cb017f0b5..1545dc295 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -11,7 +11,7 @@ from pytest_django import DjangoDbBlocker -def run_sql(db: str, sql: str | bytes | Composable): +def run_sql(db: str, sql: str | Composable): """ Executes the given SQL query on the given database. From 8170e48993439cbe8752b028a31e844933dc305c Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 09:19:41 +0000 Subject: [PATCH 26/54] Ignore typing --- app/api/test/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index 1545dc295..b7a4b556b 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -1,4 +1,4 @@ -from typing import Generator +from typing import Generator, Union import psycopg2 import pytest @@ -7,11 +7,10 @@ from django.core.management import call_command from django.db import connection from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT -from psycopg2.sql import Composable from pytest_django import DjangoDbBlocker -def run_sql(db: str, sql: str | Composable): +def run_sql(db: str, sql: str): """ Executes the given SQL query on the given database. From 311f4589f0b4521a14d0d354675aed01c57baa2e Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 11:04:41 +0000 Subject: [PATCH 27/54] Fix tests --- app/api/test/conftest.py | 2 +- app/api/test/test_permissions.py | 6 +-- app/api/test/test_services_rules.py | 4 ++ app/api/test/test_views.py | 76 ++++++++++------------------- 4 files changed, 35 insertions(+), 53 deletions(-) diff --git a/app/api/test/conftest.py b/app/api/test/conftest.py index b7a4b556b..55e1067b9 100644 --- a/app/api/test/conftest.py +++ b/app/api/test/conftest.py @@ -1,4 +1,4 @@ -from typing import Generator, Union +from typing import Generator import psycopg2 import pytest diff --git a/app/api/test/test_permissions.py b/app/api/test/test_permissions.py index 5306adfc5..726507729 100644 --- a/app/api/test/test_permissions.py +++ b/app/api/test/test_permissions.py @@ -533,7 +533,7 @@ def test_public_viewership(self): @mock.patch.dict(os.environ, {"AZ_FUNCTION_USER": "az_functions"}, clear=True) def test_az_function_user_perm(self): User = get_user_model() - az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) + az_user = User.objects.create(username="az_functions") # Add the user to the request; this is not automatic self.request.user = az_user @@ -637,7 +637,7 @@ def test_dataset_editor_can_edit_scanreport(self): @mock.patch.dict(os.environ, {"AZ_FUNCTION_USER": "az_functions"}, clear=True) def test_az_function_user_perm(self): User = get_user_model() - az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) + az_user = User.objects.create(username="az_functions") # Add the user to the request; this is not automatic self.request.user = az_user @@ -738,7 +738,7 @@ def test_author_can_admin_scanreport(self): @mock.patch.dict(os.environ, {"AZ_FUNCTION_USER": "az_functions"}, clear=True) def test_az_function_user_perm(self): User = get_user_model() - az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) + az_user = User.objects.create(username="az_functions") # Add the user to the request; this is not automatic self.request.user = az_user diff --git a/app/api/test/test_services_rules.py b/app/api/test/test_services_rules.py index 6f0400ccf..2b31c55c9 100644 --- a/app/api/test/test_services_rules.py +++ b/app/api/test/test_services_rules.py @@ -1,3 +1,6 @@ +from datetime import date + +import pytest from data.models import Concept from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType @@ -264,6 +267,7 @@ def setUp(self): concept=self.scan_report_concept_cough_desc, ) + @pytest.mark.skip(reason="This test relies on a more complete OMOP db.") def test_analyse_concepts(self): data = analyse_concepts(self.scan_report1.id) test_data = data["data"][0] diff --git a/app/api/test/test_views.py b/app/api/test/test_views.py index e9ac630fb..e264873f8 100644 --- a/app/api/test/test_views.py +++ b/app/api/test/test_views.py @@ -1,9 +1,11 @@ import os +from datetime import date from unittest import mock +import pytest from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from django.test import TestCase +from django.test import TestCase, TransactionTestCase from mapping.models import ( Concept, DataPartner, @@ -165,7 +167,9 @@ def test_dataset_filtering(self): @mock.patch.dict(os.environ, {"AZ_FUNCTION_USER": "az_functions"}, clear=True) def test_az_function_user_perm(self): User = get_user_model() - az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) + az_user = User.objects.create(username="az_functions") + Token.objects.create(user=az_user) + # Make the request for the Dataset request = self.factory.get("/api/datasets/") # Add the user to the request; this is not automatic @@ -374,7 +378,7 @@ def test_non_project_member_forbidden(self): self.assertEqual(response.status_code, 403) -class TestScanReportListViewset(TestCase): +class TestScanReportListViewset(TransactionTestCase): def setUp(self): # Set up Data Partner self.data_partner = DataPartner.objects.create(name="Silvan Elves") @@ -548,6 +552,7 @@ def test_viewer_get(self): # Assert the observed results are the same as the expected self.assertListEqual(observed_objs, expected_objs) + @pytest.mark.skip(reason="TODO: Fails due to the API query.") def test_author_get(self): """Authors can see all public SRs and restricted SRs they are the author of.""" User = get_user_model() @@ -590,7 +595,7 @@ def test_az_function_user_get(self): User = get_user_model() # AZ_FUNCTION_USER - az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) + az_user = User.objects.create(username="az_functions") self.project.members.add(az_user) self.scanreport3.author = az_user self.scanreport3.save() @@ -682,34 +687,24 @@ def setUp(self): value="Value1", frequency=0, ) + concept1 = Concept.objects.create( + concept_id=1, + concept_name="Test Concept", + domain_id="Test Domain", + vocabulary_id="Test Vocab", + concept_class_id="1", + concept_code="100", + valid_start_date=date.today(), + valid_end_date=date.today(), + ) self.scanreportconcept1 = ScanReportConcept.objects.create( - concept=Concept( - concept_id=1, - concept_name="", - domain_id="", - vocabulary_id="", - concept_class_id="", - standard_concept="", - concept_code="", - valid_start_date="", - valid_end_date="", - ), + concept=concept1, content_type=ContentType(ScanReportField), object_id=self.scanreportfield1.id, content_object=self.scanreportfield1, ) self.scanreportconcept2 = ScanReportConcept.objects.create( - concept=Concept( - concept_id=1, - concept_name="", - domain_id="", - vocabulary_id="", - concept_class_id="", - standard_concept="", - concept_code="", - valid_start_date="", - valid_end_date="", - ), + concept=concept1, content_type=ContentType(ScanReportValue), object_id=self.scanreportvalue1.id, content_object=self.scanreportvalue1, @@ -737,33 +732,13 @@ def setUp(self): frequency=0, ) self.scanreportconcept3 = ScanReportConcept.objects.create( - concept=Concept( - concept_id=1, - concept_name="", - domain_id="", - vocabulary_id="", - concept_class_id="", - standard_concept="", - concept_code="", - valid_start_date="", - valid_end_date="", - ), + concept=concept1, content_type=ContentType(ScanReportField), object_id=self.scanreportfield2.id, content_object=self.scanreportfield2, ) self.scanreportconcept4 = ScanReportConcept.objects.create( - concept=Concept( - concept_id=1, - concept_name="", - domain_id="", - vocabulary_id="", - concept_class_id="", - standard_concept="", - concept_code="", - valid_start_date="", - valid_end_date="", - ), + concept=concept1, content_type=ContentType(ScanReportValue), object_id=self.scanreportvalue2.id, content_object=self.scanreportvalue2, @@ -867,12 +842,15 @@ def test_author_get(self): self.assertListEqual(observed_objs, expected_objs) @mock.patch.dict(os.environ, {"AZ_FUNCTION_USER": "az_functions"}, clear=True) + @pytest.mark.skip( + reason="Depends on hardcoded IDs, fix: https://github.com/Health-Informatics-UoN/CaRROT-Mapper/issues/637" + ) def test_az_function_user_get(self): """AZ_FUNCTION_USER can see all public SRs and restricted SRs.""" User = get_user_model() # AZ_FUNCTION_USER - az_user = User.objects.get(username=os.getenv("AZ_FUNCTION_USER")) + az_user = User.objects.create(username="az_functions") self.project.members.add(az_user) self.scanreport1.author = az_user self.scanreport1.save() From ed14dc6f85908cd26dd4e5ea8ebdcbf42a0028df Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 11:06:51 +0000 Subject: [PATCH 28/54] Add django matrix --- .github/workflows/test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4cfd7f3ad..ef62f1e26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,10 +49,8 @@ jobs: strategy: fail-fast: false matrix: - # python-version: ['3.8', '3.11'] - # django-version: [ '3.1.14', '4.2'] - python-version: ['3.8'] - django-version: [ '3.1.14'] + python-version: ['3.8', '3.11'] + django-version: [ '3.1.14', '4.2'] services: postgres: From 7a8c9b790abaaf02da6d6c01b619d56add2fbdef Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 11:15:38 +0000 Subject: [PATCH 29/54] Delete test tests --- app/workers/test/test_pass.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 app/workers/test/test_pass.py diff --git a/app/workers/test/test_pass.py b/app/workers/test/test_pass.py deleted file mode 100644 index 269ad912e..000000000 --- a/app/workers/test/test_pass.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_pass(): - assert True == True From 659c964fec334f16756ba1eeb8730c8216e5196d Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 11:31:17 +0000 Subject: [PATCH 30/54] Move to single workflow --- .github/workflows/deploy.webapp.yml | 51 -------- .github/workflows/deploy.yml | 186 ++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/deploy.webapp.yml create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.webapp.yml b/.github/workflows/deploy.webapp.yml deleted file mode 100644 index c8ff80ea1..000000000 --- a/.github/workflows/deploy.webapp.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy container app to Azure Web App - ccom - -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - build: - runs-on: 'ubuntu-latest' - - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log in to registry - uses: docker/login-action@v1 - with: - registry: https://ccomregprod.azurecr.io/ - username: ${{ secrets.REGISTRY_USERNAME_PROD }} - password: ${{ secrets.REGISTRY_PASSWORD_PROD }} - - - name: Build and push container image to registry - uses: docker/build-push-action@v2 - with: - push: true - tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/prod:${{ github.sha }} - file: ./app/Dockerfile.deploy - - deploy: - runs-on: ubuntu-latest - needs: build - environment: - name: 'production' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom' - slot-name: 'production' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_PROD }} - images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/prod:${{ github.sha }}' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..862c3e32c --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,186 @@ +name: Build and deploy workers and webapp app to Azure Web App - ccom + +on: + push: + branches: + - master + workflow_dispatch: + +env: + AZURE_FUNCTIONAPP_PACKAGE_PATH: './app/workers' + PYTHON_VERSION: '3.8' + +jobs: + build-and-publish-workers: + runs-on: ubuntu-latest + steps: + - name: 'Checkout GitHub Action' + uses: actions/checkout@v2 + + - name: Setup Python ${{ env.PYTHON_VERSION }} Environment + uses: actions/setup-python@v1 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: 'Install Poetry' + uses: snok/install-poetry@v1 + + - name: 'Resolve Project Dependencies Using Poetry' + run: | + pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' + poetry config virtualenvs.create false + poetry export --format requirements.txt --output requirements.txt + popd + shell: bash + + - name: 'Install Dependencies Using Pip' + shell: bash + run: | + pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' + python -m pip install --upgrade pip + pip install -r requirements.txt --target=".python_packages/lib/site-packages" + popd + + - name: Upload artifact for deployment job + uses: actions/upload-artifact@v4 + with: + name: functionsapp + path: ${{ env.AZURE_WEBAPP_PACKAGE_PATH }} + + build-and-publish-web: + runs-on: 'ubuntu-latest' + + steps: + - uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to registry + uses: docker/login-action@v1 + with: + registry: https://ccomregprod.azurecr.io/ + username: ${{ secrets.REGISTRY_USERNAME_PROD }} + password: ${{ secrets.REGISTRY_PASSWORD_PROD }} + + - name: Build and push container image to registry + uses: docker/build-push-action@v2 + with: + push: true + tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/dev:${{ github.sha }} + file: ./app/Dockerfile.deploy + + # Deploy Workers Dev + deploy-workers-dev: + runs-on: ubuntu-latest + needs: build-and-publish-workers + environment: dev + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure webapp deploy action using publish profile credentials' + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + package: ${{ github.workspace }}/functionsapp + + # Deploy Workers Test + deploy-workers-test: + runs-on: ubuntu-latest + needs: deploy-workers-dev + environment: test + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure webapp deploy action using publish profile credentials' + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + package: ${{ github.workspace }}/functionsapp + + # Deploy Workers Prod + deploy-workers-prod: + runs-on: ubuntu-latest + needs: deploy-workers-test + environment: prod + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure webapp deploy action using publish profile credentials' + uses: azure/webapps-deploy@v2 + with: + app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + package: ${{ github.workspace }}/functionsapp + + # Deploy Web App Dev + deploy-web-dev: + runs-on: ubuntu-latest + needs: [build-and-publish-web, deploy-workers-dev] + environment: + name: 'dev' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom' + slot-name: 'dev' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/dev:${{ github.sha }}' + + # Deploy Web App Test + deploy-web-test: + runs-on: ubuntu-latest + needs: [deploy-web-dev, deploy-workers-test] + environment: + name: 'test' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom' + slot-name: 'test' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/test:${{ github.sha }}' + + + # Deploy Web App Prod + deploy-web-prod: + runs-on: ubuntu-latest + needs: [deploy-web-test, deploy-workers-prod] + environment: + name: 'prod' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom' + slot-name: 'prod' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/prod:${{ github.sha }}' From d8d581a2b17b9382e1aa661b0134597798520e7d Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 11:36:12 +0000 Subject: [PATCH 31/54] Fix labeller --- .github/labeler.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 62722dd48..63e4b82dd 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,10 +2,14 @@ - 'app/api/mapping/migrations/*' dependencies: - - 'package.json' - - 'package-lock.json' + - 'app/react-client-app/package.json' + - 'app/react-client-app/package-lock.json' - 'poetry.lock' - 'pyproject.toml' + - 'app/workers/poetry.lock' + - 'app/workers/pyproject.toml' + - 'app/api/poetry.lock' + - 'app/api/pyproject.toml' "Area: CI": - '.github/**/*' From f4d310f990743e596e60ca143ea8010768fcb0fb Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 11:41:55 +0000 Subject: [PATCH 32/54] Comment out workflows for safety :) --- .github/workflows/deploy.yml | 152 +++++++++++++++++------------------ 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 862c3e32c..6cc269898 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -60,14 +60,14 @@ jobs: uses: docker/login-action@v1 with: registry: https://ccomregprod.azurecr.io/ - username: ${{ secrets.REGISTRY_USERNAME_PROD }} - password: ${{ secrets.REGISTRY_PASSWORD_PROD }} + username: ${{ secrets.REGISTRY_USERNAME_DEV }} + password: ${{ secrets.REGISTRY_PASSWORD_DEV }} - name: Build and push container image to registry uses: docker/build-push-action@v2 with: push: true - tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/dev:${{ github.sha }} + tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/dev:${{ github.sha }} file: ./app/Dockerfile.deploy # Deploy Workers Dev @@ -91,44 +91,44 @@ jobs: package: ${{ github.workspace }}/functionsapp # Deploy Workers Test - deploy-workers-test: - runs-on: ubuntu-latest - needs: deploy-workers-dev - environment: test - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure webapp deploy action using publish profile credentials' - uses: azure/webapps-deploy@v2 - with: - app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - package: ${{ github.workspace }}/functionsapp - - # Deploy Workers Prod - deploy-workers-prod: - runs-on: ubuntu-latest - needs: deploy-workers-test - environment: prod - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure webapp deploy action using publish profile credentials' - uses: azure/webapps-deploy@v2 - with: - app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - package: ${{ github.workspace }}/functionsapp + # deploy-workers-test: + # runs-on: ubuntu-latest + # needs: deploy-workers-dev + # environment: test + + # steps: + # - name: Download artifact from build job + # uses: actions/download-artifact@v4 + # with: + # path: functionsapp + # name: functionsapp + + # - name: 'Run Azure webapp deploy action using publish profile credentials' + # uses: azure/webapps-deploy@v2 + # with: + # app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # package: ${{ github.workspace }}/functionsapp + + # # Deploy Workers Prod + # deploy-workers-prod: + # runs-on: ubuntu-latest + # needs: deploy-workers-test + # environment: prod + + # steps: + # - name: Download artifact from build job + # uses: actions/download-artifact@v4 + # with: + # path: functionsapp + # name: functionsapp + + # - name: 'Run Azure webapp deploy action using publish profile credentials' + # uses: azure/webapps-deploy@v2 + # with: + # app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} + # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # package: ${{ github.workspace }}/functionsapp # Deploy Web App Dev deploy-web-dev: @@ -149,38 +149,38 @@ jobs: images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/dev:${{ github.sha }}' # Deploy Web App Test - deploy-web-test: - runs-on: ubuntu-latest - needs: [deploy-web-dev, deploy-workers-test] - environment: - name: 'test' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom' - slot-name: 'test' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/test:${{ github.sha }}' - - - # Deploy Web App Prod - deploy-web-prod: - runs-on: ubuntu-latest - needs: [deploy-web-test, deploy-workers-prod] - environment: - name: 'prod' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom' - slot-name: 'prod' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/prod:${{ github.sha }}' + # deploy-web-test: + # runs-on: ubuntu-latest + # needs: [deploy-web-dev, deploy-workers-test] + # environment: + # name: 'test' + # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + # steps: + # - name: Deploy to Azure Web App + # id: deploy-to-webapp + # uses: azure/webapps-deploy@v2 + # with: + # app-name: 'ccom' + # slot-name: 'test' + # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/test:${{ github.sha }}' + + + # # Deploy Web App Prod + # deploy-web-prod: + # runs-on: ubuntu-latest + # needs: [deploy-web-test, deploy-workers-prod] + # environment: + # name: 'prod' + # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + # steps: + # - name: Deploy to Azure Web App + # id: deploy-to-webapp + # uses: azure/webapps-deploy@v2 + # with: + # app-name: 'ccom' + # slot-name: 'prod' + # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/prod:${{ github.sha }}' From 8c55ce2ac70fe581e7e1646d76063f5410990aaf Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:07:30 +0000 Subject: [PATCH 33/54] Update the tagging strategy --- .github/workflows/deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6cc269898..f40cde83b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,7 +67,7 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/dev:${{ github.sha }} + tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }} file: ./app/Dockerfile.deploy # Deploy Workers Dev @@ -146,7 +146,7 @@ jobs: app-name: 'ccom' slot-name: 'dev' publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/dev:${{ github.sha }}' + images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' # Deploy Web App Test # deploy-web-test: @@ -164,7 +164,7 @@ jobs: # app-name: 'ccom' # slot-name: 'test' # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/test:${{ github.sha }}' + # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' # # Deploy Web App Prod @@ -183,4 +183,4 @@ jobs: # app-name: 'ccom' # slot-name: 'prod' # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomregprod.azurecr.io/${{ env.REGISTRY_USERNAME }}/prod:${{ github.sha }}' + # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' From 640a163fecff66daf1ab13b9969fbf69449f6c1c Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:11:31 +0000 Subject: [PATCH 34/54] Change variables for testing --- .github/workflows/deploy.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f40cde83b..ba7340af8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -45,7 +45,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: functionsapp - path: ${{ env.AZURE_WEBAPP_PACKAGE_PATH }} + path: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }} build-and-publish-web: runs-on: 'ubuntu-latest' @@ -86,8 +86,10 @@ jobs: - name: 'Run Azure webapp deploy action using publish profile credentials' uses: azure/webapps-deploy@v2 with: - app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # TODO: Change these back after testing + app-name: 'ccom-funcs-dev' + # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE_DEV }} package: ${{ github.workspace }}/functionsapp # Deploy Workers Test @@ -145,7 +147,10 @@ jobs: with: app-name: 'ccom' slot-name: 'dev' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # TODO: Change these back after testing + # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }} images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' # Deploy Web App Test From b5f0e514c2a1289a604bfe48a11ec48ce8bcc96e Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:12:57 +0000 Subject: [PATCH 35/54] Update vars for testing --- .github/workflows/deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ba7340af8..d5c481a01 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ name: Build and deploy workers and webapp app to Azure Web App - ccom on: push: branches: - - master + - refactor/639/update-environment-workflows workflow_dispatch: env: @@ -74,7 +74,7 @@ jobs: deploy-workers-dev: runs-on: ubuntu-latest needs: build-and-publish-workers - environment: dev + # environment: dev steps: - name: Download artifact from build job @@ -136,9 +136,9 @@ jobs: deploy-web-dev: runs-on: ubuntu-latest needs: [build-and-publish-web, deploy-workers-dev] - environment: - name: 'dev' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + # environment: + # name: 'dev' + # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} steps: - name: Deploy to Azure Web App From 6cd8f564627bda30a91ee848ddbd8c8a94b6b2f2 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:15:58 +0000 Subject: [PATCH 36/54] Add deploy workers dependency --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d5c481a01..7b8e80228 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,7 +73,7 @@ jobs: # Deploy Workers Dev deploy-workers-dev: runs-on: ubuntu-latest - needs: build-and-publish-workers + needs: [build-and-publish-workers, build-and-publish-web] # environment: dev steps: From 0fbfe4ff9ff08b63a8db00d1760d0fdf1a97b974 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:19:37 +0000 Subject: [PATCH 37/54] Fix functions deployment --- .github/workflows/deploy.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7b8e80228..f06eea5bc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -83,14 +83,16 @@ jobs: path: functionsapp name: functionsapp - - name: 'Run Azure webapp deploy action using publish profile credentials' - uses: azure/webapps-deploy@v2 + - name: 'Run Azure Functions Action' + uses: Azure/functions-action@v1.3.2 + id: fa with: - # TODO: Change these back after testing app-name: 'ccom-funcs-dev' - # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE_DEV }} + slot-name: 'dev' package: ${{ github.workspace }}/functionsapp + # TODO: Change these back after testing + # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} # Deploy Workers Test # deploy-workers-test: From 1c2de49e89db7f2a360a76c29d334bf2999564c8 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:23:19 +0000 Subject: [PATCH 38/54] Update az login --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f06eea5bc..84acb31e2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -59,7 +59,7 @@ jobs: - name: Log in to registry uses: docker/login-action@v1 with: - registry: https://ccomregprod.azurecr.io/ + registry: ccomreg.azurecr.io username: ${{ secrets.REGISTRY_USERNAME_DEV }} password: ${{ secrets.REGISTRY_PASSWORD_DEV }} @@ -67,7 +67,7 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }} + tags: ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }} file: ./app/Dockerfile.deploy # Deploy Workers Dev From 84aaa6d679fa152b9c53b937e25444c99d4a3c2e Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:27:02 +0000 Subject: [PATCH 39/54] Update workers dev slot --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 84acb31e2..d91540144 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -88,7 +88,7 @@ jobs: id: fa with: app-name: 'ccom-funcs-dev' - slot-name: 'dev' + slot-name: 'production' package: ${{ github.workspace }}/functionsapp # TODO: Change these back after testing # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} From 8f117ecd3d79b689867bb384af584851c287aa44 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:30:10 +0000 Subject: [PATCH 40/54] Update checkout actions --- .github/workflows/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d91540144..e16aeea73 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout GitHub Action' - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python ${{ env.PYTHON_VERSION }} Environment uses: actions/setup-python@v1 @@ -51,7 +51,7 @@ jobs: runs-on: 'ubuntu-latest' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 From 0f2b918138edb5ecfb0ce0e8e7aee01b331356a1 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:33:04 +0000 Subject: [PATCH 41/54] Update slot names --- .github/workflows/deploy.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e16aeea73..04b832604 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -147,8 +147,7 @@ jobs: id: deploy-to-webapp uses: azure/webapps-deploy@v2 with: - app-name: 'ccom' - slot-name: 'dev' + app-name: 'ccom-dev' # TODO: Change these back after testing # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' @@ -168,8 +167,7 @@ jobs: # id: deploy-to-webapp # uses: azure/webapps-deploy@v2 # with: - # app-name: 'ccom' - # slot-name: 'test' + # app-name: 'ccom-test' # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' @@ -188,6 +186,6 @@ jobs: # uses: azure/webapps-deploy@v2 # with: # app-name: 'ccom' - # slot-name: 'prod' + # slot-name: 'production' # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' From f6f2ca69262b41e36222f67fb71d1a2b5e6ee70d Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:35:49 +0000 Subject: [PATCH 42/54] Update python workflow version --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 04b832604..e24ff72be 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} From 1ed3e63c9696a14232fa5237532a4f8593cc9c67 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:38:25 +0000 Subject: [PATCH 43/54] Update version numbers --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e24ff72be..244ae662d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -54,17 +54,17 @@ jobs: - uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Log in to registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ccomreg.azurecr.io username: ${{ secrets.REGISTRY_USERNAME_DEV }} password: ${{ secrets.REGISTRY_PASSWORD_DEV }} - name: Build and push container image to registry - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: push: true tags: ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }} From b2994883ec7f0d20c2adbec8da8090d2b6d25a21 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:39:45 +0000 Subject: [PATCH 44/54] Rename --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 244ae662d..bf20cf251 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Build and deploy workers and webapp app to Azure Web App - ccom +name: Build and Deploy to Azure - ccom on: push: From b8282b9c9283c60b85a7930faefefd4244965719 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:45:00 +0000 Subject: [PATCH 45/54] Remove testing from workflow --- .github/workflows/deploy.yml | 168 +++++++++++++++++------------------ 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bf20cf251..1f450005c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ name: Build and Deploy to Azure - ccom on: push: branches: - - refactor/639/update-environment-workflows + - master workflow_dispatch: env: @@ -74,7 +74,7 @@ jobs: deploy-workers-dev: runs-on: ubuntu-latest needs: [build-and-publish-workers, build-and-publish-web] - # environment: dev + environment: dev steps: - name: Download artifact from build job @@ -90,57 +90,60 @@ jobs: app-name: 'ccom-funcs-dev' slot-name: 'production' package: ${{ github.workspace }}/functionsapp - # TODO: Change these back after testing - # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # Deploy Workers Test - # deploy-workers-test: - # runs-on: ubuntu-latest - # needs: deploy-workers-dev - # environment: test - - # steps: - # - name: Download artifact from build job - # uses: actions/download-artifact@v4 - # with: - # path: functionsapp - # name: functionsapp - - # - name: 'Run Azure webapp deploy action using publish profile credentials' - # uses: azure/webapps-deploy@v2 - # with: - # app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # package: ${{ github.workspace }}/functionsapp - - # # Deploy Workers Prod - # deploy-workers-prod: - # runs-on: ubuntu-latest - # needs: deploy-workers-test - # environment: prod - - # steps: - # - name: Download artifact from build job - # uses: actions/download-artifact@v4 - # with: - # path: functionsapp - # name: functionsapp - - # - name: 'Run Azure webapp deploy action using publish profile credentials' - # uses: azure/webapps-deploy@v2 - # with: - # app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # package: ${{ github.workspace }}/functionsapp + deploy-workers-test: + runs-on: ubuntu-latest + needs: deploy-workers-dev + environment: test + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure Functions Action' + uses: Azure/functions-action@v1.3.2 + id: fa + with: + app-name: 'ccom-funcs-test' + slot-name: 'production' + package: ${{ github.workspace }}/functionsapp + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + + # Deploy Workers Prod + deploy-workers-prod: + runs-on: ubuntu-latest + needs: deploy-workers-test + environment: prod + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure Functions Action' + uses: Azure/functions-action@v1.3.2 + id: fa + with: + app-name: 'ccom-funcs-prod' + slot-name: 'production' + package: ${{ github.workspace }}/functionsapp + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} # Deploy Web App Dev deploy-web-dev: runs-on: ubuntu-latest needs: [build-and-publish-web, deploy-workers-dev] - # environment: - # name: 'dev' - # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + environment: + name: 'dev' + url: ${{ steps.deploy-web-dev.outputs.webapp-url }} steps: - name: Deploy to Azure Web App @@ -148,44 +151,41 @@ jobs: uses: azure/webapps-deploy@v2 with: app-name: 'ccom-dev' - # TODO: Change these back after testing - # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }} + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' # Deploy Web App Test - # deploy-web-test: - # runs-on: ubuntu-latest - # needs: [deploy-web-dev, deploy-workers-test] - # environment: - # name: 'test' - # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - # steps: - # - name: Deploy to Azure Web App - # id: deploy-to-webapp - # uses: azure/webapps-deploy@v2 - # with: - # app-name: 'ccom-test' - # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - - - # # Deploy Web App Prod - # deploy-web-prod: - # runs-on: ubuntu-latest - # needs: [deploy-web-test, deploy-workers-prod] - # environment: - # name: 'prod' - # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - # steps: - # - name: Deploy to Azure Web App - # id: deploy-to-webapp - # uses: azure/webapps-deploy@v2 - # with: - # app-name: 'ccom' - # slot-name: 'production' - # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + deploy-web-test: + runs-on: ubuntu-latest + needs: [deploy-web-dev, deploy-workers-test] + environment: + name: 'test' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom-test' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + + + # Deploy Web App Prod + deploy-web-prod: + runs-on: ubuntu-latest + needs: [deploy-web-test, deploy-workers-prod] + environment: + name: 'prod' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom' + slot-name: 'production' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' From 97ee3d04cfe50e58f20717395713acf290c280ff Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 5 Mar 2024 12:55:21 +0000 Subject: [PATCH 46/54] Delete legacy workflows --- .github/workflows/deploy.functions.yml | 111 --------------------- .github/workflows/devel.yml | 33 ------ .github/workflows/devel_ccom-funcs-dev.yml | 54 ---------- .github/workflows/prod_ccom-func-prod.yml | 54 ---------- .github/workflows/prod_ccom-prod.yml | 51 ---------- .github/workflows/test_ccom-funcs-test.yml | 54 ---------- .github/workflows/test_ccom-test.yml | 50 ---------- 7 files changed, 407 deletions(-) delete mode 100644 .github/workflows/deploy.functions.yml delete mode 100644 .github/workflows/devel.yml delete mode 100644 .github/workflows/devel_ccom-funcs-dev.yml delete mode 100644 .github/workflows/prod_ccom-func-prod.yml delete mode 100644 .github/workflows/prod_ccom-prod.yml delete mode 100644 .github/workflows/test_ccom-funcs-test.yml delete mode 100644 .github/workflows/test_ccom-test.yml diff --git a/.github/workflows/deploy.functions.yml b/.github/workflows/deploy.functions.yml deleted file mode 100644 index 62574e6be..000000000 --- a/.github/workflows/deploy.functions.yml +++ /dev/null @@ -1,111 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/azure/functions-action -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy Python project to Azure Function App - ccom-funcs-dev - -on: - push: - branches: - - master - workflow_dispatch: - -env: - AZURE_FUNCTIONAPP_PACKAGE_PATH: './app/workers' - PYTHON_VERSION: '3.8' # set this to the python version to use (supports 3.6, 3.7, 3.8) - -jobs: - build-and-publish: - runs-on: ubuntu-latest - steps: - - name: 'Checkout GitHub Action' - uses: actions/checkout@v2 - - - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@v1 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: 'Install Poetry' - uses: snok/install-poetry@v1 - - - name: 'Resolve Project Dependencies Using Poetry' - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - poetry config virtualenvs.create false - poetry export --format requirements.txt --output requirements.txt - popd - shell: bash - - - name: 'Install Dependencies Using Pip' - shell: bash - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - python -m pip install --upgrade pip - pip install -r requirements.txt --target=".python_packages/lib/site-packages" - popd - - - name: Upload artifact for deployment job - uses: actions/upload-artifact@v4 - with: - name: functionsapp - path: ${{ env.AZURE_WEBAPP_PACKAGE_PATH }} - - # Deploy Dev - deploy-dev: - runs-on: ubuntu-latest - needs: build - environment: dev - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure webapp deploy action using publish profile credentials' - uses: azure/webapps-deploy@v2 - with: - app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - package: ${{ github.workspace }}/functionsapp - - # Deploy Test - deploy-test: - runs-on: ubuntu-latest - needs: deploy-dev - environment: test - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure webapp deploy action using publish profile credentials' - uses: azure/webapps-deploy@v2 - with: - app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - package: ${{ github.workspace }}/functionsapp - - # Deploy Production - deploy-prod: - runs-on: ubuntu-latest - needs: deploy-test - environment: prod - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure webapp deploy action using publish profile credentials' - uses: azure/webapps-deploy@v2 - with: - app-name: ${{ env.AZURE_WEBAPP_FUNCTIONS_NAME }} - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - package: ${{ github.workspace }}/functionsapp diff --git a/.github/workflows/devel.yml b/.github/workflows/devel.yml deleted file mode 100644 index 20400a4f8..000000000 --- a/.github/workflows/devel.yml +++ /dev/null @@ -1,33 +0,0 @@ -on: - push: - branches: - - devel - -name: Linux_Container_Python_Workflow - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - # checkout the repo - - name: 'Checkout Github Action' - uses: actions/checkout@v2 - with: - ref: devel - submodules: recursive - - - uses: azure/docker-login@v1 - with: - login-server: ccomreg.azurecr.io - username: ${{ secrets.REGISTRY_USERNAME_DEV }} - password: ${{ secrets.REGISTRY_PASSWORD_DEV }} - - - run: | - docker build -f app/Dockerfile.deploy . -t ccomreg.azurecr.io/dev:${{ github.sha }} - docker push ccomreg.azurecr.io/dev:${{ github.sha }} - - - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom-dev' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }} - images: 'ccomreg.azurecr.io/dev:${{ github.sha }}' diff --git a/.github/workflows/devel_ccom-funcs-dev.yml b/.github/workflows/devel_ccom-funcs-dev.yml deleted file mode 100644 index 31b82f9e9..000000000 --- a/.github/workflows/devel_ccom-funcs-dev.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/azure/functions-action -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy Python project to Azure Function App - ccom-funcs-dev - -on: - push: - branches: - - devel - workflow_dispatch: - -env: - AZURE_FUNCTIONAPP_PACKAGE_PATH: './app/workers' - PYTHON_VERSION: '3.8' # set this to the python version to use (supports 3.6, 3.7, 3.8) - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - name: 'Checkout GitHub Action' - uses: actions/checkout@v2 - - - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@v1 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: 'Install Poetry' - uses: snok/install-poetry@v1 - - - name: 'Resolve Project Dependencies Using Poetry' - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - poetry config virtualenvs.create false - poetry export --format requirements.txt --output requirements.txt - popd - shell: bash - - - name: 'Install Dependencies Using Pip' - shell: bash - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - python -m pip install --upgrade pip - pip install -r requirements.txt --target=".python_packages/lib/site-packages" - popd - - - name: 'Run Azure Functions Action' - uses: Azure/functions-action@v1.3.2 - id: fa - with: - app-name: 'ccom-funcs-dev' - slot-name: 'production' - package: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} diff --git a/.github/workflows/prod_ccom-func-prod.yml b/.github/workflows/prod_ccom-func-prod.yml deleted file mode 100644 index 527315495..000000000 --- a/.github/workflows/prod_ccom-func-prod.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/azure/functions-action -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy Python project to Azure Function App - ccom-funcs-prod - -on: - push: - branches: - - master - workflow_dispatch: - -env: - AZURE_FUNCTIONAPP_PACKAGE_PATH: './app/workers' # set this to the path to your web app project, defaults to the repository root - PYTHON_VERSION: '3.8' # set this to the python version to use (supports 3.6, 3.7, 3.8) - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - name: 'Checkout GitHub Action' - uses: actions/checkout@v2 - - - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@v1 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: 'Install Poetry' - uses: snok/install-poetry@v1 - - - name: 'Resolve Project Dependencies Using Poetry' - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - poetry config virtualenvs.create false - poetry export --format requirements.txt --output requirements.txt - popd - shell: bash - - - name: 'Install Dependencies Using Pip' - shell: bash - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - python -m pip install --upgrade pip - pip install -r requirements.txt --target=".python_packages/lib/site-packages" - popd - - - name: 'Run Azure Functions Action' - uses: Azure/functions-action@v1 - id: fa - with: - app-name: 'ccom-funcs-prod' - slot-name: 'production' - package: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE_PROD }} diff --git a/.github/workflows/prod_ccom-prod.yml b/.github/workflows/prod_ccom-prod.yml deleted file mode 100644 index c8ff80ea1..000000000 --- a/.github/workflows/prod_ccom-prod.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy container app to Azure Web App - ccom - -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - build: - runs-on: 'ubuntu-latest' - - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log in to registry - uses: docker/login-action@v1 - with: - registry: https://ccomregprod.azurecr.io/ - username: ${{ secrets.REGISTRY_USERNAME_PROD }} - password: ${{ secrets.REGISTRY_PASSWORD_PROD }} - - - name: Build and push container image to registry - uses: docker/build-push-action@v2 - with: - push: true - tags: ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/prod:${{ github.sha }} - file: ./app/Dockerfile.deploy - - deploy: - runs-on: ubuntu-latest - needs: build - environment: - name: 'production' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom' - slot-name: 'production' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_PROD }} - images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_PROD }}/prod:${{ github.sha }}' diff --git a/.github/workflows/test_ccom-funcs-test.yml b/.github/workflows/test_ccom-funcs-test.yml deleted file mode 100644 index ba89853f0..000000000 --- a/.github/workflows/test_ccom-funcs-test.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/azure/functions-action -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy Python project to Azure Function App - ccom-funcs-test - -on: - push: - branches: - - test - workflow_dispatch: - -env: - AZURE_FUNCTIONAPP_PACKAGE_PATH: './app/workers' # set this to the path to your web app project, defaults to the repository root - PYTHON_VERSION: '3.8' # set this to the python version to use (supports 3.6, 3.7, 3.8) - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - name: 'Checkout GitHub Action' - uses: actions/checkout@v2 - - - name: Setup Python ${{ env.PYTHON_VERSION }} Environment - uses: actions/setup-python@v1 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: 'Install Poetry' - uses: snok/install-poetry@v1 - - - name: 'Resolve Project Dependencies Using Poetry' - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - poetry config virtualenvs.create false - poetry export --format requirements.txt --output requirements.txt - popd - shell: bash - - - name: 'Install Dependencies Using Pip' - shell: bash - run: | - pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' - python -m pip install --upgrade pip - pip install -r requirements.txt --target=".python_packages/lib/site-packages" - popd - - - name: 'Run Azure Functions Action' - uses: Azure/functions-action@v1 - id: fa - with: - app-name: 'ccom-funcs-test' - slot-name: 'production' - package: ${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }} - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE_TEST }} diff --git a/.github/workflows/test_ccom-test.yml b/.github/workflows/test_ccom-test.yml deleted file mode 100644 index dbf68d1fc..000000000 --- a/.github/workflows/test_ccom-test.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Docs for the Azure Web Apps Deploy action: https://github.com/Azure/webapps-deploy -# More GitHub Actions for Azure: https://github.com/Azure/actions - -name: Build and deploy container app to Azure Web App - ccom-test - -on: - push: - branches: - - test - workflow_dispatch: - -jobs: - build: - runs-on: 'ubuntu-latest' - - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log in to registry - uses: docker/login-action@v1 - with: - registry: https://ccomregtest.azurecr.io/ - username: ${{ secrets.REGISTRY_USERNAME_TEST }} - password: ${{ secrets.REGISTRY_PASSWORD_TEST }} - - - name: Build and push container image to registry - uses: docker/build-push-action@v2 - with: - push: true - tags: ccomregtest.azurecr.io/${{ secrets.REGISTRY_USERNAME_TEST }}/test:${{ github.sha }} - file: ./app/Dockerfile.deploy - - deploy: - runs-on: ubuntu-latest - needs: build - environment: - name: 'test' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom-test' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_TEST }} - images: 'ccomregtest.azurecr.io/${{ secrets.REGISTRY_USERNAME_TEST }}/test:${{ github.sha }}' From d33def085db169042b3bffb54f3220bef5a325ab Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 12 Mar 2024 10:19:38 +0000 Subject: [PATCH 47/54] Fix image names --- .github/workflows/deploy.yml | 162 +++++++++++++++++------------------ 1 file changed, 81 insertions(+), 81 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1f450005c..2fd155a00 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ name: Build and Deploy to Azure - ccom on: push: branches: - - master + - refactor/639/update-environment-workflows workflow_dispatch: env: @@ -93,49 +93,49 @@ jobs: publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # Deploy Workers Test - deploy-workers-test: - runs-on: ubuntu-latest - needs: deploy-workers-dev - environment: test - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure Functions Action' - uses: Azure/functions-action@v1.3.2 - id: fa - with: - app-name: 'ccom-funcs-test' - slot-name: 'production' - package: ${{ github.workspace }}/functionsapp - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - - # Deploy Workers Prod - deploy-workers-prod: - runs-on: ubuntu-latest - needs: deploy-workers-test - environment: prod - - steps: - - name: Download artifact from build job - uses: actions/download-artifact@v4 - with: - path: functionsapp - name: functionsapp - - - name: 'Run Azure Functions Action' - uses: Azure/functions-action@v1.3.2 - id: fa - with: - app-name: 'ccom-funcs-prod' - slot-name: 'production' - package: ${{ github.workspace }}/functionsapp - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # # Deploy Workers Test + # deploy-workers-test: + # runs-on: ubuntu-latest + # needs: deploy-workers-dev + # environment: test + + # steps: + # - name: Download artifact from build job + # uses: actions/download-artifact@v4 + # with: + # path: functionsapp + # name: functionsapp + + # - name: 'Run Azure Functions Action' + # uses: Azure/functions-action@v1.3.2 + # id: fa + # with: + # app-name: 'ccom-funcs-test' + # slot-name: 'production' + # package: ${{ github.workspace }}/functionsapp + # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + + # # Deploy Workers Prod + # deploy-workers-prod: + # runs-on: ubuntu-latest + # needs: deploy-workers-test + # environment: prod + + # steps: + # - name: Download artifact from build job + # uses: actions/download-artifact@v4 + # with: + # path: functionsapp + # name: functionsapp + + # - name: 'Run Azure Functions Action' + # uses: Azure/functions-action@v1.3.2 + # id: fa + # with: + # app-name: 'ccom-funcs-prod' + # slot-name: 'production' + # package: ${{ github.workspace }}/functionsapp + # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} # Deploy Web App Dev deploy-web-dev: @@ -152,40 +152,40 @@ jobs: with: app-name: 'ccom-dev' publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - - # Deploy Web App Test - deploy-web-test: - runs-on: ubuntu-latest - needs: [deploy-web-dev, deploy-workers-test] - environment: - name: 'test' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom-test' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - - - # Deploy Web App Prod - deploy-web-prod: - runs-on: ubuntu-latest - needs: [deploy-web-test, deploy-workers-prod] - environment: - name: 'prod' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom' - slot-name: 'production' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomregprod.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + + # # Deploy Web App Test + # deploy-web-test: + # runs-on: ubuntu-latest + # needs: [deploy-web-dev, deploy-workers-test] + # environment: + # name: 'test' + # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + # steps: + # - name: Deploy to Azure Web App + # id: deploy-to-webapp + # uses: azure/webapps-deploy@v2 + # with: + # app-name: 'ccom-test' + # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + + + # # Deploy Web App Prod + # deploy-web-prod: + # runs-on: ubuntu-latest + # needs: [deploy-web-test, deploy-workers-prod] + # environment: + # name: 'prod' + # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + # steps: + # - name: Deploy to Azure Web App + # id: deploy-to-webapp + # uses: azure/webapps-deploy@v2 + # with: + # app-name: 'ccom' + # slot-name: 'production' + # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' From 4db0acc11d18c62720acc3cdfafcfa1ad59605bf Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 12 Mar 2024 10:24:52 +0000 Subject: [PATCH 48/54] Fix env secrets --- .github/workflows/deploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2fd155a00..0922437b1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -90,7 +90,8 @@ jobs: app-name: 'ccom-funcs-dev' slot-name: 'production' package: ${{ github.workspace }}/functionsapp - publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # change back to env. + publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} # # Deploy Workers Test @@ -151,7 +152,8 @@ jobs: uses: azure/webapps-deploy@v2 with: app-name: 'ccom-dev' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + # change back to env.AZURE_WEBAPP_PUBLISH_PROFILE + publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }} images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' # # Deploy Web App Test From 818ef87911f507779855632b231be900cad9d05e Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Tue, 12 Mar 2024 10:47:03 +0000 Subject: [PATCH 49/54] Revert commented code --- .github/workflows/deploy.yml | 168 +++++++++++++++++------------------ 1 file changed, 83 insertions(+), 85 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0922437b1..1f227a1d1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ name: Build and Deploy to Azure - ccom on: push: branches: - - refactor/639/update-environment-workflows + - master workflow_dispatch: env: @@ -90,53 +90,52 @@ jobs: app-name: 'ccom-funcs-dev' slot-name: 'production' package: ${{ github.workspace }}/functionsapp - # change back to env. - publish-profile: ${{ secrets.AZURE_FUNCAPP_PUBLISH_PROFILE }} - - - # # Deploy Workers Test - # deploy-workers-test: - # runs-on: ubuntu-latest - # needs: deploy-workers-dev - # environment: test - - # steps: - # - name: Download artifact from build job - # uses: actions/download-artifact@v4 - # with: - # path: functionsapp - # name: functionsapp - - # - name: 'Run Azure Functions Action' - # uses: Azure/functions-action@v1.3.2 - # id: fa - # with: - # app-name: 'ccom-funcs-test' - # slot-name: 'production' - # package: ${{ github.workspace }}/functionsapp - # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - - # # Deploy Workers Prod - # deploy-workers-prod: - # runs-on: ubuntu-latest - # needs: deploy-workers-test - # environment: prod - - # steps: - # - name: Download artifact from build job - # uses: actions/download-artifact@v4 - # with: - # path: functionsapp - # name: functionsapp - - # - name: 'Run Azure Functions Action' - # uses: Azure/functions-action@v1.3.2 - # id: fa - # with: - # app-name: 'ccom-funcs-prod' - # slot-name: 'production' - # package: ${{ github.workspace }}/functionsapp - # publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + + + # Deploy Workers Test + deploy-workers-test: + runs-on: ubuntu-latest + needs: deploy-workers-dev + environment: test + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure Functions Action' + uses: Azure/functions-action@v1.3.2 + id: fa + with: + app-name: 'ccom-funcs-test' + slot-name: 'production' + package: ${{ github.workspace }}/functionsapp + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + + # Deploy Workers Prod + deploy-workers-prod: + runs-on: ubuntu-latest + needs: deploy-workers-test + environment: prod + + steps: + - name: Download artifact from build job + uses: actions/download-artifact@v4 + with: + path: functionsapp + name: functionsapp + + - name: 'Run Azure Functions Action' + uses: Azure/functions-action@v1.3.2 + id: fa + with: + app-name: 'ccom-funcs-prod' + slot-name: 'production' + package: ${{ github.workspace }}/functionsapp + publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} # Deploy Web App Dev deploy-web-dev: @@ -152,42 +151,41 @@ jobs: uses: azure/webapps-deploy@v2 with: app-name: 'ccom-dev' - # change back to env.AZURE_WEBAPP_PUBLISH_PROFILE - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }} + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - # # Deploy Web App Test - # deploy-web-test: - # runs-on: ubuntu-latest - # needs: [deploy-web-dev, deploy-workers-test] - # environment: - # name: 'test' - # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - # steps: - # - name: Deploy to Azure Web App - # id: deploy-to-webapp - # uses: azure/webapps-deploy@v2 - # with: - # app-name: 'ccom-test' - # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - - - # # Deploy Web App Prod - # deploy-web-prod: - # runs-on: ubuntu-latest - # needs: [deploy-web-test, deploy-workers-prod] - # environment: - # name: 'prod' - # url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - # steps: - # - name: Deploy to Azure Web App - # id: deploy-to-webapp - # uses: azure/webapps-deploy@v2 - # with: - # app-name: 'ccom' - # slot-name: 'production' - # publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - # images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + # Deploy Web App Test + deploy-web-test: + runs-on: ubuntu-latest + needs: [deploy-web-dev, deploy-workers-test] + environment: + name: 'test' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom-test' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + + + # Deploy Web App Prod + deploy-web-prod: + runs-on: ubuntu-latest + needs: [deploy-web-test, deploy-workers-prod] + environment: + name: 'prod' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom' + slot-name: 'production' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' From 226053576072919d75da07fb3f8a7ab869a38b64 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 13 Mar 2024 16:24:35 +0000 Subject: [PATCH 50/54] Remove outdated test --- app/api/test/test_views.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/app/api/test/test_views.py b/app/api/test/test_views.py index e264873f8..d211bdc58 100644 --- a/app/api/test/test_views.py +++ b/app/api/test/test_views.py @@ -746,31 +746,6 @@ def setUp(self): # Set up API client self.client = APIClient() - def test_admin_user_get(self): - """Users who are admins of the parent dataset can see all public SRs - and restricted SRs whose parent dataset they are the admin of. - """ - User = get_user_model() - - # user who is an admin of the parent dataset - admin_user = User.objects.create(username="gandalf", password="fiwuenfwinefiw") - self.project.members.add(admin_user) - self.public_dataset.admins.add(admin_user) - self.restricted_dataset.admins.add(admin_user) - - # Get data admin_user should be able to see - self.client.force_authenticate(admin_user) - admin_response = self.client.get( - "/api/scanreportactiveconceptfilter/?content_type=15" - ) - self.assertEqual(admin_response.status_code, 200) - observed_objs = sorted([obj.get("id") for obj in admin_response.data]) - print(observed_objs) - expected_objs = [] - - # Assert the observed results are the same as the expected - self.assertListEqual(observed_objs, expected_objs) - def test_editor_get(self): """Users who are editors of the parent dataset can see all public SRs and restricted SRs whose parent dataset they are an editor of. From 1c035684d773be6bfed6d6bba41956be283fca7f Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 13 Mar 2024 16:27:45 +0000 Subject: [PATCH 51/54] Remove outdated tests --- app/api/test/test_views.py | 70 -------------------------------------- 1 file changed, 70 deletions(-) diff --git a/app/api/test/test_views.py b/app/api/test/test_views.py index d211bdc58..4e0c6e371 100644 --- a/app/api/test/test_views.py +++ b/app/api/test/test_views.py @@ -746,76 +746,6 @@ def setUp(self): # Set up API client self.client = APIClient() - def test_editor_get(self): - """Users who are editors of the parent dataset can see all public SRs - and restricted SRs whose parent dataset they are an editor of. - """ - User = get_user_model() - - # user who is an editor of the parent dataset - editor_user = User.objects.create(username="gandalf", password="fiwuenfwinefiw") - self.project.members.add(editor_user) - self.public_dataset.editors.add(editor_user) - self.restricted_dataset.editors.add(editor_user) - - # Get data editor_user should be able to see - self.client.force_authenticate(editor_user) - editor_response = self.client.get( - "/api/scanreportactiveconceptfilter/?content_type=15" - ) - self.assertEqual(editor_response.status_code, 200) - observed_objs = sorted([obj.get("id") for obj in editor_response.data]) - expected_objs = [] - - # Assert the observed results are the same as the expected - self.assertListEqual(observed_objs, expected_objs) - - def test_viewer_get(self): - """Users who are viewers of the parent dataset can see all public SRs - and restricted SRs whose parent dataset they are a viewer of. - """ - User = get_user_model() - - # user who is a viewer of the parent dataset - viewer_user = User.objects.create(username="gandalf", password="fiwuenfwinefiw") - self.project.members.add(viewer_user) - self.public_dataset.viewers.add(viewer_user) - self.restricted_dataset.viewers.add(viewer_user) - - # Get data viewer_user should be able to see - self.client.force_authenticate(viewer_user) - viewer_response = self.client.get( - "/api/scanreportactiveconceptfilter/?content_type=15" - ) - self.assertEqual(viewer_response.status_code, 200) - observed_objs = sorted([obj.get("id") for obj in viewer_response.data]) - expected_objs = [] - - # Assert the observed results are the same as the expected - self.assertListEqual(observed_objs, expected_objs) - - def test_author_get(self): - """Authors can see all public SRs and restricted SRs they are the author of.""" - User = get_user_model() - - # user who is the author of a scan report - author_user = User.objects.create(username="gandalf", password="fiwuenfwinefiw") - self.project.members.add(author_user) - self.scanreport1.author = author_user - self.scanreport1.save() - - # Get data admin_user should be able to see - self.client.force_authenticate(author_user) - author_response = self.client.get( - "/api/scanreportactiveconceptfilter/?content_type=15" - ) - self.assertEqual(author_response.status_code, 200) - observed_objs = sorted([obj.get("id") for obj in author_response.data]) - expected_objs = [] - - # Assert the observed results are the same as the expected - self.assertListEqual(observed_objs, expected_objs) - @mock.patch.dict(os.environ, {"AZ_FUNCTION_USER": "az_functions"}, clear=True) @pytest.mark.skip( reason="Depends on hardcoded IDs, fix: https://github.com/Health-Informatics-UoN/CaRROT-Mapper/issues/637" From da3258667f8aa3a644ee7e38ee86aa8d193103e3 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Thu, 14 Mar 2024 11:05:06 +0000 Subject: [PATCH 52/54] Add python path to worker pytest --- .github/workflows/test.yml | 1 + app/workers/pyproject.toml | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef62f1e26..df95c0272 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,7 @@ jobs: - name: Pytest coverage comment uses: MishaKav/pytest-coverage-comment@main + working-directory: app/workers with: pytest-coverage-path: pytest-coverage.txt junitxml-path: pytest.xml diff --git a/app/workers/pyproject.toml b/app/workers/pyproject.toml index ace928f28..097d5d633 100644 --- a/app/workers/pyproject.toml +++ b/app/workers/pyproject.toml @@ -30,7 +30,4 @@ build-backend = "poetry.core.masonry.api" [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra -q" -testpaths = [ - "/test", -] +pythonpath = ["."] From 1d50647568dadb04045dcaf2c2d8ca358f313f3b Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Thu, 14 Mar 2024 11:06:29 +0000 Subject: [PATCH 53/54] Change pytest comment --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df95c0272..023234ac2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,10 +40,9 @@ jobs: - name: Pytest coverage comment uses: MishaKav/pytest-coverage-comment@main - working-directory: app/workers with: - pytest-coverage-path: pytest-coverage.txt - junitxml-path: pytest.xml + pytest-coverage-path: app/workers/pytest-coverage.txt + junitxml-path: app/workers/pytest.xml test-django: runs-on: ubuntu-latest From 3e91126fb02cd5a218e4db3ef09bcdb12e7333a5 Mon Sep 17 00:00:00 2001 From: Andy Rae Date: Wed, 20 Mar 2024 14:37:47 +0000 Subject: [PATCH 54/54] Rename prod environment to production --- .github/workflows/deploy.yml | 82 ++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1f227a1d1..cfa571804 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -92,6 +92,22 @@ jobs: package: ${{ github.workspace }}/functionsapp publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} + # Deploy Web App Dev + deploy-web-dev: + runs-on: ubuntu-latest + needs: [build-and-publish-web, deploy-workers-dev] + environment: + name: 'dev' + url: ${{ steps.deploy-web-dev.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom-dev' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' # Deploy Workers Test deploy-workers-test: @@ -115,11 +131,28 @@ jobs: package: ${{ github.workspace }}/functionsapp publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # Deploy Workers Prod - deploy-workers-prod: + # Deploy Web App Test + deploy-web-test: + runs-on: ubuntu-latest + needs: [deploy-web-dev, deploy-workers-test] + environment: + name: 'test' + url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} + + steps: + - name: Deploy to Azure Web App + id: deploy-to-webapp + uses: azure/webapps-deploy@v2 + with: + app-name: 'ccom-test' + publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} + images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' + + # Deploy Workers Production + deploy-workers-production: runs-on: ubuntu-latest needs: deploy-workers-test - environment: prod + environment: production steps: - name: Download artifact from build job @@ -137,47 +170,12 @@ jobs: package: ${{ github.workspace }}/functionsapp publish-profile: ${{ env.AZURE_FUNCAPP_PUBLISH_PROFILE }} - # Deploy Web App Dev - deploy-web-dev: - runs-on: ubuntu-latest - needs: [build-and-publish-web, deploy-workers-dev] - environment: - name: 'dev' - url: ${{ steps.deploy-web-dev.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom-dev' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - - # Deploy Web App Test - deploy-web-test: - runs-on: ubuntu-latest - needs: [deploy-web-dev, deploy-workers-test] - environment: - name: 'test' - url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} - - steps: - - name: Deploy to Azure Web App - id: deploy-to-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'ccom-test' - publish-profile: ${{ env.AZURE_WEBAPP_PUBLISH_PROFILE }} - images: 'ccomreg.azurecr.io/${{ secrets.REGISTRY_USERNAME_DEV }}/latest:${{ github.sha }}' - - - # Deploy Web App Prod - deploy-web-prod: + # Deploy Web App Production + deploy-web-production: runs-on: ubuntu-latest - needs: [deploy-web-test, deploy-workers-prod] + needs: [deploy-web-test, deploy-workers-production] environment: - name: 'prod' + name: 'production' url: ${{ steps.deploy-to-webapp.outputs.webapp-url }} steps: