docs: mention using docs.openverse.org links for plans in project_thread.md #12546
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI + CD | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: "The tag to assign to the images built in the workflow." | |
| type: string | |
| required: false | |
| default: "" | |
| # This is useful when dispatching on a branch other than `main`. | |
| perform_deploy: | |
| description: "Publish images and deploy staging?" | |
| type: boolean | |
| required: false | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| # Don't continue building images for a PR if the PR is updated quickly | |
| # For other workflows, allow them to complete and just block on them. This | |
| # ensures deployments in particular to happen in series rather than parallel. | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| ########### | |
| # Helpers # | |
| ########### | |
| get-changes: | |
| name: Get changes | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| changes: ${{ steps.paths-filter.outputs.changes }} | |
| catalog: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'catalog') }} | |
| ingestion_server: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ingestion_server') }} | |
| indexer_worker: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'indexer_worker') }} | |
| api: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'api') }} | |
| frontend: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'frontend') }} | |
| documentation: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'documentation') }} | |
| ci_cd: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'ci_cd') }} | |
| py_packages: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'py_packages') }} | |
| js_packages: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'js_packages') }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get changes | |
| id: paths-filter | |
| uses: ./.github/actions/get-changes | |
| get-image-tag: | |
| name: Get image tag | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image_tag: ${{ steps.get-image-tag.outputs.image_tag }} | |
| steps: | |
| - name: Get image tag | |
| id: get-image-tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.image_tag }}" != "" ]]; then | |
| echo "image_tag=${{ inputs.image_tag }}" | tee "$GITHUB_OUTPUT" | |
| else | |
| echo "image_tag=${{ github.sha }}" | tee "$GITHUB_OUTPUT" | |
| fi | |
| determine-images: | |
| name: Determine images to build and publish | |
| runs-on: ubuntu-latest | |
| outputs: | |
| do_build: ${{ steps.set-matrix.outputs.do_build }} | |
| build_matrix: ${{ steps.set-matrix.outputs.build_matrix }} | |
| do_publish: ${{ steps.set-matrix.outputs.do_publish }} | |
| publish_matrix: ${{ steps.set-matrix.outputs.publish_matrix }} | |
| needs: | |
| - get-changes | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set matrix images | |
| id: set-matrix | |
| env: | |
| CHANGES: ${{ needs.get-changes.outputs.changes }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python set_matrix_images.py | |
| ############# | |
| # Universal # | |
| ############# | |
| lint: # This includes type-checking of the frontend. | |
| name: Lint files | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| # PDM is needed to run Vale | |
| setup_python: "true" | |
| # Node.js is needed by lint actions. | |
| install_recipe: "node-install" | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Run pre-commit to lint files | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| just precommit | |
| just lint | |
| validate-codeowners: | |
| name: Validate CODEOWNERS | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: mszostok/[email protected] | |
| with: | |
| checks: "files,duppatterns,syntax" | |
| experimental_checks: "notowned,avoid-shadowing" | |
| build-images: | |
| name: Build Docker images | |
| if: needs.determine-images.outputs.do_build == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.determine-images.outputs.build_matrix) }} | |
| needs: | |
| - get-image-tag | |
| - lint | |
| - determine-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| # Sets up `just` and Node.js. | |
| # Node.js is needed for the frontend to download translations. | |
| # `just` is needed to run the recipes and set build args. | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| setup_nodejs: ${{ matrix.image == 'frontend' && 'true' || 'false' }} | |
| install_recipe: ${{ matrix.image == 'frontend' && 'node-install' || '' }} | |
| locales: ${{ matrix.image == 'frontend' && 'production' || '' }} | |
| # Sets build args specifying versions needed to build Docker image. | |
| - name: Prepare build args | |
| id: prepare-build-args | |
| run: | | |
| just versions | tee "$GITHUB_OUTPUT" | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| install: true | |
| - name: Build image `${{ matrix.image }}` | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ${{ matrix.context }} | |
| target: ${{ matrix.target }} | |
| push: false | |
| tags: openverse-${{ matrix.image }} | |
| file: ${{ matrix.file }} | |
| cache-from: type=gha,scope=${{ matrix.image }} | |
| cache-to: type=gha,scope=${{ matrix.image }} | |
| outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar | |
| build-contexts: ${{ matrix.build-contexts }} | |
| build-args: | | |
| SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} | |
| CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }} | |
| CATALOG_AIRFLOW_VERSION=${{ steps.prepare-build-args.outputs.catalog_airflow_version }} | |
| INDEXER_WORKER_PY_VERSION=${{ steps.prepare-build-args.outputs.indexer_worker_py_version }} | |
| API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }} | |
| INGESTION_PY_VERSION=${{ steps.prepare-build-args.outputs.ingestion_py_version }} | |
| FRONTEND_NODE_VERSION=${{ steps.prepare-build-args.outputs.frontend_node_version }} | |
| FRONTEND_PNPM_VERSION=${{ steps.prepare-build-args.outputs.frontend_pnpm_version }} | |
| PGCLI_VERSION=${{ steps.prepare-build-args.outputs.pgcli_version }} | |
| ${{ matrix.build-args || '' }} | |
| - name: Upload image `${{ matrix.image }}` | |
| id: upload-img | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.image }} | |
| path: /tmp/${{ matrix.image }}.tar | |
| - name: Show artifact ID | |
| run: | | |
| echo '${{ matrix.image }} artifact ID is ${{ steps.upload-img.outputs.artifact-id }}' | |
| ########### | |
| # Catalog # | |
| ########### | |
| test-cat: | |
| name: Run tests for the catalog | |
| if: | | |
| needs.get-changes.outputs.catalog == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run the tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db | |
| # Sets build args specifying versions needed to build Docker image. | |
| - name: Prepare build args | |
| id: prepare-build-args | |
| run: | | |
| just versions | tee "$GITHUB_OUTPUT" | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| install: true | |
| - name: Build catalog dev image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: catalog | |
| target: cat | |
| push: false | |
| load: true | |
| tags: openverse-catalog | |
| cache-from: type=gha,scope=catalog_dev | |
| cache-to: type=gha,scope=catalog_dev | |
| build-args: | | |
| CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }} | |
| CATALOG_AIRFLOW_VERSION=${{ steps.prepare-build-args.outputs.catalog_airflow_version }} | |
| REQUIREMENTS_FILE=requirements-dev.txt | |
| - name: Run tests | |
| run: | | |
| just catalog/test --extended | |
| catalog-checks: | |
| name: Run catalog checks | |
| if: | | |
| needs.get-changes.outputs.catalog == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: true | |
| # Node.js is not needed to run the tests. | |
| setup_nodejs: false | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db catalog | |
| - name: Check generated documentation | |
| run: | | |
| just catalog/generate-docs dag true | |
| just catalog/generate-docs media-props true | |
| ################## | |
| # Indexer worker # | |
| ################## | |
| test-indexer-worker: | |
| name: Run tests for indexer worker | |
| if: | | |
| needs.get-changes.outputs.indexer_worker == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run the tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db | |
| # Sets build args specifying versions needed to build Docker image. | |
| - name: Prepare build args | |
| id: prepare-build-args | |
| run: | | |
| just versions | tee "$GITHUB_OUTPUT" | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| install: true | |
| - name: Build indexer worker dev image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: indexer_worker | |
| target: indexer_worker | |
| push: false | |
| load: true | |
| tags: openverse-catalog_indexer_worker | |
| cache-from: type=gha,scope=indexer_worker_dev | |
| cache-to: type=gha,scope=indexer_worker_dev | |
| build-args: | | |
| INDEXER_WORKER_PY_VERSION=${{ steps.prepare-build-args.outputs.indexer_worker_py_version }} | |
| PDM_INSTALL_ARGS=--dev | |
| - name: Run tests | |
| run: | | |
| just indexer_worker/test | |
| #################### | |
| # Ingestion server # | |
| #################### | |
| test-ing: | |
| name: Run tests for ingestion-server | |
| if: | | |
| needs.get-changes.outputs.ingestion_server == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| # Python is needed to run the test. | |
| setup_nodejs: false | |
| # Node.js is not needed to run ingestion server tests. | |
| install_recipe: ingestion_server/install | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db ingestion_server | |
| - name: Run ingestion-server tests | |
| run: | | |
| just env | |
| just ingestion_server/test-local | |
| - name: Print ingestion-server test logs | |
| run: | | |
| just ingestion_server/test-logs > ingestion_server/test/ingestion_logs.txt | |
| cat ingestion_server/test/ingestion_logs.txt | |
| - name: Upload ingestion test logs | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ing_logs | |
| path: ingestion_server/test/ingestion_logs.txt | |
| ####### | |
| # API # | |
| ####### | |
| test-api: | |
| name: Run tests for the API | |
| if: | | |
| needs.get-changes.outputs.ingestion_server == 'true' || | |
| needs.get-changes.outputs.api == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - build-images | |
| - get-image-tag | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run API tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db ingestion_server catalog | |
| # Sets build args specifying versions needed to build Docker image. | |
| - name: Prepare build args | |
| id: prepare-build-args | |
| run: | | |
| just versions | tee "$GITHUB_OUTPUT" | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| install: true | |
| - name: Build API dev image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: api | |
| target: api | |
| push: false | |
| load: true | |
| tags: openverse-api | |
| cache-from: type=gha,scope=api_dev | |
| cache-to: type=gha,scope=api_dev | |
| build-contexts: packages=./packages/python | |
| build-args: | | |
| SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} | |
| API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }} | |
| PDM_INSTALL_ARGS=--dev | |
| - name: Start Catalog | |
| run: just catalog/up | |
| - name: Start API, ingest and index test data | |
| run: just api/init | |
| - name: Run API tests | |
| run: just api/test | |
| - name: Print API test logs | |
| if: success() || failure() | |
| run: | | |
| just logs > api_logs | |
| cat api_logs | |
| - name: Upload API test logs | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: api_logs | |
| path: api_logs | |
| django-checks: | |
| name: Run Django checks | |
| if: | | |
| needs.get-changes.outputs.api == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - build-images | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - check_django | |
| - validate_openapi | |
| - check_migrations | |
| - test_doc | |
| - test_media_props | |
| include: | |
| - name: check_django | |
| recipe: api/dj check | |
| - name: validate_openapi | |
| recipe: api/dj spectacular --format openapi-json --validate --file openapi.json | |
| - name: check_migrations | |
| recipe: api/dj makemigrations --check | |
| - name: test_doc | |
| recipe: api/doc-test | |
| - name: test_media_props | |
| recipe: api/generate-docs | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| # Python is not needed to run the tests. | |
| setup_nodejs: false | |
| # Node.js is not needed to run API tests. | |
| install_recipe: "" | |
| - name: Load Docker images | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: upstream_db ingestion_server api | |
| - name: Run check recipe | |
| run: just ${{ matrix.recipe }} | |
| env: | |
| DC_USER: root | |
| - name: Upload schema | |
| if: matrix.name == 'test_doc' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: openapi.json | |
| path: ./api/openapi.json | |
| # This job runs when `django-checks` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-django-checks: | |
| name: Run Django checks | |
| if: | | |
| !cancelled() && | |
| needs.django-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - django-checks | |
| strategy: | |
| matrix: | |
| name: | |
| - check_django | |
| - validate_openapi | |
| - check_migrations | |
| - test_doc | |
| steps: | |
| - name: Pass | |
| run: echo 'Django checks are skipped because API is unchanged.' | |
| py-package-checks: | |
| name: Run checks for Python packages/* | |
| if: | | |
| needs.get-changes.outputs.py_packages == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| packages: | |
| - packages/python/openverse-attribution | |
| script: | |
| - build | |
| - test | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| install_recipe: packages/python/openverse-attribution/install | |
| - name: Run Python packages checks | |
| run: pdm run -p ${{ matrix.packages }} ${{ matrix.script }} | |
| # This job runs when `py-package-checks` doesn't and always passes, thus | |
| # allowing PRs to meet the required checks criteria and be merged. | |
| bypass-py-package-checks: | |
| name: Run checks for Python packages/* | |
| if: | | |
| !cancelled() && | |
| needs.py-package-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - py-package-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - build | |
| - unit_test | |
| steps: | |
| - name: Pass | |
| run: echo 'Checks for Python packages are skipped because they are unchanged.' | |
| ############ | |
| # Frontend # | |
| ############ | |
| nuxt-build: | |
| name: Check Nuxt build | |
| if: | | |
| needs.get-changes.outputs.frontend == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| locales: "production" | |
| - name: Run build | |
| run: just frontend/run build | |
| env: | |
| DEPLOYMENT_ENV: production | |
| nuxt-load-test: | |
| name: Load test local frontend | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nuxt-build | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| locales: "test" | |
| - name: Build and run Nuxt and Talkback | |
| run: | | |
| just frontend/run build | |
| just frontend/run talkback & | |
| env NUXT_PUBLIC_API_URL=http://127.0.0.1:49153/ just frontend/run start & | |
| - name: Setup k6 | |
| uses: grafana/setup-k6-action@v1 | |
| - name: Wait for local Talkback to be available | |
| # 10 seconds, talkback has a slow startup; this step will fail if Talkback never becomes available | |
| run: npx wait-port -t 10000 :49153 | |
| - name: Wait for local Nuxt to be available | |
| # 2 seconds, this step will fail if Nuxt never becomes available | |
| run: npx wait-port -t 2000 http://127.0.0.1:8443/healthcheck | |
| - name: Run k6 frontend all against local Nuxt | |
| run: | | |
| just k6 frontend all \ | |
| -e service_url=http://127.0.0.1:8443/ \ | |
| -e text_summary=/tmp/k6-summary.txt | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: k6-output | |
| path: /tmp/k6-summary.txt | |
| - name: Check if comment needs to be posted | |
| # Do not post comments on forks, or when merging a PR | |
| id: set-post-comment | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" && \ | |
| "${{ github.event.pull_request.head.repo.owner.login }}" == "WordPress" && \ | |
| "${{ github.actor }}" != "dependabot[bot]" ]]; then | |
| echo "post_comment=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "post_comment=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Make comment body | |
| if: steps.set-post-comment.outputs.post_comment == 'true' | |
| shell: python | |
| run: | | |
| from pathlib import Path | |
| summary = Path("/tmp/k6-summary.txt").read_text() | |
| Path("/tmp/k6-summary-comment.txt").write_text(f""" | |
| ## Latest k6 run output[^update] | |
| ``` | |
| {summary} | |
| ``` | |
| [^update]: This comment will automatically update with new output each time k6 runs for this PR | |
| """) | |
| - uses: peter-evans/find-comment@v3 | |
| if: steps.set-post-comment.outputs.post_comment == 'true' | |
| id: k6-summary-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: Latest k6 run output | |
| - name: Post comment summary | |
| uses: peter-evans/create-or-update-comment@v4 | |
| if: steps.set-post-comment.outputs.post_comment == 'true' | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| edit-mode: replace | |
| body-path: /tmp/k6-summary-comment.txt | |
| comment-id: ${{ steps.k6-summary-comment.outputs.comment-id }} | |
| nuxt-checks: | |
| name: Run Nuxt checks | |
| if: | | |
| needs.get-changes.outputs.frontend == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - unit_test | |
| - test_media_props | |
| - frontend_init | |
| include: | |
| - name: unit_test | |
| recipe: frontend/run test:unit | |
| - name: test_media_props | |
| recipe: frontend/generate-docs | |
| - name: frontend_init | |
| recipe: frontend/init | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| locales: "test" | |
| - name: Run check recipe | |
| run: just ${{ matrix.recipe }} | |
| # This job runs when `nuxt-checks` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-nuxt-checks: | |
| name: Run Nuxt checks | |
| if: | | |
| !cancelled() && | |
| needs.nuxt-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nuxt-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - unit_test | |
| steps: | |
| - name: Pass | |
| run: echo 'Playwright tests are skipped because frontend is unchanged.' | |
| js-package-checks: | |
| name: Run checks for JS packages/* | |
| if: | | |
| needs.get-changes.outputs.js_packages == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - build | |
| - unit_test | |
| include: | |
| - name: build | |
| script: "build" | |
| - name: unit_test | |
| script: "test:unit" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| - name: Run JS packages checks | |
| run: pnpm --filter ./packages/js/* run --aggregate-output ${{ matrix.script }} | |
| # This job runs when `js-package-checks` doesn't and always passes, thus | |
| # allowing PRs to meet the required checks criteria and be merged. | |
| bypass-js-package-checks: | |
| name: Run checks for JS packages/* | |
| if: | | |
| !cancelled() && | |
| needs.js-package-checks.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - js-package-checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - build | |
| - unit_test | |
| steps: | |
| - name: Pass | |
| run: echo 'Checks for JS packages are skipped because they are unchanged.' | |
| playwright: | |
| name: Run Playwright tests | |
| if: | | |
| needs.get-changes.outputs.frontend == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: | |
| - get-changes | |
| - lint | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: | |
| - playwright_vr | |
| - playwright_e2e | |
| - storybook | |
| include: | |
| - name: playwright_vr | |
| script: "test:playwright visual-regression -u" | |
| - name: playwright_e2e | |
| script: "test:playwright e2e" | |
| - name: storybook | |
| script: "test:storybook -u" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| locales: "test" | |
| - name: Run Playwright tests | |
| run: just frontend/run ${{ matrix.script }} --workers=2 | |
| - name: Check for changes after running tests with -u | |
| id: check_snapshots_changed | |
| if: failure() || success() | |
| run: | | |
| diff_names="$(git diff --binary --name-only)" | |
| if [ -n "$diff_names" ]; then | |
| git add . && git diff --staged --binary > /tmp/snapshot_diff.patch | |
| echo "snapshots_changed=true" | tee "$GITHUB_OUTPUT" | |
| printf "The following snapshots were updated:\n" | |
| while IFS= read -r filename; do | |
| printf "\t- %s\n" "$filename" | |
| done <<< "$diff_names" | |
| exit 1 | |
| else | |
| echo "snapshots_changed=false" | tee "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() | |
| id: test-results | |
| with: | |
| name: ${{ matrix.name }}_test_results | |
| path: frontend/test-results | |
| # This step only runs if there was a failure and snapshots indeed changed. | |
| - uses: actions/upload-artifact@v4 | |
| if: failure() && steps.check_snapshots_changed.outputs.snapshots_changed == 'true' | |
| id: snapshot-patch | |
| with: | |
| name: ${{ matrix.name }}_snapshot_diff | |
| path: /tmp/snapshot_diff.patch | |
| # This job runs when `playwright` doesn't and always passes, thus allowing | |
| # PRs to meet the required checks criteria and be merged. | |
| bypass-playwright: | |
| name: Run Playwright tests | |
| if: | | |
| !cancelled() && | |
| needs.playwright.result == 'skipped' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - playwright | |
| strategy: | |
| matrix: | |
| name: | |
| - playwright_vr | |
| - playwright_e2e | |
| - storybook | |
| steps: | |
| - name: Pass | |
| run: echo 'Playwright tests are skipped because frontend is unchanged.' | |
| playwright-test-failure-comment: | |
| name: Post Playwright test debugging instructions | |
| if: | | |
| !cancelled() && | |
| github.event_name == 'pull_request' && | |
| needs.playwright.result != '' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - playwright | |
| steps: | |
| - uses: peter-evans/find-comment@v3 | |
| id: test-results-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: Playwright failure test results | |
| - name: Delete existing results comment | |
| uses: actions/github-script@v7 | |
| if: steps.test-results-comment.outputs.comment-id != 0 | |
| with: | |
| script: | | |
| await github.rest.issues.deleteComment({ | |
| repo: context.repo.repo, | |
| owner: context.repo.owner, | |
| comment_id: ${{ steps.test-results-comment.outputs.comment-id }} | |
| }) | |
| console.log('Deleted comment with ID ${{ steps.test-results-comment.outputs.comment-id }}') | |
| - name: Build help body | |
| if: needs.playwright.result == 'failure' | |
| id: help-body | |
| run: | | |
| EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) # Security hardening: https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#understanding-the-risk-of-script-injections | |
| MESSAGE=$(cat <<HEREDOC | |
| help_body<<$EOF | |
| **Playwright failure test results**: <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> | |
| It looks like some of the Playwright tests failed. If you made changes to the frontend UI without updating snapshots, this might be the cause. You can download zipped patches containing the updated snapshots alongside a general trace of the tests under the "Artifacts" section in the above page. They're named in the form \`*_snapshot_diff\` and \`*_test_results\` respectively. | |
| You can read more [on how to use these artifacts in the docs](https://docs.openverse.org/frontend/reference/testing_guidelines.html#debugging). | |
| If the test is flaky, follow the [flaky test triage procedure](https://docs.openverse.org/general/test.html#flaky-tests). | |
| $EOF | |
| HEREDOC | |
| ) | |
| echo "$MESSAGE" | tee "$GITHUB_OUTPUT" | |
| - uses: peter-evans/create-or-update-comment@v4 | |
| id: create-comment | |
| # Do not leave a comment on forks | |
| if: | | |
| needs.playwright.result == 'failure' && | |
| ( | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.owner.login == 'WordPress' && | |
| github.actor != 'dependabot[bot]' | |
| ) | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body: ${{ steps.help-body.outputs.help_body }} | |
| lighthouse-ci: | |
| name: Collect Lighthouse CI results | |
| runs-on: ubuntu-latest | |
| needs: | |
| - nuxt-build | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| with: | |
| setup_python: false | |
| install_recipe: node-install | |
| - name: Run build | |
| run: just frontend/run build | |
| env: | |
| DEPLOYMENT_ENV: production | |
| NODE_ENV: production | |
| - name: Run Lighthouse CI | |
| id: lhci-autorun | |
| # Lighthouse CI runs the webserver for us, as configured in lighthouserc | |
| env: | |
| NODE_ENV: production | |
| run: | | |
| pnpm --package=@lhci/cli dlx lhci autorun --config .github/.lighthouserc.yml | |
| - name: Display Report | |
| if: always() | |
| uses: jackywithawhitedog/lighthouse-viewer-action@v2 | |
| with: | |
| resultsPath: .lighthouseci | |
| lighthouseOutcome: ${{ steps.lhci-autorun.outcome }} | |
| ################# | |
| # Documentation # | |
| ################# | |
| build-docs: | |
| name: Build full-stack docs | |
| if: | | |
| needs.get-changes.outputs.documentation == 'true' || | |
| needs.get-changes.outputs.ci_cd == 'true' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - lint | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup CI env | |
| uses: ./.github/actions/setup-env | |
| - name: Compile documentation | |
| uses: ./.github/actions/build-docs | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # Docs will be located at `/tmp/docs`. | |
| - name: Upload documentation | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: documentation | |
| path: /tmp/docs/ | |
| emit-docs: | |
| name: Emit full-stack docs | |
| # https://github.com/actions/runner/issues/491#issuecomment-850884422 | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| ( | |
| github.event_name == 'push' && | |
| github.repository == 'WordPress/openverse' | |
| ) || | |
| ( | |
| github.event_name == 'pull_request' && | |
| github.event.pull_request.head.repo.owner.login == 'WordPress' && | |
| github.actor != 'dependabot[bot]' | |
| ) | |
| ) && | |
| needs.get-changes.outputs.documentation == 'true' && | |
| (needs.test-ing.result == 'success' || needs.test-ing.result == 'skipped') && | |
| (needs.test-api.result == 'success' || needs.test-api.result == 'skipped') && | |
| (needs.test-cat.result == 'success' || needs.test-cat.result == 'skipped') && | |
| (needs.playwright.result == 'success' || needs.playwright.result == 'skipped') && | |
| needs.build-docs.result == 'success' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - get-changes | |
| - test-cat | |
| - test-ing | |
| - test-api | |
| - playwright | |
| - build-docs | |
| steps: | |
| - name: Download documentation | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: documentation | |
| path: /tmp/docs | |
| - name: Recreate working directory # to avoid superfluous files from getting tracked automatically | |
| run: | | |
| cd .. | |
| sudo rm -rf openverse | |
| mkdir openverse | |
| - name: Checkout repository at `gh-pages` branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: gh-pages | |
| - name: Checkout automations from repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: automation-checkout | |
| - name: Copy existing previews | |
| if: github.event_name == 'push' | |
| run: | | |
| mv /tmp/docs /tmp/gh-pages | |
| mv gh-pages/_preview /tmp/gh-pages/_preview | |
| - name: Replace preview of current PR | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| cp -r gh-pages /tmp/gh-pages | |
| sudo rm -rf /tmp/gh-pages/_preview/${{ github.event.pull_request.number }} | |
| mv /tmp/docs /tmp/gh-pages/_preview/${{ github.event.pull_request.number }} | |
| - name: Determine which files have changed | |
| if: github.event_name == 'pull_request' | |
| id: preview_diff | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| PYTHONPATH: ${{ github.workspace }}/automation-checkout/automations/python | |
| working-directory: ${{ github.workspace }}/automation-checkout/automations/python/workflows | |
| run: python get_folder_differences.py | |
| - name: Deploy | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: /tmp/gh-pages | |
| force_orphan: true | |
| cname: docs.openverse.org | |
| - uses: peter-evans/find-comment@v3 | |
| if: github.event_name == 'pull_request' | |
| id: final-preview-comment | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: Full-stack documentation | |
| - uses: peter-evans/create-or-update-comment@v4 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-id: ${{ steps.final-preview-comment.outputs.comment-id }} | |
| edit-mode: replace | |
| body: ${{ steps.preview_diff.outputs.body }} | |
| - name: Checkout repository # again, to enable cleaning | |
| uses: actions/checkout@v4 | |
| if: success() || failure() | |
| ################# | |
| # Docker images # | |
| ################# | |
| publish-images: | |
| name: Publish Docker images | |
| runs-on: ubuntu-latest | |
| # prevent running on fork PRs | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| needs.determine-images.outputs.do_publish == 'true' && | |
| (needs.test-ing.result == 'success' || needs.test-ing.result == 'skipped') && | |
| (needs.test-api.result == 'success' || needs.test-api.result == 'skipped') && | |
| (needs.test-cat.result == 'success' || needs.test-cat.result == 'skipped') && | |
| (needs.playwright.result == 'success' || needs.playwright.result == 'skipped') | |
| needs: | |
| - determine-images | |
| - get-image-tag | |
| - build-images | |
| - test-ing # test for ingestion server | |
| - test-api # test for API | |
| - test-cat # test for catalog | |
| - playwright # test for frontend | |
| permissions: | |
| packages: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.determine-images.outputs.publish_matrix) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Docker Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: https://ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Load Docker image `${{ matrix.image }}` | |
| uses: ./.github/actions/load-img | |
| with: | |
| run_id: ${{ github.run_id }} | |
| setup_images: ${{ matrix.image }} | |
| - name: Load and tag image `${{ matrix.image }}` (latest & sha) | |
| run: | | |
| docker tag openverse-${{ matrix.image }} \ | |
| ghcr.io/wordpress/openverse-${{ matrix.image }}:latest | |
| docker tag openverse-${{ matrix.image }} \ | |
| ghcr.io/wordpress/openverse-${{ matrix.image }}:${{ needs.get-image-tag.outputs.image_tag }} | |
| docker push --all-tags ghcr.io/wordpress/openverse-${{ matrix.image }} | |
| ############## | |
| # Deployment # | |
| ############## | |
| # See https://github.com/WordPress/openverse/issues/1033 for why | |
| # we don't use the standard reusable workflow approach for these. | |
| deploy-frontend: | |
| name: Deploy staging frontend | |
| runs-on: ubuntu-latest | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| needs.get-changes.outputs.frontend == 'true' && | |
| needs.playwright.result == 'success' && | |
| needs.publish-images.result == 'success' | |
| needs: | |
| - get-changes | |
| - playwright | |
| - get-image-tag | |
| - publish-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get commit message | |
| id: commit | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python get_commit_message.py | |
| - name: Deploy staging frontend | |
| uses: felixp8/[email protected] | |
| with: | |
| owner: WordPress | |
| repo: openverse-infrastructure | |
| token: ${{ secrets.ACCESS_TOKEN }} | |
| event_type: deploy_staging_nuxt | |
| client_payload: | | |
| { | |
| "actor": "${{ github.actor }}", | |
| "tag": "${{ needs.get-image-tag.outputs.image_tag }}", | |
| "run_name": "${{ steps.commit.outputs.commit_message }}" | |
| } | |
| wait_time: 60 # check every minute | |
| max_time: 1800 # allow up to 30 minutes for a deployment | |
| - name: Trigger staging Playwright smoketests | |
| uses: convictional/[email protected] | |
| with: | |
| owner: WordPress | |
| repo: openverse | |
| github_token: ${{ secrets.ACCESS_TOKEN }} | |
| workflow_file_name: playwright_deployment_smoketest.yml | |
| wait_interval: 60 | |
| # TODO: Set to true once we see that this test is stable, and we can fail the deployment if it fails. | |
| # @see https://github.com/WordPress/openverse/pull/4991 | |
| propagate_failure: false | |
| client_payload: | | |
| { | |
| "service_url": "https://staging.openverse.org/" | |
| } | |
| - name: Trigger staging k6 load test | |
| uses: convictional/[email protected] | |
| with: | |
| owner: WordPress | |
| repo: openverse | |
| github_token: ${{ secrets.ACCESS_TOKEN }} | |
| workflow_file_name: k6.yml | |
| wait_interval: 60 | |
| # TODO: Set to true once we see that this test is stable, and we can fail the deployment if it fails. | |
| # @see https://github.com/WordPress/openverse/pull/4991 | |
| propagate_failure: false | |
| client_payload: | | |
| { | |
| "namespace": "frontend", | |
| "scenario": "all", | |
| "service_url": "https://staging.openverse.org/", | |
| "report": true | |
| } | |
| deploy-api: | |
| name: Deploy staging API | |
| runs-on: ubuntu-latest | |
| if: | | |
| !failure() && !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| needs.get-changes.outputs.api == 'true' && | |
| needs.publish-images.result == 'success' | |
| needs: | |
| - get-changes | |
| - get-image-tag | |
| - publish-images | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get commit message | |
| id: commit | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python get_commit_message.py | |
| - name: Deploy staging API | |
| uses: felixp8/[email protected] | |
| with: | |
| owner: WordPress | |
| repo: openverse-infrastructure | |
| token: ${{ secrets.ACCESS_TOKEN }} | |
| event_type: deploy_staging_api | |
| client_payload: | | |
| { | |
| "actor": "${{ github.actor }}", | |
| "tag": "${{ needs.get-image-tag.outputs.image_tag }}", | |
| "run_name": "${{ steps.commit.outputs.commit_message }}" | |
| } | |
| wait_time: 60 # check every minute | |
| max_time: 1800 # allow up to 30 minutes for a deployment | |
| ################ | |
| # Notification # | |
| ################ | |
| send-report: | |
| name: Send Slack report | |
| runs-on: ubuntu-latest | |
| if: | | |
| !cancelled() && | |
| ( | |
| (github.event_name == 'push' && github.repository == 'WordPress/openverse') || | |
| (github.event_name == 'workflow_dispatch' && inputs.perform_deploy) | |
| ) && | |
| ( | |
| ((github.event_name == 'push' && needs.get-changes.outputs.documentation == 'true') && needs.emit-docs.result != 'success') || | |
| (needs.determine-images.outputs.do_publish == 'true' && needs.publish-images.result != 'success') || | |
| (needs.get-changes.outputs.frontend == 'true' && needs.deploy-frontend.result != 'success') || | |
| (needs.get-changes.outputs.api == 'true' && needs.deploy-api.result != 'success') | |
| ) | |
| needs: # the end products of the CI + CD workflow | |
| - get-changes | |
| - determine-images | |
| - emit-docs | |
| - publish-images | |
| - deploy-frontend | |
| - deploy-api | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get commit message | |
| id: commit | |
| env: | |
| COMMIT_MESSAGE: ${{ github.event.head_commit.message }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| working-directory: automations/python/workflows | |
| run: python get_commit_message.py | |
| - name: Generate report | |
| id: report | |
| env: | |
| COMMIT_MESSAGE: ${{ steps.commit.outputs.commit_message }} | |
| DEPLOY_API_RESULT: ${{ needs.deploy-api.result }} | |
| DEPLOY_FRONTEND_RESULT: ${{ needs.deploy-frontend.result }} | |
| EMIT_DOCS_RESULT: ${{ needs.emit-docs.result }} | |
| GH_SLACK_USERNAME_MAP: ${{ secrets.GH_SLACK_USERNAME_MAP }} | |
| GITHUB_ACTOR: ${{ github.event.head_commit.author.username }} | |
| PUBLISH_IMAGES_RESULT: ${{ needs.publish-images.result }} | |
| PYTHONPATH: ${{ github.workspace }}/automations/python | |
| REPOSITORY: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| SERVER_URL: ${{ github.server_url }} | |
| working-directory: automations/python/workflows | |
| run: python generate_report.py | |
| - name: Send report | |
| uses: slackapi/slack-github-action@v2 | |
| with: | |
| webhook: ${{ secrets.SLACK_OV_ALERTS_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload: ${{ steps.report.outputs.payload }} |