Feature/melscale frequency #1481
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # We only need `push` triggering event internal, otherwise the workflow will run | |
| # twice. But for forks, we need to run workflow on both `push` and `pull_request`. | |
| # So in case of `pull_request` we check that HEAD is not 'Renumics/spotlight'. | |
| on: | |
| - push | |
| - pull_request | |
| # As long as we use local actions, checkout should be made in workflow before | |
| # and not in the local actions. Otherwise, no action file found. | |
| # Checkout with `fetch-depth: 0` is needed by `poetry-dynamic-versioning` to get | |
| # the right package version. | |
| jobs: | |
| # Prepare stage | |
| prepare-python: | |
| name: '🐍 Prepare Python' | |
| if: github.event.pull_request.head.repo.full_name != 'Renumics/spotlight' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| id: setup-poetry | |
| uses: ./.github/actions/setup-poetry | |
| outputs: | |
| version: ${{ steps.setup-poetry.outputs.package-version }} | |
| pip-cache-dir: ${{ steps.setup-poetry.outputs.pip-cache-dir }} | |
| prepare-node: | |
| name: '⬢ Prepare Node.js' | |
| if: github.event.pull_request.head.repo.full_name != 'Renumics/spotlight' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '⬢ Set up pnpm environment' | |
| uses: ./.github/actions/setup-pnpm | |
| # Check stage | |
| check-pr: | |
| name: '🔍 Check if pull request' | |
| if: github.event.pull_request.head.repo.full_name != 'Renumics/spotlight' | |
| permissions: | |
| pull-requests: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: '🔍 Check if pull request' | |
| id: check-pr | |
| uses: 8BitJonny/[email protected] | |
| with: | |
| filterOutClosed: true | |
| outputs: | |
| is-pr: ${{ steps.check-pr.outputs.pr_found }} | |
| number: ${{ steps.check-pr.outputs.number }} | |
| check: | |
| name: '🔍 Check' | |
| needs: | |
| - prepare-python | |
| - prepare-node | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '⬢ Set up pnpm environment' | |
| id: setup | |
| uses: ./.github/actions/setup-pnpm | |
| - name: Run shellcheck | |
| if: success() || steps.setup.outcome == 'success' | |
| uses: ludeeus/[email protected] | |
| with: | |
| ignore_paths: .venv node_modules | |
| - name: Audit dependencies | |
| if: success() || steps.setup.outcome == 'success' | |
| run: make audit | |
| - name: Check code formatting | |
| if: success() || steps.setup.outcome == 'success' | |
| run: make check-format | |
| - name: Check code types | |
| if: success() || steps.setup.outcome == 'success' | |
| run: make typecheck | |
| - name: Lint code | |
| if: success() || steps.setup.outcome == 'success' | |
| run: make lint | |
| # Build stage | |
| build-test-matrix: | |
| name: '🧱 Build test matrix' | |
| needs: | |
| - check-pr | |
| runs-on: ubuntu-latest | |
| # `MATRIX` environment variable is set by the local `generate-matrix` | |
| # action, so use it as check. | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: '🧱 Build test matrix for pull request' | |
| if: needs.check-pr.outputs.is-pr == 'true' | |
| uses: druzsan/setup-matrix@v2 | |
| with: | |
| matrix: | | |
| os: [ubuntu-latest] | |
| python-version: [3.8, 3.9, 3.10, 3.11] | |
| - name: '🧱 Build test matrix for release' | |
| if: env.MATRIX == '' && startsWith(github.ref, 'refs/tags/v') | |
| uses: druzsan/setup-matrix@v2 | |
| with: | |
| matrix: | | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: [3.8, 3.9, 3.10, 3.11] | |
| - name: '🧱 Build test matrix for main branch' | |
| if: env.MATRIX == '' && github.ref == 'refs/heads/main' | |
| uses: druzsan/setup-matrix@v2 | |
| with: | |
| matrix: | | |
| os: [ubuntu-latest] | |
| python-version: [3.8, 3.9, 3.10, 3.11] | |
| include: | |
| - os: windows-latest | |
| python-version: 3.8 | |
| - os: macos-latest | |
| python-version: 3.8 | |
| - name: '🧱 Build test matrix for development branch' | |
| if: env.MATRIX == '' | |
| uses: druzsan/setup-matrix@v2 | |
| with: | |
| matrix: | | |
| os: [ubuntu-latest] | |
| python-version: [3.8] | |
| - name: Print matrix | |
| run: echo "$MATRIX" | yq -P '{"matrix":.}' | |
| - name: Set output | |
| id: set-matrix | |
| run: echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| build-docs: | |
| name: '📝 Build API docs' | |
| needs: | |
| - prepare-python | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '📝 Build API docs' | |
| run: make docs | |
| - name: '📥 Store API docs' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docs-${{ needs.prepare-python.outputs.version }} | |
| path: build/docs/api/ | |
| if-no-files-found: error | |
| build-datasets: | |
| name: '🗃 Build datasets' | |
| needs: | |
| - prepare-python | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '🗃 Build datasets' | |
| if: github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/v') | |
| run: make datasets | |
| - name: '🗃🗃 Build all datasets' | |
| if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') | |
| run: make all-datasets | |
| - name: '📥 Store datasets' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: datasets-${{ needs.prepare-python.outputs.version }} | |
| path: build/datasets/ | |
| if-no-files-found: error | |
| build-spotlight: | |
| name: '🧱 Build Spotlight' | |
| needs: | |
| - prepare-python | |
| - prepare-node | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry' | |
| uses: ./.github/actions/setup-poetry | |
| with: | |
| install-dependencies: false | |
| - name: '⬢ Set up pnpm environment' | |
| uses: ./.github/actions/setup-pnpm | |
| - name: '🧱 Build frontend' | |
| run: make build-frontend | |
| - name: '📥 Store frontend' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: .frontend-${{ needs.prepare-python.outputs.version }} | |
| path: build/frontend/ | |
| if-no-files-found: error | |
| - name: Build Spotlight | |
| run: make build-wheel | |
| - name: '📥 Store Spotlight' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: renumics-spotlight-${{ needs.prepare-python.outputs.version }} | |
| path: build/dist/renumics_spotlight*.whl | |
| if-no-files-found: error | |
| # Test stage | |
| check-wheel-contents: | |
| name: '🔍 Check wheel contents' | |
| needs: | |
| - prepare-python | |
| - build-spotlight | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '📤 Restore Spotlight Wheel' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: renumics-spotlight-${{ needs.prepare-python.outputs.version }} | |
| path: build/dist | |
| - name: Check wheel contents | |
| run: make check-wheel | |
| unit-test: | |
| name: '🧪 Unit Test' | |
| needs: | |
| - prepare-python | |
| - prepare-node | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '⬢ Set up pnpm environment' | |
| uses: ./.github/actions/setup-pnpm | |
| - name: Execute unit tests | |
| run: make unit-test | |
| doc-test: | |
| name: '🧪 Doc Test' | |
| needs: | |
| - prepare-python | |
| - prepare-node | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '⬢ Set up pnpm environment' | |
| uses: ./.github/actions/setup-pnpm | |
| - name: Execute doc tests | |
| run: make doc-test | |
| ui-test: | |
| name: '🧪 UI Test' | |
| needs: | |
| - prepare-python | |
| - build-datasets | |
| - build-spotlight | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| browser: | |
| - chrome | |
| - firefox | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry environment' | |
| uses: ./.github/actions/setup-poetry | |
| - name: '📤 Restore datasets' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: datasets-${{ needs.prepare-python.outputs.version }} | |
| path: build/datasets/ | |
| - name: '📤 Restore frontend' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: .frontend-${{ needs.prepare-python.outputs.version }} | |
| path: build/frontend/ | |
| - name: '🎨 Set up Chrome driver' | |
| if: matrix.browser == 'chrome' | |
| id: setup-chromedriver | |
| uses: nanasess/setup-chromedriver@v2 | |
| - name: '🦊 Set up Gecko driver' | |
| id: setup-geckodriver | |
| if: matrix.browser == 'firefox' | |
| run: | | |
| curl -s https://api.github.com/repos/mozilla/geckodriver/releases/latest \ | |
| | jq -r '.assets | map(select(.name | test("^geckodriver-v.*-linux64.tar.gz$")))[0].browser_download_url' \ | |
| | xargs wget -O ${{ runner.temp }}/geckodriver.tar.gz | |
| tar -xzf ${{ runner.temp }}/geckodriver.tar.gz -C /usr/local/bin/ | |
| - name: Download screenshots of the last commit on main | |
| continue-on-error: true | |
| env: | |
| AZURE_FOLDER_URL: https://spotlightpublic.blob.core.windows.net/github-public/${{ github.repository }} | |
| run: | | |
| MAIN_COMMITS="$(curl -L \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{ github.repository }}/commits)" | |
| INDEX=$([ "$GITHUB_REF" = "refs/heads/main" ] && echo 1 || echo 0) | |
| COMMIT_SHA=$(echo "$MAIN_COMMITS" | jq -r ".[$INDEX].sha") | |
| mkdir -p build/ui_tests/old-screenshots | |
| cd build/ui_tests/old-screenshots | |
| curl -fLO ${AZURE_FOLDER_URL}/${COMMIT_SHA}/screenshots/gui-${{ matrix.browser }}.png | |
| - name: '🍱 Test UI on ${{ matrix.browser }}' | |
| run: make ui-test-${{ matrix.browser }} | |
| - name: Prepare GUI screenshots | |
| if: | | |
| success() | |
| || steps.setup-chromedriver.outcome == 'success' | |
| || steps.setup-geckodriver.outcome == 'success' | |
| run: | | |
| mkdir -p build/upload/screenshots | |
| cp build/ui_tests/screenshots/gui*.png build/upload/screenshots | |
| if [ "$GITHUB_REF" = "refs/heads/main" ] | |
| then | |
| touch build/upload/screenshots/.is-main | |
| fi | |
| - name: '📸 Upload GUI screenshots' | |
| if: | | |
| ( | |
| success() | |
| || steps.setup-chromedriver.outcome == 'success' | |
| || steps.setup-geckodriver.outcome == 'success' | |
| ) && ( | |
| ( | |
| github.event_name == 'push' | |
| && github.repository == 'Renumics/spotlight' | |
| ) || ( | |
| github.event_name == 'pull_request' | |
| && github.event.pull_request.head.repo.full_name == 'Renumics/spotlight' | |
| ) | |
| ) | |
| uses: LanceMcCarthy/Action-AzureBlobUpload@v3 | |
| with: | |
| connection_string: ${{ secrets.AZURE_CONNECTION_STRING }} | |
| container_name: github-public | |
| source_folder: build/upload/screenshots | |
| destination_folder: ${{ github.repository }}/${{ github.sha }}/screenshots | |
| delete_if_exists: true | |
| - name: '📥 Store UI test on ${{ matrix.browser }} results' | |
| if: | | |
| success() | |
| || steps.setup-chromedriver.outcome == 'success' | |
| || steps.setup-geckodriver.outcome == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: .ui-test-${{ matrix.browser }}-${{ needs.prepare-python.outputs.version }} | |
| path: build/ui_tests | |
| if-no-files-found: error | |
| integration-test: | |
| name: '🧪 Integration Test' | |
| needs: | |
| - prepare-python | |
| - build-test-matrix | |
| - build-datasets | |
| - build-spotlight | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.build-test-matrix.outputs.matrix)}} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '🐍 Set up Poetry' | |
| id: setup-poetry | |
| uses: ./.github/actions/setup-poetry | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| install-dependencies: false | |
| - name: '♻️ Cache pip cache folder' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.setup-poetry.outputs.pip-cache-dir }} | |
| key: pip-cache-${{ runner.os }}-python-${{ matrix.python-version }} | |
| - name: '📤 Restore Spotlight' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: renumics-spotlight-${{ needs.prepare-python.outputs.version }} | |
| path: build/dist | |
| - name: '📤 Restore datasets' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: datasets-${{ needs.prepare-python.outputs.version }} | |
| path: build/datasets/ | |
| - name: Upgrade system packages | |
| run: python -m pip install --upgrade pip setuptools wheel | |
| - name: Install Spotlight | |
| id: setup | |
| run: python -m pip install --find-links build/dist/ renumics-spotlight[all]==${{ needs.prepare-python.outputs.version }} | |
| - name: Test Spotlight start (Windows) | |
| if: runner.os == 'Windows' && (success() || steps.setup.outcome == 'success') | |
| run: ./scripts/Test-SpotlightStart.ps1 | |
| - name: Test Spotlight start (Ubuntu, MacOS) | |
| if: runner.os != 'Windows' && (success() || steps.setup.outcome == 'success') | |
| run: | | |
| function teardown { | |
| while kill -INT %% 2>/dev/null; do sleep 0; done # kill all child processes | |
| } | |
| trap teardown EXIT | |
| PORT="5005" | |
| spotlight --host 127.0.0.1 --port $PORT --no-browser data/tables/tallymarks-small.h5 & | |
| URL="http://127.0.0.1:${PORT}" | |
| wget -t20 -w0.5 --retry-connrefused --delete-after $URL | |
| sleep 0.5 | |
| GENERATION_ID=$(wget -qO- "${URL}/api/table/" | jq ".generation_id") | |
| wget --delete-after "${URL}/api/table/number/42?generation_id=${GENERATION_ID}" | |
| - name: Install pytest | |
| id: setup-tests | |
| run: python -m pip install pytest | |
| - name: Execute integration tests | |
| if: success() || steps.setup-tests.outcome == 'success' | |
| run: pytest --durations=5 tests/integration | |
| # Release stage | |
| release: | |
| name: '🚀 Release Spotlight' | |
| if: startsWith(github.ref, 'refs/tags/v') && github.repository == 'Renumics/spotlight' | |
| needs: | |
| - prepare-python | |
| - prepare-node | |
| - check-pr | |
| - check | |
| - build-test-matrix | |
| - build-docs | |
| - build-datasets | |
| - build-spotlight | |
| - check-wheel-contents | |
| - unit-test | |
| - integration-test | |
| - ui-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: '📤 Restore core Spotlight' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: renumics-spotlight-${{ needs.prepare-python.outputs.version }} | |
| path: dist | |
| - name: '🚀 Publish Spotlight to PyPI' | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.PYPI_API_TOKEN }} |