Update tutorial-eval.md #12866
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # We use self-hosted runners for performance: we can rely on docker caching | |
| # on the runner machines for 0s startup times for our tests. | |
| # There is an issue with self-hosted runners where if you use a mix of | |
| # host and container jobs, the host jobs will fail to start | |
| # (https://github.com/actions/checkout/issues/273). | |
| # So we create a separate "builder" tagged runner to do the build job on | |
| # the host, and then all the tests run on "runner" tagged runners. | |
| name: test | |
| on: | |
| push: | |
| jobs: | |
| build-container: | |
| name: Build test container | |
| timeout-minutes: 30 | |
| runs-on: [self-hosted, builder] | |
| # runs-on: ubuntu-latest | |
| env: | |
| REGISTRY: us-east4-docker.pkg.dev/weave-support-367421/weave-images | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: us-east4-docker.pkg.dev | |
| username: _json_key | |
| password: ${{ secrets.gcp_sa_key }} | |
| # this script is hardcoded to build for linux/amd64 | |
| - name: Prune docker cache | |
| run: docker system prune -f | |
| - name: Build unit test image | |
| run: python3 docker_build.py build_deps weave-test builder . Dockerfile.test | |
| - name: Build integration test image | |
| run: python3 docker_build.py build weave-integration-test . Dockerfile.test | |
| lint: | |
| name: Python lint | |
| timeout-minutes: 7 | |
| needs: | |
| - build-container | |
| # runs-on: [self-hosted, gke-runner] | |
| runs-on: ubuntu-latest | |
| container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test:${{ github.sha }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - run: source /root/venv/bin/activate && pre-commit run --hook-stage=pre-push --all-files | |
| - name: Show changes if pre-commit fails | |
| if: failure() | |
| run: | | |
| echo "Files were modified, printing names of changed files" | |
| git diff --name-only | |
| echo "Failing this step to expand logs in the UI" | |
| exit 1 | |
| weavejs-lint-compile: | |
| name: WeaveJS Lint and Compile | |
| runs-on: ubuntu-latest | |
| # runs-on: [self-hosted, gke-runner] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-node@v1 | |
| with: | |
| node-version: "18.x" | |
| - run: | | |
| set -e | |
| cd weave-js | |
| yarn install --frozen-lockfile | |
| yarn generate | |
| yarn eslint --max-warnings=0 | |
| yarn tslint | |
| yarn prettier | |
| yarn run tsc | |
| test: | |
| name: Python unit tests | |
| timeout-minutes: 15 # do not raise! running longer than this indicates an issue with the tests. fix there. | |
| needs: | |
| - build-container | |
| # runs-on: [self-hosted, gke-runner] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job_num: [0, 1] | |
| # runs-on: ubuntu-latest | |
| container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-integration-test:${{ github.sha }} | |
| services: | |
| wandbservice: | |
| image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master | |
| credentials: | |
| username: _json_key | |
| password: ${{ secrets.gcp_wb_sa_key }} | |
| env: | |
| CI: 1 | |
| WANDB_ENABLE_TEST_CONTAINER: true | |
| ports: | |
| - "8080:8080" | |
| - "8083:8083" | |
| - "9015:9015" | |
| options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s | |
| weave_clickhouse: | |
| image: clickhouse/clickhouse-server | |
| ports: | |
| - "8123:8123" | |
| options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s | |
| steps: | |
| # - uses: datadog/[email protected] | |
| # with: | |
| # api_key: ${{ secrets.DD_API_KEY }} | |
| - uses: actions/checkout@v2 | |
| - name: Verify wandb server is running | |
| run: curl -s http://wandbservice:8080/healthz | |
| - name: Run Python Unit Tests (Clickhouse Client Only) | |
| env: | |
| DD_SERVICE: weave-python | |
| DD_ENV: ci | |
| WEAVE_SENTRY_ENV: ci | |
| # This runner specifically runs the tests that use the `client` fixture (those that support clickhouse client tests) | |
| # However, we skip tests marked with `skip_clickhouse_client`. These should be considered TODOs and an exception | |
| run: CI=1 WB_SERVER_HOST=http://wandbservice WF_CLICKHOUSE_HOST=weave_clickhouse WEAVE_SERVER_DISABLE_ECOSYSTEM=1 source /root/venv/bin/activate && cd weave && pytest -m "weave_client and not skip_clickhouse_client" --weave-server=clickhouse --job-num=${{ matrix.job_num }} --timeout=90 ./flow ./integrations ./tests ./legacy/ops_arrow ./legacy/ecosystem ./trace_server ./trace --ddtrace --durations=5 | |
| - name: Run Python Unit Tests | |
| env: | |
| DD_SERVICE: weave-python | |
| DD_ENV: ci | |
| WEAVE_SENTRY_ENV: ci | |
| run: CI=1 WB_SERVER_HOST=http://wandbservice WF_CLICKHOUSE_HOST=weave_clickhouse WEAVE_SERVER_DISABLE_ECOSYSTEM=1 source /root/venv/bin/activate && pip install 'dspy-ai>=0.1.5' && pip install pydantic -U && cd weave && pytest --job-num=${{ matrix.job_num }} --timeout=90 ./flow ./integrations ./tests ./legacy/ops_arrow ./legacy/ecosystem ./trace_server ./trace --ddtrace --durations=5 | |
| # nbmake: | |
| # name: Run notebooks with nbmake | |
| # runs-on: self-hosted | |
| # container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test:latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Run notebooks | |
| # run: source /root/venv/bin/activate && export PYTHONPATH=$(pwd) && pytest -n=4 --nbmake --overwrite examples | |
| # - name: Upload executed notebooks | |
| # uses: actions/upload-artifact@v3 | |
| # if: always() | |
| # with: | |
| # name: notebooks | |
| # path: examples | |
| cypress-run: | |
| name: Notebook and UI tests | |
| timeout-minutes: 25 # 15 minute timeout routinely trips on rerun | |
| needs: | |
| - build-container | |
| # - lint | |
| # - test | |
| if: always() | |
| # runs-on: [self-hosted, gke-runner] | |
| runs-on: ubuntu-latest | |
| container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-integration-test:${{ github.sha }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| containers: [1, 2, 3, 4, 5, 6] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup W&B API key | |
| run: echo "WANDB_API_KEY=${{ secrets.WANDB_API_KEY }}" >> $GITHUB_ENV | |
| - name: Setup Replicate API token | |
| run: echo "REPLICATE_API_TOKEN=${{ secrets.REPLICATE_API_TOKEN }}" > $GITHUB_ENV | |
| - name: Copy node_modules from container to checkout | |
| run: cp -R /root/integration_test/node_modules ./integration_test/ | |
| - name: Activate venv | |
| run: source /root/venv/bin/activate && echo "PATH=$PATH" >> $GITHUB_ENV | |
| - name: Make Log Dir | |
| run: mkdir -p /tmp/weave/log | |
| - name: Copy over built assets | |
| run: cp -r /root/weave-js-build/* ./weave/frontend | |
| - name: Start weave server | |
| # github actions does something funky with the std file descriptors, they end up | |
| # being closed. tqdm (for example) raises an exception when the descriptor it | |
| # wants to write to is closed. | |
| run: nohup ./weave_server_test.sh < /dev/null &> /tmp/weave/log/stdout.log & | |
| shell: bash | |
| - name: Cypress run | |
| # Use the following to run just a single test | |
| # run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium --spec "cypress/e2e/notebooks/Ops that return images.cy.ts" | |
| # run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium | |
| run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium | |
| # uses: cypress-io/github-action@v4 | |
| # with: | |
| # working-directory: ./integration_testG | |
| # record: true | |
| # parallel: true | |
| env: | |
| REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SPLIT: ${{ strategy.job-total }} | |
| SPLIT_INDEX: ${{ strategy.job-index }} | |
| # - uses: actions/upload-artifact@v3 | |
| - name: Upload logs | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: weave-server-logs | |
| path: /tmp/weave/log | |
| retention-days: 3 |