test_doc_snippets.yml

name: docs | snippets & examples

on:
  pull_request:
    branches:
      - master
      - devel
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}

  # RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
  RUNTIME__LOG_LEVEL: ERROR
  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}

  # Slack hook for chess in production example
  RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }}
  # Path to local qdrant database
  DESTINATION__QDRANT__CREDENTIALS__PATH: zendesk.qdb
  # detect if the workflow is executed in a repo fork
  IS_FORK: ${{ github.event.pull_request.head.repo.fork }}

jobs:

  run_lint:
    name: docs | snippets & examples lint and test
    runs-on: ubuntu-latest
    # Do not run on forks, unless allowed, secrets are used here
    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}

    # Service containers to run with `container-job`
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres
        # Provide the password for postgres
        env:
          POSTGRES_DB: dlt_data
          POSTGRES_USER: loader
          POSTGRES_PASSWORD: loader
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:

      - name: Check out
        uses: actions/checkout@master

      - name: Start weaviate
        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10.x"

      - name: Setup node 20
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: run docs preprocessor
        run: make preprocess-docs

      - name: Install dependencies
        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant -E bigquery -E postgres -E lancedb --with docs,sentry-sdk --without airflow -E s3

      - name: create secrets.toml for examples
        run: pwd && echo "$DLT_SECRETS_TOML" > docs/examples/.dlt/secrets.toml

      - name: create secrets.toml for snippets
        run: pwd && echo "$DLT_SECRETS_TOML" > docs/website/docs/.dlt/secrets.toml

      - name: Run linter and tests on examples
        run: make lint-and-test-examples

      - name: Run linter and tests on snippets
        run: make lint-and-test-snippets