Skip to content

test | dremio

test | dremio #57

name: test | dremio
on:
pull_request:
branches:
- master
- devel
workflow_dispatch:
schedule:
- cron: '0 2 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
RUNTIME__LOG_LEVEL: ERROR
RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
ACTIVE_DESTINATIONS: "[\"dremio\"]"
ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
jobs:
get_docs_changes:
name: docs changes
uses: ./.github/workflows/get_docs_changes.yml
if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
run_loader:
name: test | dremio tests
needs: get_docs_changes
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
defaults:
run:
shell: bash
runs-on: "ubuntu-latest"
steps:
- name: Check out
uses: actions/checkout@master
- name: Start dremio
run: docker compose -f "tests/load/dremio/docker-compose.yml" up -d
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10.x"
- name: Install Poetry
uses: snok/[email protected]
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
- name: Install dependencies
run: poetry install --no-interaction -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
- run: |
poetry run pytest tests/load --ignore tests/load/sources
if: runner.os != 'Windows'
name: Run tests Linux/MAC
env:
DESTINATION__DREMIO__CREDENTIALS: grpc://dremio:dremio123@localhost:32010/nas
DESTINATION__DREMIO__STAGING_DATA_SOURCE: minio
DESTINATION__MINIO__BUCKET_URL: s3://dlt-ci-test-bucket
DESTINATION__MINIO__CREDENTIALS__AWS_ACCESS_KEY_ID: minioadmin
DESTINATION__MINIO__CREDENTIALS__AWS_SECRET_ACCESS_KEY: minioadmin
DESTINATION__MINIO__CREDENTIALS__ENDPOINT_URL: http://127.0.0.1:9010
- run: |
poetry run pytest tests/load --ignore tests/load/sources
if: runner.os == 'Windows'
name: Run tests Windows
shell: cmd
- name: Stop dremio
if: always()
run: docker compose -f "tests/load/dremio/docker-compose.yml" down -v