Skip to content

Scrapy source using scrapy #596

Scrapy source using scrapy

Scrapy source using scrapy #596

name: test postgres and duckdb
on:
pull_request:
branches:
- master
- devel
workflow_dispatch:
env:
ALL_DESTINATIONS: '["postgres", "duckdb"]'
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
# DESTINATIONS_SECRETS: ${{ secrets.DESTINATIONS_SECRETS }}
# SOURCES_SECRETS: ${{ secrets.SOURCES_SECRETS }}
RUNTIME__LOG_LEVEL: ERROR
jobs:
get_changed_sources:
uses: ./.github/workflows/get_changed_sources.yml
# Tests that require credentials do not run in forks
if: ${{ !github.event.pull_request.head.repo.fork }}
run_loader:
name: test destinations postgres and duckdb
needs: get_changed_sources
if: needs.get_changed_sources.outputs.sources_list != ''
strategy:
max-parallel: 2
fail-fast: false
matrix:
os: ["macos-latest", "windows-latest"]
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
steps:
- name: Check out
uses: actions/checkout@master
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: "3.9.x"
- name: Install Poetry
uses: snok/[email protected]
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v2
with:
# path: ${{ steps.pip-cache.outputs.dir }}
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-destinations
- name: Install dependencies
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --all-extras
- name: create secrets.toml
run: pwd && echo "$DLT_SECRETS_TOML" > sources/.dlt/secrets.toml
# run: pwd && echo "$DESTINATIONS_SECRETS" > sources/.dlt/secrets.toml && echo "$SOURCES_SECRETS" >> sources/.dlt/secrets.toml
# - name: Setup upterm session
# uses: lhotari/action-upterm@v1
- run: |
sources_list="${{ needs.get_changed_sources.outputs.sources_list }}"
test_paths=$(echo "$sources_list" | awk '{for(i=1;i<=NF;i++) printf "tests/%s ", $i}')
poetry run pytest $test_paths
if: runner.os != 'Windows'
name: Run tests on Linux/macOS
- run: |
FOR %%i IN (${{ needs.get_changed_sources.outputs.sources_list }}) DO (
echo Running tests for: %%i
poetry run pytest tests/%%i
)
if: runner.os == 'Windows'
name: Run tests on Windows
shell: cmd