verivox-de.yml

name: Update Verivox Data DE

on:
  # Schedule makes this bot run in regular intervals, without any interventions (GMT timezone)
  schedule:
    - cron: "10 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21 * * *"
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: bots/verivox-de/
    steps:
      - uses: actions/checkout@v4
      - name: Install system dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y libxml2-dev libcurl4-openssl-dev libssl-dev
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Install node packages
        run: |
          npm install -g @nzz/q-cli
          npm install -g mapshaper
          npm audit fix --force
      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10.x"
      - name: Get pip cache dir
        id: pip-cache
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
      - name: Cache python packages
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-${{ hashFiles('./bots/verivox-de/requirements.txt') }}
      - name: Install python packages
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
      - name: Run python scripts
        # execute smard-sources.py after imports-countries.py
        run: |
          python agsi-gas-storage.py
          python entsog-imports.py
          python bnetza-exporte.py
          python ttf-gas-stock.py
          # python epex-spot-ac.py
          python bkarta-sprit.py
          # python bnetza-verbrauch.py
          python verivox-daily-new.py
          python imports-countries.py
          python smard-sources.py
          python eex-futures.py
          python verivox-daily-diff.py
          python gas-dashboard.py
        env:
          DATAWRAPPER_API: ${{ secrets.DATAWRAPPER_API }}
          VERIVOX_SNOWFLAKE_USER: ${{ secrets.VERIVOX_SNOWFLAKE_USER }}
          VERIVOX_SNOWFLAKE_PASS: ${{ secrets.VERIVOX_SNOWFLAKE_PASS }}
          VERIVOX_SNOWFLAKE_ACC: ${{ secrets.VERIVOX_SNOWFLAKE_ACC }}
      - name: Run Q cli
        run: Q update-item
        env:
          Q_PRODUCTION_SERVER: ${{ secrets.Q_PRODUCTION_SERVER }}
          Q_PRODUCTION_ACCESSTOKEN: ${{ secrets.Q_PRODUCTION_ACCESSTOKEN }}
      - uses: stefanzweifel/git-auto-commit-action@v5
        with:
          file_pattern: :!*.json
          commit_message: Update Verivox data
      - name: Log failure to slack
        if: always()
        uses: ravsamhq/notify-slack-action@2.5.0
        with:
          status: ${{ job.status }}
          notify_when: "failure"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_DE_SLACK }}