DEV_aepg600m_precipWeighing_update_dag.yml

name: DEV-aepg600m-precipWeighing-update-dag
on:
  push:
    branches:
      - 'master'
    paths:
      - 'pipe/aepg600m/*.yaml'
      - 'pipe/aepg600m/pipe_list_aepg600m.txt'
      - 'pipe/precipWeighing/*.yaml'
      - 'pipe/precipWeighing/pipe_list_precipWeighing.txt'
  workflow_dispatch: {} # Allows trigger of workflow from web interface

jobs:
  # -------------------------------------------------------------
  # Using GitHub's API is not supported for push events
  # -------------------------------------------------------------
  # 
  # ----------------------------------------------------------------------------------------------
  # Using local .git history
  # ----------------------------------------------------------------------------------------------
  # Event `push`: Compare the preceding remote commit -> to the current commit of the main branch 
  # ----------------------------------------------------------------------------------------------

  changed_files:
    runs-on: ubuntu-latest  # windows-latest || macos-latest
    outputs:
      changed_file_list: ${{ steps.changed-files-list.outputs.changed_file_list }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.

      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v41
      # NOTE: `since_last_remote_commit: true` is implied by default and falls back to the previous local commit.

      - name: Make list of changed files
        id: changed-files-list
        env:
          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
        run: |
          file_list=""
          for file in $ALL_CHANGED_FILES; do
            if [ $file_list ]
            then
              file_list=$file_list,$file
            else
              file_list=$file
            fi
            echo "$file was changed"
          done
          echo "Full changed file list: $file_list"
          echo "changed_file_list=$file_list" >> "$GITHUB_OUTPUT"

  update-pipelines:
    needs: changed_files
    runs-on: arc-neon-gke
    #runs-on: ubuntu-latest 
    env:
      PACHD_ADDRESS: grpcs://pachyderm-dev.transitions-nonprod.gcp.neoninternal.org:443
      PACH_TOKEN: ${{ secrets.RepoOwnerPachydermDev }}
      PATHS: 'pipe/aepg600m,pipe/precipWeighing' # Separate multiple with comma (e.g. 'pipe/pqs1,pipe/parWaterSurface'). Order matters.
      TRANSACTION: True
      UPDATE_SCOPE: changed # 'all' or 'changed'. If not specified, all will be updated. 'changed' will update/create any changed/non-existent pipelines.
      CHANGED_FILES: ${{needs.changed_files.outputs.changed_file_list}}
    steps:
      - uses: actions/checkout@v4
      - run: ls -la
      
      - uses: actions/setup-python@v3
        with:
          python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax
          cache: 'pip'
      - run: pip install -r .github/workflow_scripts/update_dag_pipelines/requirements.txt
      - run: python3 -u .github/workflow_scripts/update_dag_pipelines/update_dag_pipelines.py