ci.yaml

name: Parsl

on:
  pull_request:
    types:
      - opened
      - synchronize

jobs:
  main-test-suite:
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
    runs-on: ubuntu-20.04
    timeout-minutes: 60

    steps:
    - uses: actions/checkout@master

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Collect Job Information
      id: job-info
      run: |
        echo "Python Version: ${{ matrix.python-version }}" >> ci_job_info.txt
        echo "CI Triggering Event: ${{ github.event_name }}" >> ci_job_info.txt
        echo "Triggering Git Ref: ${{ github.ref }}" >> ci_job_info.txt
        echo "Triggering Git SHA: ${{ github.sha }}" >> ci_job_info.txt
        echo "Workflow Run: ${{ github.run_number }}" >> ci_job_info.txt
        echo "Workflow Attempt: ${{ github.run_attempt }}" >> ci_job_info.txt
        as_ascii="$(echo "${{ github.ref_name }}" | perl -pe "s/[^A-z0-9-]+/-/g; s/^-+|-+\$//g; s/--+/-/g;")"
        echo "as-ascii=$as_ascii" >> $GITHUB_OUTPUT

    - name: Non-requirements based install
      run: |
        # libpython3.5: make workqueue binary installer happy
        # mpich: required by radical executor
        sudo apt-get update -q
        sudo apt-get install -qy libpython3.5 mpich

    - name: setup virtual env
      run: |
        make virtualenv
        source .venv/bin/activate

    - name: make deps clean_coverage
      run: |
        source .venv/bin/activate
        make deps
        make clean_coverage

    - name: make test
      run: |
        source .venv/bin/activate

        # temporary; until test-matrixification
        export PARSL_TEST_PRESERVE_NUM_RUNS=7

        make test
        ln -s .pytest/parsltest-current test_runinfo

    - name: Documentation checks
      run: |
        source .venv/bin/activate
        pip install .[docs]
        sudo apt-get install -y pandoc
        cd docs

        test ! -e stubs

        # check we can build the docs without warnings
        PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages make SPHINXOPTS=-W html

        cd ..

        # assert that none of the runs in this test have put an ERROR message into a
        # database manager log file or monitoring router log file. It would be better if
        # the tests themselves failed immediately when there was a monitoring error, but
        # in the absence of that, this is a dirty way to check.
        bash -c '! grep ERROR .pytest/parsltest-current/runinfo*/*/database_manager.log'
        bash -c '! grep ERROR .pytest/parsltest-current/runinfo*/*/monitoring_router.log'

        # temporary; until test-matrixification
        rm -f .pytest/parsltest-current test_runinfo

    - name: Checking parsl-visualize
      run: |
        source .venv/bin/activate
        sudo apt-get install -y graphviz
        pip install .[monitoring,visualization]
        parsl/tests/test-viz.sh

    # config_local_test comes after viz so that the large monitoring.db
    # created by `make test` is still around
    - name: make config_local_test
      run: |
        source .venv/bin/activate
        make config_local_test

    - name: make coverage
      run: |
        source .venv/bin/activate
        make coverage

    - name: Archive runinfo logs
      if: ${{ always() }}
      uses: actions/upload-artifact@v4
      with:
        name: runinfo-${{ matrix.python-version }}-${{ steps.job-info.outputs.as-ascii }}-${{ github.sha }}
        path: |
          runinfo/
          .pytest/
          ci_job_info.txt
        compression-level: 9


  gen3:
        runs-on: ${{ matrix.os }}

        strategy:
            matrix:
                os: [ ubuntu-latest ]
                py: [ "3.11" ]
                CC: [ gcc ]
                CXX: [ g++ ]

        defaults:
            run:
                # cf. https://github.com/conda-incubator/setup-miniconda#important
                shell: bash -l {0}

        steps:
            - uses: actions/checkout@v2
              with:
                  repository: LSSTDESC/gen3_workflow

            - uses: actions/checkout@master
              with:
                  path: ./parsl-repo

            - name: Setup conda
              uses: conda-incubator/setup-miniconda@v2
              with:
                  activate-environment: stack
                  python-version: "3.11"
                  condarc-file: etc/.condarc

            - name: Install conda deps
              run: |
                conda info
                conda list
                conda install -y mamba
                mamba install -y --file conda_requirements.txt
                conda info
                conda list

            
            - name: Install workflow packages
              run: |
                # pip install -U --no-deps 'parsl[monitoring,workqueue] @ git+https://github.com/parsl/parsl@desc'
                pip install -U --no-deps './parsl-repo[monitoring,workqueue]'
                pip install typeguard tblib paramiko dill pyzmq globus-sdk sqlalchemy_utils
                conda install -c conda-forge ndcctools=7.6.1=py311h689c632_0 --no-deps

            - name: Clone gen3_workflow checkout the branch
              shell: bash -l {0}
              run: |
                echo pwd
                pwd 

                echo ls
                ls

                echo Cloning gen3_workflow repo
                git clone https://github.com/LSSTDESC/gen3_workflow

                echo declaring gen3_workflow to eups
                eups declare gen3_workflow -r ${PWD}/gen3_workflow -t current

                cd gen3_workflow
                git fetch origin master:TESTING
                git checkout TESTING

            - name: Run the test pipelines
              run: |
                setup gen3_workflow
                (eups list lsst_distrib) 2>&1 | grep -v "Unknown tag"
                (eups list gen3_workflow) 2>&1 | grep -v "Unknown tag"
                cd tests
                pytest test_query_workflow.py test_bps_restart.py


            - name: Archive test logs
              if: ${{ always() }}
              uses: actions/upload-artifact@v2
              with:
                name: testinfo-${{ matrix.python-version }}
                path: gen3_workflow/