[not for merge] DESC release branch #4423
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parsl | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| merge_group: | |
| jobs: | |
| main-test-suite: | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12"] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@master | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Collect Job Information | |
| id: job-info | |
| run: | | |
| echo "Python Version: ${{ matrix.python-version }}" >> ci_job_info.txt | |
| echo "CI Triggering Event: ${{ github.event_name }}" >> ci_job_info.txt | |
| echo "Triggering Git Ref: ${{ github.ref }}" >> ci_job_info.txt | |
| echo "Triggering Git SHA: ${{ github.sha }}" >> ci_job_info.txt | |
| echo "Workflow Run: ${{ github.run_number }}" >> ci_job_info.txt | |
| echo "Workflow Attempt: ${{ github.run_attempt }}" >> ci_job_info.txt | |
| as_ascii="$(echo "${{ github.ref_name }}" | perl -pe "s/[^A-z0-9-]+/-/g; s/^-+|-+\$//g; s/--+/-/g;")" | |
| echo "as-ascii=$as_ascii" >> $GITHUB_OUTPUT | |
| - name: Non-requirements based install | |
| run: | | |
| # libpython3.5: make workqueue binary installer happy | |
| # mpich: required by radical executor | |
| sudo apt-get update -q | |
| sudo apt-get install -qy libpython3.5 mpich | |
| - name: setup virtual env | |
| run: | | |
| make virtualenv | |
| source .venv/bin/activate | |
| - name: make deps clean_coverage | |
| run: | | |
| source .venv/bin/activate | |
| make deps | |
| make clean_coverage | |
| - name: make test | |
| run: | | |
| source .venv/bin/activate | |
| # temporary; until test-matrixification | |
| export PARSL_TEST_PRESERVE_NUM_RUNS=7 | |
| make test | |
| ln -s pytest-parsl/parsltest-current test_runinfo | |
| - name: Documentation checks | |
| run: | | |
| source .venv/bin/activate | |
| pip install .[docs] | |
| sudo apt-get install -y pandoc | |
| cd docs | |
| test ! -e stubs | |
| # check we can build the docs without warnings | |
| PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages make SPHINXOPTS=-W html | |
| cd .. | |
| # assert that none of the runs in this test have put an ERROR message into a | |
| # database manager log file or monitoring router log file. It would be better if | |
| # the tests themselves failed immediately when there was a monitoring error, but | |
| # in the absence of that, this is a dirty way to check. | |
| bash -c '! grep ERROR pytest-parsl/parsltest-current/runinfo*/*/database_manager.log' | |
| bash -c '! grep ERROR pytest-parsl/parsltest-current/runinfo*/*/monitoring_router.log' | |
| # temporary; until test-matrixification | |
| rm -f pytest-parsl/parsltest-current test_runinfo | |
| - name: Checking parsl-visualize | |
| run: | | |
| source .venv/bin/activate | |
| sudo apt-get install -y graphviz | |
| pip install .[monitoring,visualization] | |
| parsl/tests/test-viz.sh | |
| # config_local_test comes after viz so that the large monitoring.db | |
| # created by `make test` is still around | |
| - name: make config_local_test | |
| run: | | |
| source .venv/bin/activate | |
| make config_local_test | |
| - name: make coverage | |
| run: | | |
| source .venv/bin/activate | |
| make coverage | |
| - name: Archive runinfo logs | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: runinfo-${{ matrix.python-version }}-${{ steps.job-info.outputs.as-ascii }}-${{ github.sha }} | |
| path: | | |
| runinfo/ | |
| pytest-parsl/ | |
| ci_job_info.txt | |
| compression-level: 9 | |
| gen3: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest ] | |
| py: [ "3.11" ] | |
| CC: [ gcc ] | |
| CXX: [ g++ ] | |
| defaults: | |
| run: | |
| # cf. https://github.com/conda-incubator/setup-miniconda#important | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: LSSTDESC/gen3_workflow | |
| - uses: actions/checkout@master | |
| with: | |
| path: ./parsl-repo | |
| - name: Setup conda | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| activate-environment: stack | |
| python-version: "3.11" | |
| condarc-file: etc/.condarc | |
| - name: Install conda deps | |
| run: | | |
| conda info | |
| conda list | |
| conda install -y mamba | |
| mamba install -y --file conda_requirements.txt | |
| conda info | |
| conda list | |
| - name: Install workflow packages | |
| run: | | |
| # pip install -U --no-deps 'parsl[monitoring,workqueue] @ git+https://github.com/parsl/parsl@desc' | |
| pip install -U --no-deps './parsl-repo[monitoring,workqueue]' | |
| pip install typeguard tblib paramiko dill pyzmq globus-sdk sqlalchemy_utils | |
| conda install -c conda-forge ndcctools=7.6.1=py311h689c632_0 --no-deps | |
| - name: Clone gen3_workflow checkout the branch | |
| shell: bash -l {0} | |
| run: | | |
| echo pwd | |
| pwd | |
| echo ls | |
| ls | |
| echo Cloning gen3_workflow repo | |
| git clone https://github.com/LSSTDESC/gen3_workflow | |
| echo declaring gen3_workflow to eups | |
| eups declare gen3_workflow -r ${PWD}/gen3_workflow -t current | |
| cd gen3_workflow | |
| git fetch origin master:TESTING | |
| git checkout TESTING | |
| - name: Run the test pipelines | |
| run: | | |
| setup gen3_workflow | |
| (eups list lsst_distrib) 2>&1 | grep -v "Unknown tag" | |
| (eups list gen3_workflow) 2>&1 | grep -v "Unknown tag" | |
| cd tests | |
| pytest test_query_workflow.py test_bps_restart.py | |
| - name: Archive test logs | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: testinfo-${{ matrix.python-version }} | |
| path: gen3_workflow/ | |