From d4c65f3c31757d2d565a188e509ad191d296f056 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Tue, 19 Nov 2024 10:59:17 -0500 Subject: [PATCH 1/4] Add romanisim to downstream --- .github/workflows/downstream.yml | 25 +++++++++++++++ .github/workflows/get_webbpsf.yml | 31 ++++++++++++++++++ .github/workflows/webbpsf_data.yml | 51 ++++++++++++++++++++++++++++++ tox.ini | 7 ++++ 4 files changed, 114 insertions(+) create mode 100644 .github/workflows/get_webbpsf.yml create mode 100644 .github/workflows/webbpsf_data.yml diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index f15f1ec1..cda547b8 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -39,6 +39,7 @@ jobs: crds_contexts: uses: ./.github/workflows/contexts.yml + jwst: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 needs: [ environment, crds_contexts ] @@ -67,6 +68,30 @@ jobs: envs: | - linux: py311-test-romancal-cov-xdist + get_romanism_data: + uses: ./.github/workflows/webbpsf_data.yml + + romanisim_data_cache: + needs: [ get_romanism_data ] + uses: ./.github/workflows/get_webbpsf.yml + + romanisim: + needs: [ romanisim_data_cache ] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 + with: + libraries: | + brew: + - eigen + - fftw + setenv: | + WEBBPSF_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}webbpsf-data/ + GALSIM_CAT_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits + FFTW_DIR: /opt/homebrew/opt/fftw/lib/ + cache-path: ${{ needs.romanisim_data_cache.outputs.cache_path }} + cache-key: ${{ needs.romanisim_data_cache.outputs.cache_key }} + envs: | + - linux: py311-test-romanisim-cov-xdist + astropy: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 if: (github.repository == 'spacetelescope/gwcs' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Downstream CI'))) diff --git a/.github/workflows/get_webbpsf.yml b/.github/workflows/get_webbpsf.yml new file mode 100644 index 00000000..0b7d7c7c --- /dev/null +++ b/.github/workflows/get_webbpsf.yml @@ -0,0 +1,31 @@ +name: retrieve latest data cache key + +on: + workflow_call: + outputs: + cache_key: + value: ${{ jobs.retrieve_cache.outputs.cache_key }} + cache_path: + value: ${{ jobs.retrieve_cache.outputs.cache_path }} + +jobs: + retrieve_cache: + name: retrieve latest data cache key + runs-on: ubuntu-latest + steps: + - name: retrieve latest data cache key + id: latest_cache_key + run: | + # use actions/gh-actions-cache to allow filtering by key + gh extension install actions/gh-actions-cache + CACHE_KEY=$(gh actions-cache list -R ${{ github.repository }} --key data- --sort created-at | cut -f 1 | head -n 1) + if [ "$CACHE_KEY" == '' ]; then exit 1; fi + echo cache_key=$CACHE_KEY >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + - run: echo cache_key=${{ steps.latest_cache_key.outputs.cache_key }} + - id: cache_path + run: echo cache_path=/tmp/data/ >> $GITHUB_OUTPUT + outputs: + cache_key: ${{ steps.latest_cache_key.outputs.cache_key }} + cache_path: ${{ steps.cache_path.outputs.cache_path }} diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml new file mode 100644 index 00000000..9d45e9d5 --- /dev/null +++ b/.github/workflows/webbpsf_data.yml @@ -0,0 +1,51 @@ +name: download and cache data + +on: + workflow_call: + schedule: + - cron: "42 4 * * 3" + workflow_dispatch: + inputs: + webbpsf_minimal: + description: minimal WebbPSF dataset + type: boolean + required: false + default: true + +jobs: + download_webbpsf_data: + uses: spacetelescope/webbpsf/.github/workflows/download_data.yml@beda656c80a0254e6f80649d9c9c49235634522f # v1.4.0 + with: + minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} + combine_data_cache: + needs: [ download_webbpsf_data ] + runs-on: ubuntu-latest + env: + GALSIM_DATA_URL: https://github.com/GalSim-developers/GalSim/raw/releases/2.4/examples/data/ + steps: + - name: download GalSim data + run: | + mkdir galsim_data + wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example.fits -O galsim_data/real_galaxy_catalog_23.5_example.fits + wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_selection.fits -O galsim_data/real_galaxy_catalog_23.5_example_selection.fits + wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_fits.fits -O galsim_data/real_galaxy_catalog_23.5_example_fits.fits + - id: galsim_data + run: echo "hash=${{ hashFiles( 'galsim_data/' ) }}" >> $GITHUB_OUTPUT + - run: echo GALSIM_PATH=/tmp/data/galsim_data/ >> $GITHUB_ENV + - run: | + mkdir -p ${{ env.GALSIM_PATH }} + mv ./galsim_data/* ${{ env.GALSIM_PATH }} + - run: echo GALSIM_CAT_PATH=${{ env.GALSIM_PATH }}real_galaxy_catalog_23.5_example.fits >> $GITHUB_ENV + - name: retrieve cached WebbPSF data + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: ${{ needs.download_webbpsf_data.outputs.cache_path }} + key: ${{ needs.download_webbpsf_data.outputs.cache_key }} + - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ /tmp/data/ + - run: echo WEBBPSF_PATH=/tmp/data/webbpsf-data/ >> $GITHUB_ENV + # save a new cache to the same generalized data directory, combined with extra data + - name: save a single combined data cache + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: /tmp/data/ + key: data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }} diff --git a/tox.ini b/tox.ini index f5e2ae74..52500f21 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ description = run tests jwst: of JWST pipeline romancal: of Romancal pipeline + romanisim: of Romanisim image simulation dev: with the latest developer version of key dependencies pyargs: with --pyargs on installed package warnings: treating warnings as errors @@ -57,6 +58,10 @@ passenv = CI CODECOV_* DISPLAY + WEBBPSF_PATH + GALSIM_CAT_PATH + FFTW_DIR + LIBRARY_PATH set_env = dev: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple @@ -70,6 +75,7 @@ deps = cov: pytest-cov jwst: jwst[test] @ git+https://github.com/spacetelescope/jwst.git romancal: romancal[test] @ git+https://github.com/spacetelescope/romancal.git + romanisim: romanisim[test] @ git+https://github.com/spacetelescope/romanisim.git numpy123: numpy==1.23.* numpy125: numpy==1.25.* pass_env = @@ -84,6 +90,7 @@ commands = pyargs: {toxinidir}/docs --pyargs gwcs \ jwst: --pyargs jwst --ignore-glob=timeconversion --ignore-glob=associations --ignore-glob=scripts --show-capture=no \ romancal: --pyargs romancal \ + romanisim: --pyargs romanisim \ cov: --cov=. --cov-config=pyproject.toml --cov-report=term-missing --cov-report=xml \ {posargs} From 98808a93df9220a700d9059ddf9c5a07226f2e06 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Wed, 20 Nov 2024 09:36:07 -0500 Subject: [PATCH 2/4] call romanisim workflows directly --- .github/workflows/ci.yml | 5 +-- .github/workflows/downstream.yml | 33 ++++++++---------- .github/workflows/get_webbpsf.yml | 31 ----------------- .github/workflows/romanisim_data.yml | 18 ++++++++++ .github/workflows/webbpsf_data.yml | 51 ---------------------------- 5 files changed, 33 insertions(+), 105 deletions(-) delete mode 100644 .github/workflows/get_webbpsf.yml create mode 100644 .github/workflows/romanisim_data.yml delete mode 100644 .github/workflows/webbpsf_data.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fde2ce6e..806b19b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: 'CI' +name: test on: push: @@ -17,9 +17,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - DATA_PATH: /tmp/data - jobs: check: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index cda547b8..4d8b4d1f 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -1,4 +1,4 @@ -name: Downstream +name: test downstream packages on: workflow_dispatch: @@ -16,9 +16,6 @@ on: tags: - '*' -env: - DATA_PATH: /tmp/data - # Only cancel in-progress jobs or runs for the current workflow # This cancels the already triggered workflows for a specific PR without canceling # other instances of this workflow (other PRs, scheduled triggers, etc) when something @@ -32,13 +29,12 @@ jobs: runs-on: ubuntu-latest steps: - id: data_path - run: echo "path=${{ env.DATA_PATH }}" >> $GITHUB_OUTPUT + run: echo "path=${{ runner.temp }}/data" >> $GITHUB_OUTPUT outputs: data_path: ${{ steps.data_path.outputs.path }} crds_contexts: - uses: ./.github/workflows/contexts.yml - + uses: spacetelescope/crds/.github/workflows/contexts.yml@master jwst: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 @@ -68,15 +64,14 @@ jobs: envs: | - linux: py311-test-romancal-cov-xdist - get_romanism_data: - uses: ./.github/workflows/webbpsf_data.yml - - romanisim_data_cache: - needs: [ get_romanism_data ] - uses: ./.github/workflows/get_webbpsf.yml + romanisim_data: + needs: [ environment ] + uses: spacetelescope/romanisim/.github/workflows/retrieve_cache.yml@develop + with: + cache_path: ${{ needs.environment.outputs.data_path }} romanisim: - needs: [ romanisim_data_cache ] + needs: [ romanisim_data ] uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 with: libraries: | @@ -84,11 +79,11 @@ jobs: - eigen - fftw setenv: | - WEBBPSF_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}webbpsf-data/ - GALSIM_CAT_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits + WEBBPSF_PATH: ${{ needs.romanisim_data.outputs.cache_path }}/webbpsf-data/ + GALSIM_CAT_PATH: ${{ needs.romanisim_data.outputs.cache_path }}/galsim_data/real_galaxy_catalog_23.5_example.fits FFTW_DIR: /opt/homebrew/opt/fftw/lib/ - cache-path: ${{ needs.romanisim_data_cache.outputs.cache_path }} - cache-key: ${{ needs.romanisim_data_cache.outputs.cache_key }} + cache-path: ${{ needs.romanisim_data.outputs.cache_path }} + cache-key: ${{ needs.romanisim_data.outputs.cache_key }} envs: | - linux: py311-test-romanisim-cov-xdist @@ -111,4 +106,4 @@ jobs: default_python: '3.11' envs: | - linux: ndcube - - linux: dkist \ No newline at end of file + - linux: dkist diff --git a/.github/workflows/get_webbpsf.yml b/.github/workflows/get_webbpsf.yml deleted file mode 100644 index 0b7d7c7c..00000000 --- a/.github/workflows/get_webbpsf.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: retrieve latest data cache key - -on: - workflow_call: - outputs: - cache_key: - value: ${{ jobs.retrieve_cache.outputs.cache_key }} - cache_path: - value: ${{ jobs.retrieve_cache.outputs.cache_path }} - -jobs: - retrieve_cache: - name: retrieve latest data cache key - runs-on: ubuntu-latest - steps: - - name: retrieve latest data cache key - id: latest_cache_key - run: | - # use actions/gh-actions-cache to allow filtering by key - gh extension install actions/gh-actions-cache - CACHE_KEY=$(gh actions-cache list -R ${{ github.repository }} --key data- --sort created-at | cut -f 1 | head -n 1) - if [ "$CACHE_KEY" == '' ]; then exit 1; fi - echo cache_key=$CACHE_KEY >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - - run: echo cache_key=${{ steps.latest_cache_key.outputs.cache_key }} - - id: cache_path - run: echo cache_path=/tmp/data/ >> $GITHUB_OUTPUT - outputs: - cache_key: ${{ steps.latest_cache_key.outputs.cache_key }} - cache_path: ${{ steps.cache_path.outputs.cache_path }} diff --git a/.github/workflows/romanisim_data.yml b/.github/workflows/romanisim_data.yml new file mode 100644 index 00000000..405e30b7 --- /dev/null +++ b/.github/workflows/romanisim_data.yml @@ -0,0 +1,18 @@ +name: download and cache romanisim data + +on: + schedule: + - cron: "42 4 * * 3" + workflow_dispatch: + inputs: + webbpsf_minimal: + description: minimal WebbPSF dataset + type: boolean + required: false + default: true + +jobs: + download_romanisim_data: + uses: spacetelescope/romanisim/.github/workflows/retrieve_cache.yml@develop + with: + minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml deleted file mode 100644 index 9d45e9d5..00000000 --- a/.github/workflows/webbpsf_data.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: download and cache data - -on: - workflow_call: - schedule: - - cron: "42 4 * * 3" - workflow_dispatch: - inputs: - webbpsf_minimal: - description: minimal WebbPSF dataset - type: boolean - required: false - default: true - -jobs: - download_webbpsf_data: - uses: spacetelescope/webbpsf/.github/workflows/download_data.yml@beda656c80a0254e6f80649d9c9c49235634522f # v1.4.0 - with: - minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} - combine_data_cache: - needs: [ download_webbpsf_data ] - runs-on: ubuntu-latest - env: - GALSIM_DATA_URL: https://github.com/GalSim-developers/GalSim/raw/releases/2.4/examples/data/ - steps: - - name: download GalSim data - run: | - mkdir galsim_data - wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example.fits -O galsim_data/real_galaxy_catalog_23.5_example.fits - wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_selection.fits -O galsim_data/real_galaxy_catalog_23.5_example_selection.fits - wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_fits.fits -O galsim_data/real_galaxy_catalog_23.5_example_fits.fits - - id: galsim_data - run: echo "hash=${{ hashFiles( 'galsim_data/' ) }}" >> $GITHUB_OUTPUT - - run: echo GALSIM_PATH=/tmp/data/galsim_data/ >> $GITHUB_ENV - - run: | - mkdir -p ${{ env.GALSIM_PATH }} - mv ./galsim_data/* ${{ env.GALSIM_PATH }} - - run: echo GALSIM_CAT_PATH=${{ env.GALSIM_PATH }}real_galaxy_catalog_23.5_example.fits >> $GITHUB_ENV - - name: retrieve cached WebbPSF data - uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: ${{ needs.download_webbpsf_data.outputs.cache_path }} - key: ${{ needs.download_webbpsf_data.outputs.cache_key }} - - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ /tmp/data/ - - run: echo WEBBPSF_PATH=/tmp/data/webbpsf-data/ >> $GITHUB_ENV - # save a new cache to the same generalized data directory, combined with extra data - - name: save a single combined data cache - uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /tmp/data/ - key: data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }} From f46a8383f60bb613d8a41e9b7737b9c09b86f3c9 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Wed, 20 Nov 2024 13:41:09 -0500 Subject: [PATCH 3/4] fix ref --- .github/workflows/downstream.yml | 2 +- .github/workflows/romanisim_data.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 4d8b4d1f..66a7d1ae 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -66,7 +66,7 @@ jobs: romanisim_data: needs: [ environment ] - uses: spacetelescope/romanisim/.github/workflows/retrieve_cache.yml@develop + uses: spacetelescope/romanisim/.github/workflows/retrieve_cache.yml@main with: cache_path: ${{ needs.environment.outputs.data_path }} diff --git a/.github/workflows/romanisim_data.yml b/.github/workflows/romanisim_data.yml index 405e30b7..aba64646 100644 --- a/.github/workflows/romanisim_data.yml +++ b/.github/workflows/romanisim_data.yml @@ -13,6 +13,6 @@ on: jobs: download_romanisim_data: - uses: spacetelescope/romanisim/.github/workflows/retrieve_cache.yml@develop + uses: spacetelescope/romanisim/.github/workflows/retrieve_cache.yml@main with: minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} From 60de35040bc54832e80c5d098b9ab95e4fd286a2 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Wed, 20 Nov 2024 15:23:32 -0500 Subject: [PATCH 4/4] de-duplicate `pass_env` in `tox.ini` --- tox.ini | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 52500f21..902b40f1 100644 --- a/tox.ini +++ b/tox.ini @@ -51,17 +51,18 @@ description = warnings: treating warnings as errors cov: with coverage xdist: using parallel processing -passenv = +pass_env = HOME GITHUB_* TOXENV CI CODECOV_* DISPLAY - WEBBPSF_PATH - GALSIM_CAT_PATH - FFTW_DIR - LIBRARY_PATH + jwst,romancal: CRDS_* + romanisim,romancal: WEBBPSF_PATH + romanisim: GALSIM_CAT_PATH + romanisim: FFTW_DIR + romanisim: LIBRARY_PATH set_env = dev: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/liberfa/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple @@ -78,8 +79,6 @@ deps = romanisim: romanisim[test] @ git+https://github.com/spacetelescope/romanisim.git numpy123: numpy==1.23.* numpy125: numpy==1.25.* -pass_env = - jwst,romancal: CRDS_* commands_pre = dev: pip install -r requirements-dev.txt -U --upgrade-strategy eager pip freeze