diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index cda547b8..a413a442 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -68,15 +68,11 @@ jobs: envs: | - linux: py311-test-romancal-cov-xdist - get_romanism_data: - uses: ./.github/workflows/webbpsf_data.yml - - romanisim_data_cache: - needs: [ get_romanism_data ] - uses: ./.github/workflows/get_webbpsf.yml + romanisim_data: + uses: zacharyburnett/romanisim/.github/workflows/retrieve_cache.yml@ci/abstract_data romanisim: - needs: [ romanisim_data_cache ] + needs: [ romanisim_data ] uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6 # v1.13.0 with: libraries: | @@ -84,8 +80,8 @@ jobs: - eigen - fftw setenv: | - WEBBPSF_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}webbpsf-data/ - GALSIM_CAT_PATH: ${{ needs.romanisim_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits + WEBBPSF_PATH: ${{ needs.romanisim_data.outputs.cache_path }}webbpsf-data/ + GALSIM_CAT_PATH: ${{ needs.romanisim_data.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits FFTW_DIR: /opt/homebrew/opt/fftw/lib/ cache-path: ${{ needs.romanisim_data_cache.outputs.cache_path }} cache-key: ${{ needs.romanisim_data_cache.outputs.cache_key }} @@ -111,4 +107,4 @@ jobs: default_python: '3.11' envs: | - linux: ndcube - - linux: dkist \ No newline at end of file + - linux: dkist diff --git a/.github/workflows/get_webbpsf.yml b/.github/workflows/get_webbpsf.yml deleted file mode 100644 index 0b7d7c7c..00000000 --- a/.github/workflows/get_webbpsf.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: retrieve latest data cache key - -on: - workflow_call: - outputs: - cache_key: - value: ${{ jobs.retrieve_cache.outputs.cache_key }} - cache_path: - value: ${{ jobs.retrieve_cache.outputs.cache_path }} - -jobs: - retrieve_cache: - name: retrieve latest data cache key - runs-on: ubuntu-latest - steps: - - name: retrieve latest data cache key - id: latest_cache_key - run: | - # use actions/gh-actions-cache to allow filtering by key - gh extension install actions/gh-actions-cache - CACHE_KEY=$(gh actions-cache list -R ${{ github.repository }} --key data- --sort created-at | cut -f 1 | head -n 1) - if [ "$CACHE_KEY" == '' ]; then exit 1; fi - echo cache_key=$CACHE_KEY >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - - run: echo cache_key=${{ steps.latest_cache_key.outputs.cache_key }} - - id: cache_path - run: echo cache_path=/tmp/data/ >> $GITHUB_OUTPUT - outputs: - cache_key: ${{ steps.latest_cache_key.outputs.cache_key }} - cache_path: ${{ steps.cache_path.outputs.cache_path }} diff --git a/.github/workflows/romanisim_data.yml b/.github/workflows/romanisim_data.yml new file mode 100644 index 00000000..20085420 --- /dev/null +++ b/.github/workflows/romanisim_data.yml @@ -0,0 +1,19 @@ +name: download and cache romanisim data + +on: + workflow_call: + schedule: + - cron: "42 4 * * 3" + workflow_dispatch: + inputs: + webbpsf_minimal: + description: minimal WebbPSF dataset + type: boolean + required: false + default: true + +jobs: + download_romanisim_data: + uses: zacharyburnett/romanisim/.github/workflows/data.yml@ci/abstract_data + with: + minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} diff --git a/.github/workflows/webbpsf_data.yml b/.github/workflows/webbpsf_data.yml index 9d45e9d5..c7a32334 100644 --- a/.github/workflows/webbpsf_data.yml +++ b/.github/workflows/webbpsf_data.yml @@ -1,51 +1,12 @@ -name: download and cache data +name: download and cache WebbPSF data on: - workflow_call: schedule: - cron: "42 4 * * 3" workflow_dispatch: - inputs: - webbpsf_minimal: - description: minimal WebbPSF dataset - type: boolean - required: false - default: true jobs: - download_webbpsf_data: - uses: spacetelescope/webbpsf/.github/workflows/download_data.yml@beda656c80a0254e6f80649d9c9c49235634522f # v1.4.0 + download_webbpsf: + uses: spacetelescope/webbpsf/.github/workflows/download_data.yml@develop with: - minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }} - combine_data_cache: - needs: [ download_webbpsf_data ] - runs-on: ubuntu-latest - env: - GALSIM_DATA_URL: https://github.com/GalSim-developers/GalSim/raw/releases/2.4/examples/data/ - steps: - - name: download GalSim data - run: | - mkdir galsim_data - wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example.fits -O galsim_data/real_galaxy_catalog_23.5_example.fits - wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_selection.fits -O galsim_data/real_galaxy_catalog_23.5_example_selection.fits - wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_fits.fits -O galsim_data/real_galaxy_catalog_23.5_example_fits.fits - - id: galsim_data - run: echo "hash=${{ hashFiles( 'galsim_data/' ) }}" >> $GITHUB_OUTPUT - - run: echo GALSIM_PATH=/tmp/data/galsim_data/ >> $GITHUB_ENV - - run: | - mkdir -p ${{ env.GALSIM_PATH }} - mv ./galsim_data/* ${{ env.GALSIM_PATH }} - - run: echo GALSIM_CAT_PATH=${{ env.GALSIM_PATH }}real_galaxy_catalog_23.5_example.fits >> $GITHUB_ENV - - name: retrieve cached WebbPSF data - uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: ${{ needs.download_webbpsf_data.outputs.cache_path }} - key: ${{ needs.download_webbpsf_data.outputs.cache_key }} - - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ /tmp/data/ - - run: echo WEBBPSF_PATH=/tmp/data/webbpsf-data/ >> $GITHUB_ENV - # save a new cache to the same generalized data directory, combined with extra data - - name: save a single combined data cache - uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 - with: - path: /tmp/data/ - key: data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }} + minimal: true