allow data workflows (WebbPSF + GALSIM) to be used by upstream repos …

…for downstream testing allow user to specify cache path don't assume cache path fix context availability force /tmp/data for compatibility with macOS add descriptive job name rename outputs ls directory fix outputs
spacetelescope · Nov 20, 2024 · 7facf28 · 7facf28
1 parent 240e9a4
commit 7facf28
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 11 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -27,6 +27,8 @@ jobs:
         - linux: check-style
   latest_data_cache:
     uses: ./.github/workflows/retrieve_cache.yml
+    with:
+      cache_path: /tmp/data
   test:
     needs: [ latest_data_cache ]
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@924441154cf3053034c6513d5e06c69d262fb9a6  # v1.13.0
@@ -36,8 +38,8 @@ jobs:
           - eigen
           - fftw
       setenv: |
-        WEBBPSF_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}webbpsf-data/
-        GALSIM_CAT_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits
+        WEBBPSF_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}/webbpsf-data/
+        GALSIM_CAT_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}/galsim_data/real_galaxy_catalog_23.5_example.fits
         FFTW_DIR: /opt/homebrew/opt/fftw/lib/
       cache-path: ${{ needs.latest_data_cache.outputs.cache_path }}
       cache-key: ${{ needs.latest_data_cache.outputs.cache_key }}

diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
@@ -25,8 +25,8 @@ jobs:
           - eigen
           - fftw
       setenv: |
-        WEBBPSF_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}webbpsf-data/
-        GALSIM_CAT_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}galsim_data/real_galaxy_catalog_23.5_example.fits
+        WEBBPSF_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}/webbpsf-data/
+        GALSIM_CAT_PATH: ${{ needs.latest_data_cache.outputs.cache_path }}/galsim_data/real_galaxy_catalog_23.5_example.fits
         FFTW_DIR: /opt/homebrew/opt/fftw/lib/
       cache-path: ${{ needs.latest_data_cache.outputs.cache_path }}
       cache-key: ${{ needs.latest_data_cache.outputs.cache_key }}

diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml
@@ -3,6 +3,21 @@ name: download and cache data
 on:
   schedule:
     - cron: "42 4 * * 3"
+  workflow_call:
+    inputs:
+      webbpsf_minimal:
+        type: boolean
+        required: false
+        default: true
+      cache_path:
+        type: string
+        required: false
+        default: ""
+    outputs:
+      cache_key:
+        value: ${{ jobs.combine_data_cache.outputs.cache_key }}
+      cache_path:
+        value: ${{ jobs.combine_data_cache.outputs.cache_path }}
   workflow_dispatch:
     inputs:
       webbpsf_minimal:
@@ -17,11 +32,14 @@ jobs:
     with:
       minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.webbpsf_minimal }}
   combine_data_cache:
+    name: combine GalSim and WebbPSF data into single cache
     needs: [ download_webbpsf_data ]
     runs-on: ubuntu-latest
     env:
       GALSIM_DATA_URL: https://github.com/GalSim-developers/GalSim/raw/releases/2.4/examples/data/
     steps:
+      - id: cache_path
+        run: echo cache_path=${{ inputs.cache_path != '' && inputs.cache_path || format('{0}/data', runner.temp) }} >> $GITHUB_OUTPUT
       - name: download GalSim data    
         run: |
           mkdir galsim_data
@@ -30,7 +48,7 @@ jobs:
           wget ${{ env.GALSIM_DATA_URL }}/real_galaxy_catalog_23.5_example_fits.fits -O  galsim_data/real_galaxy_catalog_23.5_example_fits.fits
       - id: galsim_data
         run: echo "hash=${{ hashFiles( 'galsim_data/' ) }}" >> $GITHUB_OUTPUT
-      - run: echo GALSIM_PATH=/tmp/data/galsim_data/ >> $GITHUB_ENV
+      - run: echo GALSIM_PATH=${{ steps.cache_path.outputs.cache_path }}/galsim_data/ >> $GITHUB_ENV
       - run: |
           mkdir -p ${{ env.GALSIM_PATH }}
           mv ./galsim_data/* ${{ env.GALSIM_PATH }}
@@ -40,11 +58,18 @@ jobs:
         with:
           path: ${{ needs.download_webbpsf_data.outputs.cache_path }}
           key: ${{ needs.download_webbpsf_data.outputs.cache_key }}
-      - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ /tmp/data/
-      - run: echo WEBBPSF_PATH=/tmp/data/webbpsf-data/ >> $GITHUB_ENV
+      - run: mv ${{ needs.download_webbpsf_data.outputs.cache_path }}/webbpsf-data/ ${{ steps.cache_path.outputs.cache_path }}
+        continue-on-error: true
+      - run: ls ${{ steps.cache_path.outputs.cache_path }}/*
+      - run: echo WEBBPSF_PATH=${{ steps.cache_path.outputs.cache_path }}/webbpsf-data/ >> $GITHUB_ENV
+      - id: cache_key
+        run: echo cache_key=data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }} >> $GITHUB_OUTPUT
       # save a new cache to the same generalized data directory, combined with extra data
       - name: save a single combined data cache
         uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
         with:
-          path: /tmp/data/
-          key: data-${{ needs.download_webbpsf_data.outputs.cache_key }}-galsim-data-${{ steps.galsim_data.outputs.hash }}
+          path: ${{ steps.cache_path.outputs.cache_path }}
+          key: ${{ steps.cache_key.outputs.cache_key }}
+    outputs:
+      cache_key: ${{ steps.cache_key.outputs.cache_key }}
+      cache_path: ${{ steps.cache_path.outputs.cache_path }}
diff --git a/.github/workflows/retrieve_cache.yml b/.github/workflows/retrieve_cache.yml
@@ -2,6 +2,11 @@ name: retrieve latest data cache key
 
 on:
   workflow_call:
+    inputs:
+      cache_path:
+        type: string
+        required: false
+        default: ""
     outputs:
       cache_key:
         value: ${{ jobs.retrieve_cache.outputs.cache_key }}
@@ -13,6 +18,8 @@ jobs:
     name: retrieve latest data cache key
     runs-on: ubuntu-latest
     steps:
+      - id: cache_path
+        run: echo cache_path=${{ inputs.cache_path != '' && inputs.cache_path || format('{0}/data', runner.temp) }} >> $GITHUB_OUTPUT
       - name: retrieve latest data cache key
         id: latest_cache_key
         run: |
@@ -24,8 +31,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
       - run: echo cache_key=${{ steps.latest_cache_key.outputs.cache_key }}
-      - id: cache_path
-        run: echo cache_path=/tmp/data/ >> $GITHUB_OUTPUT
+      - run: echo cache_path=${{ steps.cache_path.outputs.cache_path }}
     outputs:
       cache_key: ${{ steps.latest_cache_key.outputs.cache_key }}
       cache_path: ${{ steps.cache_path.outputs.cache_path }}