Feature: perfect restart and n-cores CI (#778)

* Add n-cores and perfect restart GitHub action tests * Add generic|n-cores|perfect-restart string to title of plots
NCAR · Dec 10, 2024 · 644b446 · 644b446
1 parent b69c231
commit 644b446
Show file tree

Hide file tree

Showing 2 changed files with 235 additions and 14 deletions.
diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml
@@ -113,7 +113,7 @@ jobs:
           cd $GITHUB_WORKSPACE/candidate/build/Run
           make run-croton-${{ matrix.configuration }}
 
-      - name: Compare HYDRO_RST.* output with xrcmp
+      - name: generic - Compare HYDRO_RST.* output with xrcmp
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
@@ -124,8 +124,7 @@ jobs:
               --log_file $file_diff.txt \
               --n_cores 1; \
           done
-
-      - name: Compare RESTART.* output with xrcmp
+      - name: generic - Compare RESTART.* output with xrcmp
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
@@ -136,8 +135,7 @@ jobs:
               --log_file $file_diff.txt \
               --n_cores 1; \
           done
-
-      - name: Compare last *.CHANOBS_DOMAIN1 output with xrcmp
+      - name: generic - Compare last *.CHANOBS_DOMAIN1 output with xrcmp
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
@@ -147,8 +145,116 @@ jobs:
               --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
               --log_file $file_diff.txt \
               --n_cores 1; \
+      - name: generic - Compare last *.CHRTOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+      - name: generic - Compare last *.LSMOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+
+      - name: generic - Compare last *.RTOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
 
-      - name: Compare last *.CHRTOUT_DOMAIN1 output with xrcmp
+      - name: generic - Compare output with compare_output
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          mkdir output_diff
+          python -c \
+          "import sys; \
+          sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \
+          import compare_output; \
+          from pathlib import Path; \
+          compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \
+            '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \
+            '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \
+            '${{ matrix.configuration }}')"
+
+      - name: generic - Copy test results from container
+        if: ${{ always() }}
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/test_report
+          cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/
+
+      - name: generic - Attach diff plots to PR
+        if: ${{ failure() }}
+        shell: bash
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/tests/local/utils
+          bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} generic
+
+      - name: generic - Archive test results to GitHub
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-reports
+          path: |
+            ${{ github.workspace }}/test_report/*
+
+
+      # n-cores test
+      - name: Run parallel candidate model
+        run: |
+          rm -r $GITHUB_WORKSPACE/test_report/
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          make clean
+          make run-croton-${{ matrix.configuration }}-parallel
+
+      - name: n-cores - Compare HYDRO_RST.* output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          for file in output_${{ matrix.configuration }}/HYDRO_RST.*; do\
+            python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+          done
+      - name: n-cores - Compare RESTART.* output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          for file in output_${{ matrix.configuration }}/RESTART.*; do\
+            python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+          done
+      - name: n-cores - Compare last *.CHANOBS_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+      - name: n-cores - Compare last *.CHRTOUT_DOMAIN1 output with xrcmp
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
@@ -158,8 +264,120 @@ jobs:
               --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
               --log_file $file_diff.txt \
               --n_cores 1; \
+      - name: n-cores - Compare last *.LSMOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.LSMOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+
+      - name: n-cores - Compare last *.RTOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.RTOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+
+      - name: n-cores - Compare output with compare_output
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          rm -rf output_diff
+          mkdir output_diff
+          python -c \
+          "import sys; \
+          sys.path.append('${GITHUB_WORKSPACE}/candidate/tests/utils'); \
+          import compare_output; \
+          from pathlib import Path; \
+          compare_output.plot_diffs('${GITHUB_WORKSPACE}/candidate/build/Run/output_diff', \
+            '${GITHUB_WORKSPACE}/candidate/build/Run/output_${{ matrix.configuration }}/', \
+            '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \
+            '${{ matrix.configuration }}')"
 
-      - name: Compare last *.LSMOUT_DOMAIN1 output with xrcmp
+      - name: n-cores - Copy test results from container
+        if: ${{ always() }}
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/test_report
+          cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/
+
+      - name: n-cores - Attach diff plots to PR
+        if: ${{ failure() }}
+        shell: bash
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/tests/local/utils
+          bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} n-cores
+
+      - name: n-cores - Archive test results to GitHub
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-reports
+          path: |
+            ${{ github.workspace }}/test_report/*
+
+
+      # Testing perfect restart, not cleaning candidate model output
+      - name: Setup and run candidate model perfect restart startup
+        run: |
+          rm -r $GITHUB_WORKSPACE/test_report/
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          sed -i 's|RESTART_FILENAME_REQUESTED = "RESTART/RESTART.2011082600_DOMAIN1"|RESTART_FILENAME_REQUESTED = "./RESTART.2011090100_DOMAIN1"|' namelist.hrldas
+          sed -i 's/KDAY = 7/KDAY = 1/' namelist.hrldas
+          rm output_${{ matrix.configuration }}/RESTART.2011090200_DOMAIN1
+          rm output_${{ matrix.configuration }}/HYDRO_RST.2011-09-02_00:00_DOMAIN1
+          make run-croton-${{ matrix.configuration }}-parallel
+
+      - name: restart - Compare HYDRO_RST.* output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          for file in output_${{ matrix.configuration }}/HYDRO_RST.2011-09-02_00:00_DOMAIN1; do\
+            python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+          done
+      - name: restart - Compare RESTART.* output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          for file in output_${{ matrix.configuration }}/RESTART.2011090200_DOMAIN1; do\
+            python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+          done
+      - name: restart - Compare last *.CHANOBS_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.CHANOBS_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+      - name: restart - Compare last *.CHRTOUT_DOMAIN1 output with xrcmp
+        if: ${{ always() }}
+        run: |
+          cd $GITHUB_WORKSPACE/candidate/build/Run
+          file=$(ls -t output_${{ matrix.configuration }}/*.CHRTOUT_DOMAIN1 | head -n 1)
+          python ${GITHUB_WORKSPACE}/candidate/tests/utils/xrcmp.py \
+              --candidate $file \
+              --reference $GITHUB_WORKSPACE/reference/build/Run/$file \
+              --log_file $file_diff.txt \
+              --n_cores 1; \
+      - name: restart - Compare last *.LSMOUT_DOMAIN1 output with xrcmp
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
@@ -170,7 +388,7 @@ jobs:
               --log_file $file_diff.txt \
               --n_cores 1; \
 
-      - name: Compare last *.RTOUT_DOMAIN1 output with xrcmp
+      - name: restart - Compare last *.RTOUT_DOMAIN1 output with xrcmp
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
@@ -181,10 +399,11 @@ jobs:
               --log_file $file_diff.txt \
               --n_cores 1; \
 
-      - name: Compare output with compare_output
+      - name: restart - Compare output with compare_output
         if: ${{ always() }}
         run: |
           cd $GITHUB_WORKSPACE/candidate/build/Run
+          rm -rf output_diff
           mkdir output_diff
           python -c \
           "import sys; \
@@ -196,19 +415,20 @@ jobs:
             '${GITHUB_WORKSPACE}/reference/build/Run/output_${{ matrix.configuration }}/', \
             '${{ matrix.configuration }}')"
 
-      - name: Copy test results from container
+      - name: restart - Copy test results from container
         if: ${{ always() }}
         run: |
           mkdir -p $GITHUB_WORKSPACE/test_report
           cp -r $GITHUB_WORKSPACE/candidate/build/Run/output_diff/diff_plots/* $GITHUB_WORKSPACE/test_report/
 
-      - name: Attach diff plots to PR
+      - name: restart - Attach diff plots to PR
         if: ${{ failure() }}
+        shell: bash
         run: |
           cd $GITHUB_WORKSPACE/candidate/tests/local/utils
-          bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }}
+          bash attach_all_plots.bash $(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") ${{ matrix.configuration }} perfect-restart
 
-      - name: Archive test results to GitHub
+      - name: restart - Archive test results to GitHub
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:

diff --git a/tests/local/utils/attach_all_plots.bash b/tests/local/utils/attach_all_plots.bash
@@ -2,6 +2,7 @@
 
 PR=$1
 CONFIG=$2
+TEST_TYPE=$3
 cwd=`pwd`
 diffs=$GITHUB_WORKSPACE/test_report/$CONFIG
 
@@ -10,7 +11,7 @@ if [[ ! -d $diffs ]]; then
     exit 0
 fi
 
-title="Difference plots for configuration '$CONFIG'"
+title="Difference plots for configuration '$CONFIG' and $TEST_TYPE test type"
 
 # set for repo authentication
 git config --global user.email "[email protected]"