PR for #25 - Validation workflow (commented out until true reference files available)

.github/workflows/CPU_Beltrami_build_run.yml

@@ -1,10 +1,7 @@
-name: Samurai CPU test with Beltrami
+name: Samurai CPU CI with Beltrami
 
 on:
   workflow_dispatch:
-    branches: 
-      - "main"
-      - "thermo_asap"
   push:
     branches: 
       - "main"
@@ -15,42 +12,56 @@ on:
       - "thermo_asap"
 
 jobs:
-  docker-build-and-test:
-    name: Build and Test - ${{ matrix.dockerfile }}
+  build-and-run:
+    name: Build and Run
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest]
-        dockerfile:
-          - Dockerfile
-          - Dockerfile.coverage 
     steps:
     - name: Checkout code from a pull request or push
       uses: actions/checkout@v3
 
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
     - name: Delete huge unnecessary tools folder
       run: rm -rf /opt/hostedtoolcache
 
     - name: Build Docker image
-      run: docker build -f docker/${{ matrix.dockerfile }} -t samurai .
+      run: docker build -f docker/Dockerfile -t samurai:latest .
+
+    - name: Running the Beltrami case in container
+      run: docker run -d --name test-container samurai:latest
 
-    - name: Running the Docker image for Beltrami case
-      # only run this if we are not running codecoverage tests
-      if: matrix.dockerfile != 'Dockerfile.coverage'
-      run: docker run --name test-container -t samurai
+   #uncomment to include validation tests   
+   # - name: Setup validation environment
+   #   run: docker build -f docker/Dockerfile.validation -t samurai-validation .
+
+   # - name: Run Beltrami validation test 
+   #   run: docker run --name test-container-validation --volumes-from test-container samurai-validation
+
+  coverage:
+    name: Coverage
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code from a pull request or push
+      uses: actions/checkout@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Delete huge unnecessary tools folder
+      run: rm -rf /opt/hostedtoolcache
+
+    - name: Build Docker image
+      run: docker build -f docker/Dockerfile.coverage -t samurai .
 
     - name: Run coverage tests in container
-      if: matrix.dockerfile == 'Dockerfile.coverage'
       run: docker run --name test-container -t samurai make coverage
 
     - name: Copy coverage from container
-      if: matrix.dockerfile == 'Dockerfile.coverage'
       run: docker cp test-container:/app/samurai/build/coverage.info .
 
     - name: Upload coverage reports to Codecov
-      if: matrix.dockerfile == 'Dockerfile.coverage'
       uses: codecov/codecov-action@v4
       with:
         files: coverage.info
-        token: ${{ secrets.CODECOV_TOKEN }}
+        token: ${{ secrets.CODECOV_TOKEN }}

docker/Dockerfile

@@ -25,12 +25,19 @@ RUN sed -i 's|data_directory = "/glade/campaign/cisl/asap/samurai/data/beltrami"
 ENV PATH="/app/samurai/build:${PATH}"
 ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}"
 ENV PATH="/usr/local/bin:${PATH}"
+#RUN pip install --no-cache-dir numpy
 
 RUN echo "#!/bin/bash \n\
 cd /app/samurai/build/release/bin/ \n\
-./samurai -params /app/samurai/ncar_scripts/TDRP/beltrami.tdrp\
+./samurai -params /app/samurai/ncar_scripts/TDRP/beltrami.tdrp | tee /app/samurai/ncar_scripts/validation/beltrami/test_log \n\
 " > /app/run_samurai.sh
 WORKDIR /app/
 RUN chmod +x /app/run_samurai.sh
+#RUN chmod +x /app/samurai/ncar_scripts/validation/beltrami/validation_env.sh
+#RUN chmod +x /app/samurai/ncar_scripts/validation/beltrami/bel_val_nc.py
 
 CMD [ "./run_samurai.sh" ]
+
+#CMD ["python3", "/app/samurai/ncar_scripts/validation/beltrami/bel_val_nc.py", "/app/samurai/ncar_scripts/validation/beltrami/samurai_XYZ_wind_analysis_ref.nc", "/app/samurai/build/release/bin/samurai_XYZ_wind_analysis.nc"]
+#python3 /app/samurai/ncar_scripts/validation/beltrami/bel_val_nc.py /app/samurai/ncar_scripts/validation/beltrami/samurai_XYZ_wind_analysis_ref.nc /app/samurai/build/release/bin/samurai_XYZ_wind_analysis.nc > /app/beltrami_output.log \n\
+#python /app/samurai/ncar_scripts/validation/beltrami/compare_log_file.py /app/samurai/ncar_scripts/validation/beltrami/ref_log /app/samurai/ncar_scripts/validation/beltrami/test_log  \n\

docker/Dockerfile.validation

@@ -0,0 +1,27 @@
+# Use the base image built by main Dockerfile
+FROM samurai:latest
+
+# Set the working directory
+WORKDIR /app
+
+# Ensure pip is installed
+RUN apt-get update && apt-get install --reinstall zlib1g
+RUN apt-get install -y python3-venv
+RUN python3 -m venv /app/venv && \
+    /app/venv/bin/pip install --upgrade pip && \
+    /app/venv/bin/pip install numpy xarray netCDF4 scipy cftime 
+
+# Make the validation script executable
+RUN chmod +x /app/samurai/ncar_scripts/validation/beltrami/bel_val_nc.py
+
+RUN echo "#!/bin/bash \n\
+cd /app/samurai/build/release/bin/ \n\
+./samurai -params /app/samurai/ncar_scripts/TDRP/beltrami.tdrp | tee /app/samurai/ncar_scripts/validation/beltrami/test_log \n\
+cd /app/samurai/ncar_scripts/validation/beltrami/ \n\
+/app/venv/bin/python3 bel_val_nc.py samurai_XYZ_wind_analysis_ref.nc /app/samurai/build/release/bin/samurai_XYZ_wind_analysis.nc \n\
+" > /app/run_validation.sh
+
+RUN chmod +x /app/run_validation.sh
+
+# Set the CMD to run the validation script
+CMD [ "./run_validation.sh" ]

ncar_scripts/sync_repos/updateSamuraiDevelopment.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+##################################################################
+# This script is intended to update our release version of
+# clubb (the one "publicly" visible). It should be ran nightly.
+#
+# Author: Nicolas Strike, 2019
+##################################################################
+
+# To add a new branch from clubb to clubb_release, simply add it to the array here and it will be added overnight and updated automatically. Note that there must be a space between items and the parethesis.
+branches=( main thermo_asap)
+
+# List of patterns describing lfs files to untrack in git-lfs. This prevents them from being automatically downloaded when a client clones the repo, but leaves them available to download if desired (with `git lfs pull`)
+undesiredLfsPatterns=( )
+
+
+maxNetworkAttempts=15
+internetConnectivityTestUrl="github.com"
+
+# Need to fill in name of private development samurai repo (Probably [email protected]:NCAR/samurai.git)
+[email protected]:mmbell/samurai.git
+[email protected]:NCAR/samurai-dev.git
+
+updateBranch() {
+    bash updateGitFork.sh --source-repo $updateSrc --destination-repo $updateDest --source-branch $1 --dest-branch $1 -f -t
+}
+
+networkSafeOperation (){
+  #limited to 8 arguments, see below
+
+  tryAgain="true"
+  netAttempts=$maxNetworkAttempts
+  # initial wait time before the second network attempt in seconds.
+  # Sequential attempts get a x2 multiplier
+  netWaitDelay=2
+  while [ $tryAgain == "true" -a $netAttempts -gt 0 ]; do
+
+    #Run the given command with up to 8 arguments
+    $1 $2 $3 $4 $5 $6 $7 $8 $9
+
+    cloneCode=$?
+    netAttempts=$((netAttempts-1))
+    #check if the repo successfully cloned
+    if [ $cloneCode == 0 ]
+    then
+        tryAgain="false"
+    else
+        echo "Testing network connectivity."
+        ping -c 4 $internetConnectivityTestUrl
+        pingCode=$?
+        if [ $pingCode != 0 ]
+        then
+          echo "Failed to perform network operation '$1 $2 $3 $4 $5 $6 $7' due to a network issue. $netAttempts attempts left, waiting $netWaitDelay seconds."
+          sleep $netWaitDelay
+          netWaitDelay=$((netWaitDelay*2))
+        else
+          echo "Failed to perform operation '$1 $2 $3 $4 $5 $6 $7', but no network issue was detected. This must be handeled manually and is likely a git conflict."
+          tryAgain="false"
+        #pingCode
+        fi
+    #cloneCode
+    fi
+    done
+}
+
+updateVersionFiles(){
+    rm -rf /tmp/temp_release_repo
+    networkSafeOperation git clone $updateDest /tmp/temp_release_repo
+    cd /tmp/temp_release_repo
+        git log -10 > src/CLUBB_core/version_clubb_core.txt
+        git log -10 > src/SILHS/version_silhs.txt
+        git add src/CLUBB_core/version_clubb_core.txt src/SILHS/version_silhs.txt
+        git commit -a -m "Updating the clubb core and silhs version files"
+        networkSafeOperation git push
+    cd -
+    rm -rf /tmp/temp_release_repo
+}
+
+untrackLfs(){
+    for i in "${undesiredLfsPatterns[@]}"
+    do
+	    git lfs untrack $i
+    done
+}
+
+updateAllBranches(){
+    for i in "${branches[@]}"
+    do
+	    updateBranch $i
+    done
+}
+
+printf "\n\n######################################\nUpdating repo at $updateDest from $updateSrc\n######################################\n"
+
+    printf "\n###### Updating all branches ######\n"
+        updateAllBranches
+    printf "Done updating branches.\n"
+
+    #printf "\n###### Updating untracking lfs files ######\n"
+    #    untrackLfs
+    #printf "Done untracking lfs files.\n"
+
+    #printf "\n###### Updating version files ######\n"
+    #    updateVersionFiles
+    #printf "Done updating version files\n"
+
+printf "\n###### DONE ######\n"

ncar_scripts/updateSamuraiPublic.sh → ncar_scripts/sync_repos/updateSamuraiPublic.sh

@@ -2,13 +2,13 @@
 
 ##################################################################
 # This script is intended to update our release version of
-# clubb (the one "publicly" visible). It should be ran nightly.
+# samurai (the one "publicly" visible). It should be ran nightly.
 #
 # Author: Nicolas Strike, 2019
 ##################################################################
 
-# To add a new branch from clubb to clubb_release, simply add it to the array here and it will be added overnight and updated automatically. Note that there must be a space between items and the parethesis.
-branches=( main thermo_asap)
+# To add a new branch from samurai-dev to samurai (release), simply add it to the array here and it will be added overnight and updated automatically. Note that there must be a space between items and the parethesis.
+branches=( main )
 
 # List of patterns describing lfs files to untrack in git-lfs. This prevents them from being automatically downloaded when a client clones the repo, but leaves them available to download if desired (with `git lfs pull`)
 undesiredLfsPatterns=( )
@@ -17,8 +17,10 @@ undesiredLfsPatterns=( )
 maxNetworkAttempts=15
 internetConnectivityTestUrl="github.com"
 
-# Need to fill in name of private development samurai repo (Probably [email protected]:NCAR/samurai.git)
-updateSrc=
+# Need to fill in name of private development samurai repo (Probably [email protected]:NCAR/samurai-dev.git)
+
+# UNCOMMENT THE FOLLOWING LINE AND VERIFY THE REPO NAMES BEFORE RUNNING
+#[email protected]:NCAR/samurai-dev.git
 [email protected]:mmbell/samurai.git
 
 updateBranch() {

ncar_scripts/utils/reduceNetCDF.py

@@ -0,0 +1,42 @@
+import argparse
+import netCDF4 as nc
+
+def main(input_file, output_file, variables_to_keep):
+
+    # Open the original NetCDF file
+    with nc.Dataset(input_file, 'r') as src, nc.Dataset(output_file, 'w') as dst:
+        # Copy global attributes
+        dst.setncatts({attr: src.getncattr(attr) for attr in src.ncattrs()})
+
+        # Copy dimensions
+        for name, dimension in src.dimensions.items():
+            dst.createDimension(name, (len(dimension) if not dimension.isunlimited() else None))
+
+        # Copy selected variables
+        for name, variable in src.variables.items():
+            if name in variables_to_keep:
+                new_var = dst.createVariable(name, variable.datatype, variable.dimensions)
+                new_var.setncatts({attr: variable.getncattr(attr) for attr in variable.ncattrs()})
+                new_var[:] = variable[:]
+
+    print(f"New NetCDF file created at {output_file} with only the selected variables.")
+
+if __name__ == "__main__":
+    # Define the default variables to keep
+    default_variables = ['U', 'V', 'W', 'DIV', 'VORT']
+
+    # Create a string representation of the default variables
+    default_variables_str = ', '.join(default_variables)
+
+    # Set up argument parsing
+    parser = argparse.ArgumentParser(description='Reduce NetCDF file by keeping only specified variables.')
+    parser.add_argument('input_file', type=str, help='Path to the input NetCDF file.')
+    parser.add_argument('output_file', type=str, help='Path to the output NetCDF file.')
+    parser.add_argument('--variables', type=str, nargs='+', default=default_variables,
+                        help=f'List of variables to keep (default: {default_variables_str}).')
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Call the main function with parsed arguments
+    main(args.input_file, args.output_file, args.variables)