From e063d85868d439378c84df1fca729364dd884f62 Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Wed, 2 Oct 2024 14:55:11 -0400 Subject: [PATCH 001/111] checkout updated for issue #174 --- fre/pp/checkoutScript.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 5095a52c..1f06f73b 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -17,7 +17,7 @@ ############################################# -def _checkoutTemplate(experiment, platform, target, branch='main'): +def _checkoutTemplate(experiment, platform, target, branch='main',forced='off'): """ Checkout the workflow template files from the repo """ @@ -31,6 +31,15 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): # Set the name of the directory name = f"{experiment}__{platform}__{target}" + #check if version of branch is correct + branch_status=subprocess.check_output(['git','status']) + if str(branch_test).find('Your branch is up to date') < 0 and forced == 'off': + branch_error = ( + "Error in checkoutTemplate: Branch not up to date\n" + f"Turn paramter forced on to run without branches matching") + sys.exit(branch_error) + return 0 + # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) clonecmd = ( From 6e701aab8f1e55f18f8ea6e0a531bc6fae47957c Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Wed, 9 Oct 2024 09:54:12 -0400 Subject: [PATCH 002/111] Update checkoutScript.py syntax fixed on line 36 --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 1f06f73b..55b863e4 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -33,7 +33,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main',forced='off'): #check if version of branch is correct branch_status=subprocess.check_output(['git','status']) - if str(branch_test).find('Your branch is up to date') < 0 and forced == 'off': + if str(branch_status).find('Your branch is up to date') < 0 and forced == 'off': branch_error = ( "Error in checkoutTemplate: Branch not up to date\n" f"Turn paramter forced on to run without branches matching") From b9c9d01b0e923df5e45496c70a01aed3007d79b0 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 10 Oct 2024 15:20:41 -0400 Subject: [PATCH 003/111] Update checkoutScript.py switched to checking out tag 2024.01 --- fre/pp/checkoutScript.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 55b863e4..01017fae 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -17,7 +17,7 @@ ############################################# -def _checkoutTemplate(experiment, platform, target, branch='main',forced='off'): +def _checkoutTemplate(experiment, platform, target, branch='main'): """ Checkout the workflow template files from the repo """ @@ -31,19 +31,11 @@ def _checkoutTemplate(experiment, platform, target, branch='main',forced='off'): # Set the name of the directory name = f"{experiment}__{platform}__{target}" - #check if version of branch is correct - branch_status=subprocess.check_output(['git','status']) - if str(branch_status).find('Your branch is up to date') < 0 and forced == 'off': - branch_error = ( - "Error in checkoutTemplate: Branch not up to date\n" - f"Turn paramter forced on to run without branches matching") - sys.exit(branch_error) - return 0 - # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) clonecmd = ( f"git clone -b {branch} --single-branch --depth=1 --recursive " + f"git checkout tags/2024.01 " f"https://github.com/NOAA-GFDL/fre-workflows.git {name}" ) preexist_error = f"fatal: destination path '{name}' exists and is not an empty directory." click.echo(clonecmd) From 582592f579466e8ca98deea976c6131885186041 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Fri, 25 Oct 2024 13:47:40 -0400 Subject: [PATCH 004/111] added exit if local branch is not branch request --- fre/pp/checkoutScript.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 01017fae..73596933 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -30,7 +30,15 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): # Set the name of the directory name = f"{experiment}__{platform}__{target}" - + # Get name of local branch and check branch rquest + branch_names = subprocess.run(["git","branch"],capture_output=True, text=True) + local_branch_name = branch_names.stdout.split()[1] + if branch != 'main': + if branch != local_branch_name: + stop_report = ("Error in branch: local branch does not match branch input" + sys.exit(stop_report) + return 1 + # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) clonecmd = ( From a2f536f2f9388f982d72cff4f303513476f55869 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Mon, 28 Oct 2024 10:29:22 -0400 Subject: [PATCH 005/111] fixed bracket --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 73596933..dc58c15b 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -35,7 +35,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): local_branch_name = branch_names.stdout.split()[1] if branch != 'main': if branch != local_branch_name: - stop_report = ("Error in branch: local branch does not match branch input" + stop_report = ("Error in branch: local branch does not match branch input") sys.exit(stop_report) return 1 From 3a7f3784aae553d38e2c3a5279e4118c1b58ae19 Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Thu, 31 Oct 2024 13:37:38 -0400 Subject: [PATCH 006/111] tags version --- fre/pp/checkoutScript.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 27cecdc0..3eccb561 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -4,11 +4,15 @@ # Description: import os +import sys import subprocess from subprocess import PIPE from subprocess import STDOUT import re import click +from fre import fre +from click.testing import CliRunner + ############################################# @@ -29,6 +33,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): # Set the name of the directory name = f"{experiment}__{platform}__{target}" + # Get name of local branch and check branch rquest branch_names = subprocess.run(["git","branch"],capture_output=True, text=True) local_branch_name = branch_names.stdout.split()[1] @@ -37,16 +42,19 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): stop_report = ("Error in branch: local branch does not match branch input") sys.exit(stop_report) return 1 - + # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) clonecmd = ( f"git clone -b {branch} --single-branch --depth=1 --recursive " - f"git checkout tags/2024.01 " f"https://github.com/NOAA-GFDL/fre-workflows.git {name}" ) preexist_error = f"fatal: destination path '{name}' exists and is not an empty directory." click.echo(clonecmd) cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) + + if branch == 'main': + subprocess.run(['git' ,'checkout' ,'tags/2024.01]') + if not cloneproc.returncode == 0: if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None: argstring = f" -e {experiment} -p {platform} -t {target}" @@ -60,7 +68,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): f"\t cylc stop {name}\n" f"\t cylc clean {name}\n" f"\t rm -r ~/cylc-src/{name}" ) - click.echo(stop_report) + sys.exit(stop_report) return 1 else: #if not identified, just print the error From 0ac03223ee8700d54260a01dfc609c90d720c766 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 31 Oct 2024 13:41:10 -0400 Subject: [PATCH 007/111] Update __init__.py with version --- fre/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/__init__.py b/fre/__init__.py index e69de29b..4711e312 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -0,0 +1 @@ +__version__ = '2024.01' From 8a67e19d1f0d4bff84f2cc23ee904dff01aaa0af Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 31 Oct 2024 13:48:45 -0400 Subject: [PATCH 008/111] added version checker #174 --- fre/pp/checkoutScript.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 3eccb561..dc8f08af 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -53,7 +53,9 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) if branch == 'main': - subprocess.run(['git' ,'checkout' ,'tags/2024.01]') + local_version = subprocess.run(['fre' ,'--version']) + if local_version != '2024.01': + subprocess.run(['git' ,'checkout' ,'tags/2024.01']) if not cloneproc.returncode == 0: if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None: From 64d5cc8c1f6e7ef5382195196a853bb98c9a8618 Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Wed, 6 Nov 2024 16:50:13 -0500 Subject: [PATCH 009/111] git fixes --- fre/pp/checkoutScript.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index dc8f08af..55e4870e 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -20,7 +20,7 @@ ############################################# -def _checkoutTemplate(experiment, platform, target, branch='main'): +def _checkoutTemplate(experiment, platform, target): """ Checkout the workflow template files from the repo """ @@ -53,10 +53,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) if branch == 'main': - local_version = subprocess.run(['fre' ,'--version']) - if local_version != '2024.01': - subprocess.run(['git' ,'checkout' ,'tags/2024.01']) - + subprocess.run('git checkout tags/2024.01') if not cloneproc.returncode == 0: if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None: argstring = f" -e {experiment} -p {platform} -t {target}" From f1756d83b6e068c43748acd7f5851fbf67a6684c Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Fri, 8 Nov 2024 14:58:49 -0500 Subject: [PATCH 010/111] wrapper logic reworked #174 --- fre/pp/checkoutScript.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 55e4870e..f3df16eb 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -20,7 +20,7 @@ ############################################# -def _checkoutTemplate(experiment, platform, target): +def _checkoutTemplate(experiment, platform, target, branch='empty'): """ Checkout the workflow template files from the repo """ @@ -34,14 +34,31 @@ def _checkoutTemplate(experiment, platform, target): # Set the name of the directory name = f"{experiment}__{platform}__{target}" - # Get name of local branch and check branch rquest - branch_names = subprocess.run(["git","branch"],capture_output=True, text=True) - local_branch_name = branch_names.stdout.split()[1] - if branch != 'main': - if branch != local_branch_name: - stop_report = ("Error in branch: local branch does not match branch input") - sys.exit(stop_report) - return 1 + # branch and version parameters + default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] + if branch != 'empty': + if os.path.isdir(name): #scenario 4 + os.chdir(name) + name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout + name_path_branch=subprocess.run(["git","branch"],capture_output=True, text=True).stdout.split()[1] + os.chdir(directory) + if default_tag not in name_path_tag and name_path_branch != branch: + stop_report = f"Tag and branch of prexisting directory {diretory}/{name} does not match fre --version or branch requested" + sys.exit(stop_report) + return 1 + else: #scenario 2 + subprocess.run(f'git clone --branch={branch} https://github.com/NOAA-GFDL/fre-cli.git') + else: + if os.path.isdir(name): #scenario 3 + os.chdir(name) + name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split() + os.chdir(directory) + if not default_tag in name_path_tag: + stop_report = f"Tag of prexisting directory {diretory}/{name} does not match fre --version" + sys.exit(stop_report) + return 1 + else: #scenario 1 + subprocess.run(f'git checkout tags/{default_tags}') # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) @@ -52,8 +69,6 @@ def _checkoutTemplate(experiment, platform, target): click.echo(clonecmd) cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) - if branch == 'main': - subprocess.run('git checkout tags/2024.01') if not cloneproc.returncode == 0: if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None: argstring = f" -e {experiment} -p {platform} -t {target}" @@ -78,7 +93,7 @@ def _checkoutTemplate(experiment, platform, target): ############################################# @click.command() -def checkoutTemplate(experiment, platform, target, branch="main"): +def checkoutTemplate(experiment, platform, target, branch = 'empty'): ''' Wrapper script for calling checkoutTemplate - allows the decorated version of the function to be separate from the undecorated version From f161cd7f4e761e09e666d5fe562ee571fbb91b08 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Fri, 8 Nov 2024 16:13:17 -0500 Subject: [PATCH 011/111] Update checkoutScript.py --- fre/pp/checkoutScript.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index f3df16eb..de4fc6a7 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -20,7 +20,7 @@ ############################################# -def _checkoutTemplate(experiment, platform, target, branch='empty'): +def _checkoutTemplate(experiment, platform, target, branch=None): """ Checkout the workflow template files from the repo """ @@ -36,7 +36,7 @@ def _checkoutTemplate(experiment, platform, target, branch='empty'): # branch and version parameters default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] - if branch != 'empty': + if branch == None: if os.path.isdir(name): #scenario 4 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout @@ -93,7 +93,7 @@ def _checkoutTemplate(experiment, platform, target, branch='empty'): ############################################# @click.command() -def checkoutTemplate(experiment, platform, target, branch = 'empty'): +def checkoutTemplate(experiment, platform, target, branch = None): ''' Wrapper script for calling checkoutTemplate - allows the decorated version of the function to be separate from the undecorated version From 1c0a531553bc0acdd031c59afd84cad8ee2d753a Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Fri, 8 Nov 2024 16:31:35 -0500 Subject: [PATCH 012/111] varous small improvemnets to address comments --- fre/pp/checkoutScript.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index de4fc6a7..436e29be 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -39,8 +39,9 @@ def _checkoutTemplate(experiment, platform, target, branch=None): if branch == None: if os.path.isdir(name): #scenario 4 os.chdir(name) - name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout - name_path_branch=subprocess.run(["git","branch"],capture_output=True, text=True).stdout.split()[1] + name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split('*') + name_path_branch = name_path_branch[1].split()[0] + name_path_branch=subprocess.run(["git","branch"],capture_output=True, text=True).stdout os.chdir(directory) if default_tag not in name_path_tag and name_path_branch != branch: stop_report = f"Tag and branch of prexisting directory {diretory}/{name} does not match fre --version or branch requested" @@ -51,14 +52,14 @@ def _checkoutTemplate(experiment, platform, target, branch=None): else: if os.path.isdir(name): #scenario 3 os.chdir(name) - name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split() + name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split()[0] os.chdir(directory) if not default_tag in name_path_tag: stop_report = f"Tag of prexisting directory {diretory}/{name} does not match fre --version" sys.exit(stop_report) return 1 else: #scenario 1 - subprocess.run(f'git checkout tags/{default_tags}') + git clone --branch={default_tag} https://github.com/NOAA-GFDL/fre-cli.git # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) From f91514cd20ea1f86d15a381998834d76a84d2acf Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Fri, 8 Nov 2024 16:45:03 -0500 Subject: [PATCH 013/111] Update checkoutScript.py --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 436e29be..fbdf3159 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -59,7 +59,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 1 - git clone --branch={default_tag} https://github.com/NOAA-GFDL/fre-cli.git + subprocess.run(f'git clone --branch={branch} https://github.com/NOAA-GFDL/fre-cli.git') # Clone the repository with depth=1; check for errors click.echo("cloning experiment into directory " + directory + "/" + name) From 08cb924b99059c4875773c5194efc918118dabe0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 12 Nov 2024 11:19:27 -0500 Subject: [PATCH 014/111] Start yaml dev documentation --- .github/workflows/create_test_conda_env.yml | 40 ++++++++++----------- .github/workflows/publish_conda.yml | 25 +++++++------ docs/setup.rst | 3 ++ 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 25d382a5..6b6de2b1 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -29,26 +29,26 @@ jobs: # called as a module (-m pip) for explicitness $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . - - name: Run pytest in fre-cli environment - run: | - # add spack installed binaries to front of path so that - # conda's netcdf/hdf5 installs don't break compilation tests - export path_save=$PATH - export PATH="/opt/views/view/bin:$PATH" - - # run pytest - pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ - - # restore original path and install genbadge to generate coverage badge based on xml - export PATH="$path_save" - pip install genbadge - genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg - genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg - - - name: Run pylint in fre-cli environment - run: | - # run pylint, ignored modules avoid warnings arising from code internal to those modules - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." +# - name: Run pytest in fre-cli environment +# run: | +# # add spack installed binaries to front of path so that +# # conda's netcdf/hdf5 installs don't break compilation tests +# export path_save=$PATH +# export PATH="/opt/views/view/bin:$PATH" +# +# # run pytest +# pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ +# +# # restore original path and install genbadge to generate coverage badge based on xml +# export PATH="$path_save" +# pip install genbadge +# genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg +# genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg +# +# - name: Run pylint in fre-cli environment +# run: | +# # run pylint, ignored modules avoid warnings arising from code internal to those modules +# pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." - name: Install Sphinx and Build Documentation run: | diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index 5815c07d..798089ac 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -12,16 +12,15 @@ jobs: - name: Checkout Files uses: actions/checkout@v4 with: - submodules: 'recursive' - - - name: Add mkmf to PATH - run: | - echo $PWD/mkmf/bin >> $GITHUB_PATH - - name: Run Conda to Build and Publish - run: | - conda config --append channels conda-forge - conda config --append channels noaa-gfdl - conda install conda-build anaconda-client conda-verify - export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} - conda config --set anaconda_upload yes - conda build . \ No newline at end of file + submodules: 'recursive' +# - name: Add mkmf to PATH +# run: | +# echo $PWD/mkmf/bin >> $GITHUB_PATH +# - name: Run Conda to Build and Publish +# run: | +# conda config --append channels conda-forge +# conda config --append channels noaa-gfdl +# conda install conda-build anaconda-client conda-verify +# export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} +# conda config --set anaconda_upload yes +# conda build . diff --git a/docs/setup.rst b/docs/setup.rst index 304fdf68..087e596e 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -33,3 +33,6 @@ append necessary channels: install ``fre-cli`` into the activated environment from `the GFDL conda channel `_ : ``conda install noaa-gfdl::fre-cli`` +YAML Framework +======================== +In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. From 101c6d51984b5886f37421814fa7603e7711c8a9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 12 Nov 2024 11:34:37 -0500 Subject: [PATCH 015/111] Update setup.rst --- .github/workflows/create_test_conda_env.yml | 2 +- docs/setup.rst | 24 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 6b6de2b1..4b5e3096 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -59,7 +59,7 @@ jobs: - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 - if: ${{ github.ref == 'refs/heads/main' }} +# if: ${{ github.ref == 'refs/heads/main' }} with: publish_branch: gh-pages github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/setup.rst b/docs/setup.rst index 087e596e..581a4fa7 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -35,4 +35,26 @@ install ``fre-cli`` into the activated environment from `the GFDL conda channel YAML Framework ======================== -In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. +In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. + +Model Yaml +----------------- + +The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. + +Compile Yaml +----------------- + +The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. + + +Platform Yaml +_________________ + +The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. + +Post-Processing Yaml +----------------- + +The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. + From 5a3672f876fd8a6559d3ea5054e1378af2e5ab15 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 12 Nov 2024 15:40:48 -0500 Subject: [PATCH 016/111] Move yaml framework to usage --- docs/setup.rst | 25 ------------------------- docs/usage.rst | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/docs/setup.rst b/docs/setup.rst index f2b0ac72..1ba5fdf2 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -32,30 +32,5 @@ and activate it:: conda activate fre -YAML Framework -======================== -In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. - -Model Yaml ------------------ - -The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. - -Compile Yaml ------------------ - -The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. - - -Platform Yaml -_________________ - -The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. - -Post-Processing Yaml ------------------ - -The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. - ======= fre --help diff --git a/docs/usage.rst b/docs/usage.rst index bf5be2a6..46284fb9 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -3,6 +3,27 @@ Usage ============= Using a set of YAML configuration files, ``fre make`` compiles a FMS-based model, and ``fre pp`` postprocesses the history output and runs diagnostic analysis scripts. Please note that model running is not yet supported in FRE 2024; continue to use FRE Bronx frerun. +YAML Framework +======================== +In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. + +Model Yaml +----------------- +The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. + +Compile Yaml +----------------- +The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. + + +Platform Yaml +_________________ +The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. + +Post-Processing Yaml +----------------- +The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. + Build FMS model ======================= .. include:: usage/compile.rst From 5862f67f02a30655ebf75b2f761b9e729c912cb6 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:43:04 -0500 Subject: [PATCH 017/111] Update setup.rst --- docs/setup.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/setup.rst b/docs/setup.rst index 1ba5fdf2..7fd26b14 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -32,5 +32,4 @@ and activate it:: conda activate fre -======= fre --help From fff988ab799e9e3966b8c4e927272ae006f1dbc2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 12 Nov 2024 17:09:09 -0500 Subject: [PATCH 018/111] Creat yaml_framework.rst --- docs/usage.rst | 19 +------------------ docs/usage/yaml_framework.rst | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 docs/usage/yaml_framework.rst diff --git a/docs/usage.rst b/docs/usage.rst index 46284fb9..e49f106b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,24 +5,7 @@ Using a set of YAML configuration files, ``fre make`` compiles a FMS-based model YAML Framework ======================== -In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. - -Model Yaml ------------------ -The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. - -Compile Yaml ------------------ -The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. - - -Platform Yaml -_________________ -The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. - -Post-Processing Yaml ------------------ -The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. +.. include:: usage/yaml_framework.rst Build FMS model ======================= diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst new file mode 100644 index 00000000..a03ebe57 --- /dev/null +++ b/docs/usage/yaml_framework.rst @@ -0,0 +1,18 @@ +In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. + +Model Yaml +~~~~~~~~~ +The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. + +Compile Yaml +~~~~~~~~~ +The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. + + +Platform Yaml +~~~~~~~~~ +The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. + +Post-Processing Yaml +~~~~~~~~~ +The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. From 63a212678196aee357bd12554a19c0c6b624cb00 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 12 Nov 2024 17:22:04 -0500 Subject: [PATCH 019/111] subsubsection format --- docs/usage/yaml_framework.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index a03ebe57..983ed9a8 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,18 +1,17 @@ In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Model Yaml -~~~~~~~~~ +^^^^^^^^^^ The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Compile Yaml -~~~~~~~~~ +^^^^^^^^^^ The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. - Platform Yaml -~~~~~~~~~ +^^^^^^^^^^ The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. Post-Processing Yaml -~~~~~~~~~ +^^^^^^^^^^ The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. From d2307c18676baec576ccbe1a4cee366cf9fc439b Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Wed, 13 Nov 2024 09:23:37 -0500 Subject: [PATCH 020/111] Some more docs --- docs/usage/postprocess.rst | 82 ++++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 7 deletions(-) diff --git a/docs/usage/postprocess.rst b/docs/usage/postprocess.rst index 994a4f1b..4a8cdf9d 100644 --- a/docs/usage/postprocess.rst +++ b/docs/usage/postprocess.rst @@ -52,19 +52,87 @@ the 19790101.nc.tar tarfile might contain:: The name of the history file, while often predictably named, are arbitrary labels within the Diagnostic Manager configuration (diag yamls). Each history file is a CF-standard NetCDF file that can be inspected with common NetCDF tools such as the NCO or CDO tools, or even ``ncdump``. +Required configuration + +1. Set the history directory in your postprocessing yaml:: + + directories: + history: /arch5/am5/am5/am5f7c1r0/c96L65_am5f7c1r0_amip/gfdl.ncrc5-deploy-prod-openmp/history + +2. Set the segment size as an ISO8601 duration (e.g. P1Y is "one year"):: + + postprocess: + settings: + history_segment: P1Y + +3. Set the date range to postprocess as ISO8601 dates:: + + postprocess: + settings: + pp_start: 1979-01-01T0000Z + + pp_stop: 2020-01-01T0000Z + Postprocess components ---------------------- -History files are not immediately convenient for analysis. -On native grid, named in a single namespace. -Desire: regridded, renamed, ts +The history-file namespace is a single layer as shown above. By longtime tradition, FRE postprocessing namespaces are richer, with +a distinction for timeseries, timeaveraged, and static output datasets, and includes frequency and chunk-size in the directory structure. -Timeseries ----------- -Set chunk_a, and chunk_b if desired. +Postprocessed files within a "component" share a horizontal grid; which can be the native grid or regridded to lat/lon. + +Required configuration + +4. Define the atmos and ocean postprocess components:: + + postprocess: + components: + - type: atmos + + sources: [atmos_month, atmos_annual] + - type: ocean + + sources: [ocean_month, ocean_annual] XY-regridding ------------- -blahblah +Commonly, native grid history files are regridded during postprocessing. To regrid to a lat/lon grid, configure your +desired output grid, interpolation method, input grid type, and path to your FMS exchange grid definition. + +Optional configuration (i.e. if xy-regridding is desired) + +5. Regrid the atmos and ocean components to a 1x1 degree grid:: + + directories: + pp_grid_spec: /archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar + + postprocess: + components: + - type: atmos + + sources: [atmos_month, atmos_annual] + + sourceGrid: cubedsphere + + inputRealm: atmos + + xyInterp: [180, 360] + + interpMethod: conserve_order2 + - type: ocean + + sources: [ocean_month, ocean_annual] + + sourceGrid: tripolar + + inputRealm: ocean + + xyInterp: [180, 360] + + interpMethod: conserve_order1 + +Timeseries +---------- +Timeseries output is the most common type of postprocessed output. Climatologies ------------- From 806804e2dacee9a91d36a616aab639fca044ee35 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 11:10:56 -0500 Subject: [PATCH 021/111] Add yaml framework doc --- docs/usage/yaml_framework.rst | 90 ++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 983ed9a8..5a85cf03 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,17 +1,93 @@ -In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. +In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with `[key]: [value]` fields. + +Helpful information and format recommendations for creating yaml files. +1. You can define a block of values as well as individual `[key]: [value]` pairs: +``` +section name: + key: value + key: value +``` + +2. `[key]: [value]` pairs can be made a list by utilizing a `-`: +``` +section name: + - key: value + - key: value +``` + +3. If you want to associate information with a certain listed element, follow this structure: +``` +section name: + - key: value + key: value + - key: value + key: value +``` +Where each `-` indicates a different list. + +4. Yamls also allow for the capability of reusable variables. These variables are defined by: +`&ReusableVariable Value` + +5. Users can apply a reuable variable on a block of values as well. Everything under that section can be included in the reusable variable. +``` +section: &ReusableVariable + - key: value + - key: value +``` + +6. In order to use them as a reference else where in either the same or other yamls, follow: +`*ReusableVariable` + +7. If the reusable variable must be combined with other strings, the `!join` constructor is used. Example: +``` +&version "2024" +&stem !join [FRE/, *version] +``` Model Yaml -^^^^^^^^^^ -The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. +---------- +The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: `fre_properties`, `build`, `shared`, and `experiments`. + +* `fre_properties`: Reusable variables + - list + - value type: string + .. code-block:: + - &variable1 "value1" + - &variable2 "value2" +* `build`: paths to information needed for compilation + - subsections: `compileYaml`, `platformYaml` + - value type: string + .. code-block:: + compileYaml: path the compile yaml in relation to model yaml + platformYaml: path to platforms.yaml in relation to model yaml +* `shared`: shared settings across experiments + - subsections: `directories`, `postprocess` + .. code-block:: + directories: &shared_directories + key: value (string) + postprocess: + settings: &shared_settings + key: value (string) + switches: &shared_switches + key: value (boolean) +* `experiments`: list of post-processing experiments + .. code-block:: + - name: name of post-processing experiment (string) + pp: + - path to post-processing yaml for that experiment in relation to model yaml (string) + analysis: + - path to analysis yaml for that experiment in relation to model yaml (string) + +*Be sure to define directories, settings, and switches as reusable variables as well; they will be "inherited" in the post-processing yamls created.* Compile Yaml -^^^^^^^^^^ +---------- The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. Platform Yaml -^^^^^^^^^^ +---------- The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. Post-Processing Yaml -^^^^^^^^^^ -The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. +---------- +The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. The post-processing yaml can further define more `fre_properties` that may be experiment specific. If there are any repeated reuable variables, the ones set in this yaml will overwrite those set in the model yaml. From fe1078dae425b07666a495cb09d42584bef062ec Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Wed, 13 Nov 2024 11:16:12 -0500 Subject: [PATCH 022/111] reviewing checkoutScript.py --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index fbdf3159..b7df37a9 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -48,7 +48,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 2 - subprocess.run(f'git clone --branch={branch} https://github.com/NOAA-GFDL/fre-cli.git') + subprocess.run(['git', 'clone', f'--branch={branch}', '--single-branch', '--recursive', 'https://github.com/NOAA-GFDL/fre-cli.git', f'{name}']) else: if os.path.isdir(name): #scenario 3 os.chdir(name) From a6d6a824ad8f3bf8a5dbcdfaed93a7de46c187ea Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Wed, 13 Nov 2024 11:23:41 -0500 Subject: [PATCH 023/111] syntax change --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index b7df37a9..8495bebf 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -36,7 +36,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): # branch and version parameters default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] - if branch == None: + if branch is None: if os.path.isdir(name): #scenario 4 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split('*') From 653a51d658ed05ff9677cb93998500fb01017139 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 11:25:49 -0500 Subject: [PATCH 024/111] Update format --- docs/usage/yaml_framework.rst | 106 +++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 41 deletions(-) diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 5a85cf03..3f327c31 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -2,83 +2,107 @@ In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. Helpful information and format recommendations for creating yaml files. 1. You can define a block of values as well as individual `[key]: [value]` pairs: -``` -section name: - key: value - key: value -``` -2. `[key]: [value]` pairs can be made a list by utilizing a `-`: -``` -section name: - - key: value - - key: value -``` +.. code-block:: -3. If you want to associate information with a certain listed element, follow this structure: -``` -section name: - - key: value + section name: key: value - - key: value key: value -``` + +2. `[key]: [value]` pairs can be made a list by utilizing a `-`: + +.. code-block:: + + section name: + - key: value + - key: value + +3. If you want to associate information with a certain listed element, follow this structure: + +.. code-block:: + + section name: + - key: value + key: value + - key: value + key: value + Where each `-` indicates a different list. 4. Yamls also allow for the capability of reusable variables. These variables are defined by: `&ReusableVariable Value` 5. Users can apply a reuable variable on a block of values as well. Everything under that section can be included in the reusable variable. -``` -section: &ReusableVariable - - key: value - - key: value -``` + +.. code-block:: + + section: &ReusableVariable + - key: value + - key: value 6. In order to use them as a reference else where in either the same or other yamls, follow: `*ReusableVariable` -7. If the reusable variable must be combined with other strings, the `!join` constructor is used. Example: -``` -&version "2024" -&stem !join [FRE/, *version] -``` +7. If the reusable variable must be combined with other strings, the `!join` constructor is used. Example: + +.. code-block:: + + &version "2024":: + &stem !join [FRE/, *version] Model Yaml ---------- The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: `fre_properties`, `build`, `shared`, and `experiments`. * `fre_properties`: Reusable variables + - list - value type: string + .. code-block:: + - &variable1 "value1" - &variable2 "value2" + * `build`: paths to information needed for compilation + - subsections: `compileYaml`, `platformYaml` - value type: string + .. code-block:: - compileYaml: path the compile yaml in relation to model yaml - platformYaml: path to platforms.yaml in relation to model yaml + + build: + compileYaml: path the compile yaml in relation to model yaml + platformYaml: path to platforms.yaml in relation to model yaml + * `shared`: shared settings across experiments + - subsections: `directories`, `postprocess` + .. code-block:: - directories: &shared_directories - key: value (string) - postprocess: - settings: &shared_settings + + shared: + directories: &shared_directories key: value (string) - switches: &shared_switches - key: value (boolean) + + postprocess: + settings: &shared_settings + key: value (string) + switches: &shared_switches + key: value (boolean) + + - **Be sure to define directories, settings, and switches as reusable variables as well; they will be "inherited" in the post-processing yamls created.** + * `experiments`: list of post-processing experiments + .. code-block:: - - name: name of post-processing experiment (string) - pp: - - path to post-processing yaml for that experiment in relation to model yaml (string) - analysis: - - path to analysis yaml for that experiment in relation to model yaml (string) -*Be sure to define directories, settings, and switches as reusable variables as well; they will be "inherited" in the post-processing yamls created.* + experiments: + - name: name of post-processing experiment (string) + pp: + - path/to/post-processing/yaml for that experiment in relation to model yaml (string) + analysis: + - path/to/analysis/yaml for that experiment in relation to model yaml (string) Compile Yaml ---------- From a34f5592afeb64d0d350da1918b7b2490d3c08c7 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Wed, 13 Nov 2024 11:27:35 -0500 Subject: [PATCH 025/111] removed outdated aspects --- fre/pp/checkoutScript.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 8495bebf..a10e0767 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -48,7 +48,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 2 - subprocess.run(['git', 'clone', f'--branch={branch}', '--single-branch', '--recursive', 'https://github.com/NOAA-GFDL/fre-cli.git', f'{name}']) + subprocess.run(['git', 'clone', f'--branch={branch}', '--single-branch', '--depth=1', '--recursive', 'https://github.com/NOAA-GFDL/fre-cli.git', f'{name}']) else: if os.path.isdir(name): #scenario 3 os.chdir(name) @@ -61,35 +61,6 @@ def _checkoutTemplate(experiment, platform, target, branch=None): else: #scenario 1 subprocess.run(f'git clone --branch={branch} https://github.com/NOAA-GFDL/fre-cli.git') - # Clone the repository with depth=1; check for errors - click.echo("cloning experiment into directory " + directory + "/" + name) - clonecmd = ( - f"git clone -b {branch} --single-branch --depth=1 --recursive " - f"https://github.com/NOAA-GFDL/fre-workflows.git {name}" ) - preexist_error = f"fatal: destination path '{name}' exists and is not an empty directory." - click.echo(clonecmd) - cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) - - if not cloneproc.returncode == 0: - if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None: - argstring = f" -e {experiment} -p {platform} -t {target}" - stop_report = ( - "Error in checkoutTemplate: the workflow definition specified by -e/-p/-t already" - f" exists at the location ~/cylc-src/{name}!\n" - f"In the future, we will confirm that ~/cylc-src/{name} is usable and will check " - "whether it is up-to-date.\n" - "But for now, if you wish to proceed, you must delete the workflow definition.\n" - "To start over, try:\n" - f"\t cylc stop {name}\n" - f"\t cylc clean {name}\n" - f"\t rm -r ~/cylc-src/{name}" ) - sys.exit(stop_report) - return 1 - else: - #if not identified, just print the error - click.echo(clonecmd) - click.echo(cloneproc.stdout) - return 1 ############################################# From d7ff341a740489a712638cc7ae21e72962e762f2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 11:46:53 -0500 Subject: [PATCH 026/111] Update format --- docs/usage/compile.rst | 2 -- docs/usage/yaml_framework.rst | 26 ++++++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/usage/compile.rst b/docs/usage/compile.rst index 8f9dfbe5..8bb6216b 100644 --- a/docs/usage/compile.rst +++ b/docs/usage/compile.rst @@ -15,5 +15,3 @@ Fremake Canopy Supports: Guide and quickstart to `fre make` subtools: :ref:`fre-make-guide` - -https://github.com/NOAA-GFDL/fre-cli/blob/main/fre/make/README.md diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 3f327c31..e783d75c 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,6 +1,7 @@ In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with `[key]: [value]` fields. Helpful information and format recommendations for creating yaml files. + 1. You can define a block of values as well as individual `[key]: [value]` pairs: .. code-block:: @@ -9,7 +10,7 @@ Helpful information and format recommendations for creating yaml files. key: value key: value -2. `[key]: [value]` pairs can be made a list by utilizing a `-`: +2. `[key]: [value]` pairs can be made a list by utilizing a `dash`: .. code-block:: @@ -27,10 +28,13 @@ Helpful information and format recommendations for creating yaml files. - key: value key: value -Where each `-` indicates a different list. +Where each dash indicates a list. 4. Yamls also allow for the capability of reusable variables. These variables are defined by: -`&ReusableVariable Value` + +.. code-block:: + + &ReusableVariable Value 5. Users can apply a reuable variable on a block of values as well. Everything under that section can be included in the reusable variable. @@ -41,20 +45,22 @@ Where each `-` indicates a different list. - key: value 6. In order to use them as a reference else where in either the same or other yamls, follow: -`*ReusableVariable` + +.. code-block:: + *ReusableVariable 7. If the reusable variable must be combined with other strings, the `!join` constructor is used. Example: -.. code-block:: +.. code-block:: - &version "2024":: + &version "2024" &stem !join [FRE/, *version] Model Yaml ---------- The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: `fre_properties`, `build`, `shared`, and `experiments`. -* `fre_properties`: Reusable variables +* **fre_properties**: Reusable variables - list - value type: string @@ -64,7 +70,7 @@ The model yaml defines reusable variables, shared directories, switches, and pos - &variable1 "value1" - &variable2 "value2" -* `build`: paths to information needed for compilation +* **build**: paths to information needed for compilation - subsections: `compileYaml`, `platformYaml` - value type: string @@ -75,7 +81,7 @@ The model yaml defines reusable variables, shared directories, switches, and pos compileYaml: path the compile yaml in relation to model yaml platformYaml: path to platforms.yaml in relation to model yaml -* `shared`: shared settings across experiments +* **shared**: shared settings across experiments - subsections: `directories`, `postprocess` @@ -93,7 +99,7 @@ The model yaml defines reusable variables, shared directories, switches, and pos - **Be sure to define directories, settings, and switches as reusable variables as well; they will be "inherited" in the post-processing yamls created.** -* `experiments`: list of post-processing experiments +* **experiments**: list of post-processing experiments .. code-block:: From 715ef364b0c1d7bd94a9a335cbca52d7ae33d6ff Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 11:56:35 -0500 Subject: [PATCH 027/111] Update format --- docs/usage/yaml_framework.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index e783d75c..ae466622 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,4 +1,4 @@ -In order to utilize FRE Canopy tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with `[key]: [value]` fields. +In order to utilize FRE 2024.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with `[key]: [value]` fields. Helpful information and format recommendations for creating yaml files. @@ -36,7 +36,7 @@ Where each dash indicates a list. &ReusableVariable Value -5. Users can apply a reuable variable on a block of values as well. Everything under that section can be included in the reusable variable. +5. Users can apply a reusable variable on a block of values as well. For example, everything under that "section" is included in the reusable variable. .. code-block:: @@ -47,6 +47,7 @@ Where each dash indicates a list. 6. In order to use them as a reference else where in either the same or other yamls, follow: .. code-block:: + *ReusableVariable 7. If the reusable variable must be combined with other strings, the `!join` constructor is used. Example: @@ -97,7 +98,8 @@ The model yaml defines reusable variables, shared directories, switches, and pos switches: &shared_switches key: value (boolean) - - **Be sure to define directories, settings, and switches as reusable variables as well; they will be "inherited" in the post-processing yamls created.** + - **Be sure to define directories, settings, and switches as reusable variables as well** + + they will be "inherited" in the post-processing yamls created * **experiments**: list of post-processing experiments From 19a7e2aa2d6ecbf862969ff338be00a82a664c69 Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Wed, 13 Nov 2024 12:38:47 -0500 Subject: [PATCH 028/111] testing --- fre/pp/checkoutScript.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index a10e0767..977f671c 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -36,12 +36,12 @@ def _checkoutTemplate(experiment, platform, target, branch=None): # branch and version parameters default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] - if branch is None: + if branch is not None: if os.path.isdir(name): #scenario 4 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split('*') - name_path_branch = name_path_branch[1].split()[0] name_path_branch=subprocess.run(["git","branch"],capture_output=True, text=True).stdout + name_path_branch = name_path_branch.split('*')[1].split()[0] os.chdir(directory) if default_tag not in name_path_tag and name_path_branch != branch: stop_report = f"Tag and branch of prexisting directory {diretory}/{name} does not match fre --version or branch requested" @@ -59,7 +59,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 1 - subprocess.run(f'git clone --branch={branch} https://github.com/NOAA-GFDL/fre-cli.git') + subprocess.run(['git', 'clone', f'--branch={default_tag}', '--single-branch', '--depth=1', '--recursive', 'https://github.com/NOAA-GFDL/fre-cli.git', f'{name}']) ############################################# From 8f8d17ff68a5e65f0be2266180b28a9d14a6b5c7 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Wed, 13 Nov 2024 12:41:43 -0500 Subject: [PATCH 029/111] spelling change --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 977f671c..2d1c4074 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -44,7 +44,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): name_path_branch = name_path_branch.split('*')[1].split()[0] os.chdir(directory) if default_tag not in name_path_tag and name_path_branch != branch: - stop_report = f"Tag and branch of prexisting directory {diretory}/{name} does not match fre --version or branch requested" + stop_report = f"Tag and branch of prexisting directory {directory}/{name} does not match fre --version or branch requested" sys.exit(stop_report) return 1 else: #scenario 2 From e658eb7a01da95e2831dc94e7f60efa20725f04a Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Wed, 13 Nov 2024 12:57:58 -0500 Subject: [PATCH 030/111] Update checkoutScript.py --- fre/pp/checkoutScript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 2d1c4074..93bf0403 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -48,7 +48,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 2 - subprocess.run(['git', 'clone', f'--branch={branch}', '--single-branch', '--depth=1', '--recursive', 'https://github.com/NOAA-GFDL/fre-cli.git', f'{name}']) + subprocess.run(['git', 'clone', f'--branch={branch}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}']) else: if os.path.isdir(name): #scenario 3 os.chdir(name) @@ -59,7 +59,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 1 - subprocess.run(['git', 'clone', f'--branch={default_tag}', '--single-branch', '--depth=1', '--recursive', 'https://github.com/NOAA-GFDL/fre-cli.git', f'{name}']) + subprocess.run(['git', 'clone', f'--branch={default_tag}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}']) ############################################# From 1ae961237795be48af483553398f0b25098c238c Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 13:55:37 -0500 Subject: [PATCH 031/111] Update compile yaml doc --- docs/usage/compile.rst | 50 +++++++++++++++++++++++++++++------ docs/usage/yaml_framework.rst | 26 +++++++++--------- 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/docs/usage/compile.rst b/docs/usage/compile.rst index 8bb6216b..2c577ff8 100644 --- a/docs/usage/compile.rst +++ b/docs/usage/compile.rst @@ -1,17 +1,51 @@ ``fre make`` can compile a traditional "bare metal" executable or a containerized executable using a set of YAML configuration files. -Through the fre-cli, `fre make` can be used to create and run a checkout script, makefile, and compile a model. +Through the fre-cli, ``fre make`` can be used to create and run a checkout script, makefile, and compile a model. Fremake Canopy Supports: - - multiple targets; use `-t` flag to define each target - - bare-metal build - - container creation - - parallel checkouts for bare-metal build** + - multiple targets; use ``-t`` flag to define each target + - bare-metal build + - container creation + - parallel checkouts for bare-metal build -** **Note: Users will not be able to create containers without access to podman** +**Note: Users will not be able to create containers without access to podman. To get access, submit a helpdesk ticket.** -.. include:: fre_make.rst +Required configuration files: -Guide and quickstart to `fre make` subtools: + - Model Yaml + - Compile Yaml + - Platforms yaml +These yamls are combined and further parsed through the ``fre make`` tools. + +To create the compile yaml, certain fields should be included under "compile". These include ``experiment``, ``container_addlibs``, ``baremetal_linkerflags``, and ``src``. + + - The experiment can be explicitly defined or can be used in conjunction with defined ``fre_properties`` from the model yaml, as seen in the code block below + - ``container_addlibs``: list of strings of packages needed for the model to compile (used to create the link line in the Makefile) + - ``baremetal_linkerflags``: list of strings of linker flags (used to populate the link line in the Makefile + - ``src``: contains information about components needed for model compilation + +.. code-block:: + + compile: :: + experiment: !join [*group_version, "_compile"] + container_addlibs: + baremetal_linkerflags: + src: + +The ``src`` section is used to include component information. This will include: ``component``, ``repo``, ``cpdefs``, ``branch``, ``paths``, ``otherFlags``, and ``makeOverrides``. + +.. code-block:: + + src: + - component: "component name" (string) + repo: "component repository" (string) + branch: "version" (string / list of strings) + paths: (list of strings) + cppdefs: (string) + makeOverrides: 'OPENMP=""' (string) + otherFlags: "Includes needed for model" (string) + doF90Cpp: (boolean) + +Guide and quickstart to ``fre make`` subtools: :ref:`fre-make-guide` diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index ae466622..ff2ee357 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,8 +1,8 @@ -In order to utilize FRE 2024.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with `[key]: [value]` fields. +In order to utilize FRE 2024.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. Helpful information and format recommendations for creating yaml files. -1. You can define a block of values as well as individual `[key]: [value]` pairs: +1. You can define a block of values as well as individual ``[key]: [value]`` pairs: .. code-block:: @@ -10,7 +10,7 @@ Helpful information and format recommendations for creating yaml files. key: value key: value -2. `[key]: [value]` pairs can be made a list by utilizing a `dash`: +2. ``[key]: [value]`` pairs can be made a list by utilizing a ``-``: .. code-block:: @@ -59,11 +59,12 @@ Where each dash indicates a list. Model Yaml ---------- -The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: `fre_properties`, `build`, `shared`, and `experiments`. +The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: ``fre_properties``, ``build``, ``shared``, and ``experiments``. * **fre_properties**: Reusable variables - list + - these values can be extracted from ``fre_properties`` in a group's XML - value type: string .. code-block:: @@ -73,32 +74,33 @@ The model yaml defines reusable variables, shared directories, switches, and pos * **build**: paths to information needed for compilation - - subsections: `compileYaml`, `platformYaml` + - subsections: ``compileYaml``, ``platformYaml`` - value type: string .. code-block:: build: - compileYaml: path the compile yaml in relation to model yaml - platformYaml: path to platforms.yaml in relation to model yaml + compileYaml: "path the compile yaml in relation to model yaml" + platformYaml: "path to platforms.yaml in relation to model yaml" * **shared**: shared settings across experiments - - subsections: `directories`, `postprocess` + - subsections: ``directories``, ``postprocess`` .. code-block:: shared: directories: &shared_directories - key: value (string) + key: "value" (string) postprocess: settings: &shared_settings - key: value (string) + key: "value" (string) switches: &shared_switches key: value (boolean) - **Be sure to define directories, settings, and switches as reusable variables as well** + + they will be "inherited" in the post-processing yamls created * **experiments**: list of post-processing experiments @@ -114,7 +116,7 @@ The model yaml defines reusable variables, shared directories, switches, and pos Compile Yaml ---------- -The compile yaml defines compilation information including copmonent names, repos, branches, necessary flags, and necessary overrides. In order to create the compile yaml, one can refer to compile information defined in an XML. +The compile yaml defines compilation information including component names, repos, branches, necessary flags, and necessary overrides. Platform Yaml ---------- @@ -122,4 +124,4 @@ The platform yaml defines information for both bare-metal and container platform Post-Processing Yaml ---------- -The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. The post-processing yaml can further define more `fre_properties` that may be experiment specific. If there are any repeated reuable variables, the ones set in this yaml will overwrite those set in the model yaml. +The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. The post-processing yaml can further define more ``fre_properties`` that may be experiment specific. If there are any repeated reusable variables, the ones set in this yaml will overwrite those set in the model yaml. From 8cacfeca5050e70c2929b75c9706ec8134f9ba62 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 14:20:01 -0500 Subject: [PATCH 032/111] Update format --- docs/usage/compile.rst | 27 ++++++++++++++------------- docs/usage/yaml_framework.rst | 18 +++++++++--------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/usage/compile.rst b/docs/usage/compile.rst index 2c577ff8..0c999be1 100644 --- a/docs/usage/compile.rst +++ b/docs/usage/compile.rst @@ -18,19 +18,19 @@ Required configuration files: These yamls are combined and further parsed through the ``fre make`` tools. -To create the compile yaml, certain fields should be included under "compile". These include ``experiment``, ``container_addlibs``, ``baremetal_linkerflags``, and ``src``. +To create the compile yaml, reference the compile section on an XML. Certain fields should be included under "compile". These include ``experiment``, ``container_addlibs``, ``baremetal_linkerflags``, and ``src``. - The experiment can be explicitly defined or can be used in conjunction with defined ``fre_properties`` from the model yaml, as seen in the code block below - ``container_addlibs``: list of strings of packages needed for the model to compile (used to create the link line in the Makefile) - ``baremetal_linkerflags``: list of strings of linker flags (used to populate the link line in the Makefile - ``src``: contains information about components needed for model compilation -.. code-block:: +.. code-block:: - compile: :: + compile: experiment: !join [*group_version, "_compile"] - container_addlibs: - baremetal_linkerflags: + container_addlibs: "libraries and packages needed for linking in container" (string) + baremetal_linkerflags: "linker flags of libraries and packages needed" (string) src: The ``src`` section is used to include component information. This will include: ``component``, ``repo``, ``cpdefs``, ``branch``, ``paths``, ``otherFlags``, and ``makeOverrides``. @@ -38,14 +38,15 @@ The ``src`` section is used to include component information. This will include: .. code-block:: src: - - component: "component name" (string) - repo: "component repository" (string) - branch: "version" (string / list of strings) - paths: (list of strings) - cppdefs: (string) - makeOverrides: 'OPENMP=""' (string) - otherFlags: "Includes needed for model" (string) - doF90Cpp: (boolean) + - component: "component name" (string) + requires: ["list of components that this component depends on"] (list of string) + repo: "url of code repository" (string) + branch: "version of code to clone" (string / list of strings) + paths: ["paths in the component to compile"] (list of strings) + cppdefs: "CPPDEFS ot include in compiling componenet (string) + makeOverrides: "overrides openmp target for MOM6" ('OPENMP=""') (string) + otherFlags: "Include flags needed to retrieve other necessary code" (string) + doF90Cpp: True if the preprocessor needs to be run (boolean) Guide and quickstart to ``fre make`` subtools: :ref:`fre-make-guide` diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index ff2ee357..78eb7e68 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -69,8 +69,8 @@ The model yaml defines reusable variables, shared directories, switches, and pos .. code-block:: - - &variable1 "value1" - - &variable2 "value2" + - &variable1 "value1" (srting) + - &variable2 "value2" (string) * **build**: paths to information needed for compilation @@ -80,8 +80,8 @@ The model yaml defines reusable variables, shared directories, switches, and pos .. code-block:: build: - compileYaml: "path the compile yaml in relation to model yaml" - platformYaml: "path to platforms.yaml in relation to model yaml" + compileYaml: "path the compile yaml in relation to model yaml" (string) + platformYaml: "path to platforms.yaml in relation to model yaml" (string) * **shared**: shared settings across experiments @@ -91,13 +91,13 @@ The model yaml defines reusable variables, shared directories, switches, and pos shared: directories: &shared_directories - key: "value" (string) + key: "value" (string) postprocess: settings: &shared_settings - key: "value" (string) + key: "value" (string) switches: &shared_switches - key: value (boolean) + key: value (boolean) - **Be sure to define directories, settings, and switches as reusable variables as well** @@ -108,11 +108,11 @@ The model yaml defines reusable variables, shared directories, switches, and pos .. code-block:: experiments: - - name: name of post-processing experiment (string) + - name: name of post-processing experiment (string) pp: - path/to/post-processing/yaml for that experiment in relation to model yaml (string) analysis: - - path/to/analysis/yaml for that experiment in relation to model yaml (string) + - path/to/analysis/yaml for that experiment in relation to model yaml (string) Compile Yaml ---------- From 2f2c5efb69a3fd0d5a58c670ac3a0aa03c1ca914 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Wed, 13 Nov 2024 15:20:48 -0500 Subject: [PATCH 033/111] inital changes for run-fremake execute option --- fre/make/fremake.py | 6 +++++- fre/make/runFremake.py | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index f39a6be0..21cd7bfb 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -66,12 +66,16 @@ def make_cli(): "--no-parallel-checkout", is_flag = True, help = no_parallel_checkout_opt_help) +@click.option("--execute", + is_flag = True, + default = False, + help = "Use this to run the created compilation script.") @click.option("-v", "--verbose", is_flag = True, help = verbose_opt_help) @click.pass_context -def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): +def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, execute, verbose): """ - Perform all fremake functions to run checkout and compile model""" context.forward(runFremake._fremake_run) diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index ffe2ec96..f57c2af0 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -15,7 +15,7 @@ targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal ) -def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose): ''' run fremake via click''' yml = yamlfile name = yamlfile.split(".")[0] @@ -188,25 +188,27 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") ## Run the dockerfile; build the container - dockerBuild.build(containerBuild,containerRun) + if execute: + dockerBuild.build(containerBuild,containerRun) #freCheckout.cleanup() #buildDockerfile(fremakeYaml,image) if baremetalRun: if __name__ == '__main__': - # Create a multiprocessing Pool - pool = Pool(processes=nparallel) - # process data_inputs iterable with pool - pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) + if execute: + # Create a multiprocessing Pool + pool = Pool(processes=nparallel) + # process data_inputs iterable with pool + pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) @click.command() -def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose): ''' Decorator for calling _fremake_run - allows the decorated version of the function to be separate from the undecorated version ''' - return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose) + return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,execute,verbose) if __name__ == "__main__": fremake_run() From b3ceb308d8076d591fc873eebcb7613934f3eb44 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 13 Nov 2024 16:05:24 -0500 Subject: [PATCH 034/111] Add guide and quickstart in usage --- docs/tool_guides.rst | 156 ---------------------------------- docs/usage/compile.rst | 125 ++++++++++++++++++++++++++- docs/usage/yaml_framework.rst | 2 + 3 files changed, 125 insertions(+), 158 deletions(-) delete mode 100644 docs/tool_guides.rst diff --git a/docs/tool_guides.rst b/docs/tool_guides.rst deleted file mode 100644 index 9a6459db..00000000 --- a/docs/tool_guides.rst +++ /dev/null @@ -1,156 +0,0 @@ -.. NEEDS UPDATING #TODO -============= -Tool Guides -============= - -Guides for the process in which subtools are used with tools. - - -``fre app`` -============ - -``fre catalog`` -============ - -``fre cmor`` -============ - -.. _fre-make-guide: - -``fre make guide`` -============ - -1. Bare-metal Build: - -.. code-block:: - - # Create checkout script - fre make create-checkout -y [model yaml file] -p [platform] -t [target] - - # Create and run checkout script - fre make create-checkout -y [model yaml file] -p [platform] -t [target] --execute - - # Create Makefile - fre make create-makefile -y [model yaml file] -p [platform] -t [target] - - # Creat the compile script - fre make create-compile -y [model yaml file] -p [platform] -t [target] - - # Create and run the compile script - fre make create-compile -y [model yaml file] -p [platform] -t [target] --execute - - # Run all of fremake - fre make run-fremake -y [model yaml file] -p [platform] -t [target] [other options...] - -2. Container Build: - -For the container build, parallel checkouts are not supported, so the `-npc` options must be used for the checkout script. In addition the platform must be a container platform. - -Users will not be able to create containers unless they have podman access on gaea. - -.. code-block:: - - # Create checkout script - fre make create-checkout -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] -npc - - # Create and run checkout script - fre make create-checkout -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] --execute - - # Create Makefile - fre make create-makefile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] - - # Create a Dockerfile - fre make create-dockerfile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] - - # Create and run the Dockerfile - fre make create-dockerfile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] --execute - - -**Quickstart** - -1. Bare-metal Build: - -.. code-block:: - - # Create checkout script - fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod - - # Create and run checkout script - fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod --execute - - # Create Makefile - fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod - - # Create the compile script - fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod - - # Create and run the compile script - fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod --execute - -2. Bare-metal Build Multi-target: - -.. code-block:: - - # Create checkout script - fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug - - # Create and run checkout script - fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute - - # Create Makefile - fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod -t debug - - # Create the compile script - fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug - - # Create and run the compile script - fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute - -3. Container Build: - -In order for the container to build successfully, a `-npc`, or `--no-parallel-checkout` is needed. - -.. code-block:: - - # Create checkout script - fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc - - # Create and run checkout script - fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc --execute - - # Create Makefile - fre make create-makefile -y am5.yaml -p hpcme.2023 -t prod - - # Create Dockerfile - fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod - - # Create and run the Dockerfile - fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod --execute - -4. Run all of fremake: - -.. code-block:: - - # Bare-metal - fre make run-fremake -y am5.yaml -p ncrc5.intel23 -t prod - - # Container - fre make run-fremake -y am5.yaml -p hpcme.2023 -t prod -npc - -``fre pp`` -============ - -``fre yamltools`` -============ - -``fre check`` -============ - -``fre list`` -============ - -``fre run`` -============ - -``fre test`` -============ diff --git a/docs/usage/compile.rst b/docs/usage/compile.rst index 0c999be1..e836e76b 100644 --- a/docs/usage/compile.rst +++ b/docs/usage/compile.rst @@ -18,6 +18,8 @@ Required configuration files: These yamls are combined and further parsed through the ``fre make`` tools. +Compile Yaml +---------- To create the compile yaml, reference the compile section on an XML. Certain fields should be included under "compile". These include ``experiment``, ``container_addlibs``, ``baremetal_linkerflags``, and ``src``. - The experiment can be explicitly defined or can be used in conjunction with defined ``fre_properties`` from the model yaml, as seen in the code block below @@ -48,5 +50,124 @@ The ``src`` section is used to include component information. This will include: otherFlags: "Include flags needed to retrieve other necessary code" (string) doF90Cpp: True if the preprocessor needs to be run (boolean) -Guide and quickstart to ``fre make`` subtools: -:ref:`fre-make-guide` +Guide +---------- +1. Bare-metal Build: + +.. code-block:: + + # Create checkout script + fre make create-checkout -y [model yaml file] -p [platform] -t [target] + + # Create and run checkout script + fre make create-checkout -y [model yaml file] -p [platform] -t [target] --execute + + # Create Makefile + fre make create-makefile -y [model yaml file] -p [platform] -t [target] + + # Creat the compile script + fre make create-compile -y [model yaml file] -p [platform] -t [target] + + # Create and run the compile script + fre make create-compile -y [model yaml file] -p [platform] -t [target] --execute + + # Run all of fremake + fre make run-fremake -y [model yaml file] -p [platform] -t [target] [other options...] + +2. Container Build: + +For the container build, parallel checkouts are not supported, so the `-npc` options must be used for the checkout script. In addition the platform must be a container platform. + +Users will not be able to create containers unless they have podman access on gaea. + +.. code-block:: + + # Create checkout script + fre make create-checkout -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] -npc + + # Create and run checkout script + fre make create-checkout -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] --execute + + # Create Makefile + fre make create-makefile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] + + # Create a Dockerfile + fre make create-dockerfile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] + + # Create and run the Dockerfile + fre make create-dockerfile -y [model yaml file] -p [CONTAINER PLATFORM] -t [target] --execute + +Quickstart +---------- +The quickstart instructions can be used with the am5-compile examples located in the fre-examples repository: https://github.com/NOAA-GFDL/fre-examples/tree/main/AM5/am5-compile + +1. Bare-metal Build: + +.. code-block:: + + # Create checkout script + fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod + + # Create and run checkout script + fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod --execute + + # Create Makefile + fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod + + # Create the compile script + fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod + + # Create and run the compile script + fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod --execute + +2. Bare-metal Build Multi-target: + +.. code-block:: + + # Create checkout script + fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug + + # Create and run checkout script + fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute + + # Create Makefile + fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod -t debug + + # Create the compile script + fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug + + # Create and run the compile script + fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute + +3. Container Build: + +In order for the container to build successfully, a `-npc`, or `--no-parallel-checkout` is needed. + +.. code-block:: + + # Create checkout script + fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc + + # Create and run checkout script + fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc --execute + + # Create Makefile + fre make create-makefile -y am5.yaml -p hpcme.2023 -t prod + + # Create Dockerfile + fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod + + # Create and run the Dockerfile + fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod --execute + +4. Run all of fremake: + +Currently, run-fremake kicks off the compilation automatically; no ``--execute`` option needed. + +.. code-block:: + + # Bare-metal + fre make run-fremake -y am5.yaml -p ncrc5.intel23 -t prod + + # Container + fre make run-fremake -y am5.yaml -p hpcme.2023 -t prod -npc diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 78eb7e68..98b3bc0e 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -1,5 +1,7 @@ In order to utilize FRE 2024.01 tools, a distrubuted YAML structure is required. This framework includes a main model yaml, a compile yaml, a platforms yaml, and post-processing yamls. Throughout the compilation and post-processing steps, combined yamls that will be parsed for information are created. Yamls follow a dictionary-like structure with ``[key]: [value]`` fields. +Yaml Formatting +---------- Helpful information and format recommendations for creating yaml files. 1. You can define a block of values as well as individual ``[key]: [value]`` pairs: From 36b03f55d90cb0ecd2f1b5df695c3ce7c24d4475 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 14 Nov 2024 10:08:47 -0500 Subject: [PATCH 035/111] Update docs --- docs/usage/compile.rst | 19 ++++++------ docs/usage/yaml_framework.rst | 55 ++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/docs/usage/compile.rst b/docs/usage/compile.rst index e836e76b..f84ae36f 100644 --- a/docs/usage/compile.rst +++ b/docs/usage/compile.rst @@ -40,15 +40,16 @@ The ``src`` section is used to include component information. This will include: .. code-block:: src: - - component: "component name" (string) - requires: ["list of components that this component depends on"] (list of string) - repo: "url of code repository" (string) - branch: "version of code to clone" (string / list of strings) - paths: ["paths in the component to compile"] (list of strings) - cppdefs: "CPPDEFS ot include in compiling componenet (string) - makeOverrides: "overrides openmp target for MOM6" ('OPENMP=""') (string) - otherFlags: "Include flags needed to retrieve other necessary code" (string) - doF90Cpp: True if the preprocessor needs to be run (boolean) + - component: "component name" (string) + requires: ["list of components that this component depends on"] (list of string) + repo: "url of code repository" (string) + branch: "version of code to clone" (string / list of strings) + paths: ["paths in the component to compile"] (list of strings) + cppdefs: "CPPDEFS ot include in compiling componenet (string) + makeOverrides: "overrides openmp target for MOM6" ('OPENMP=""') (string) + otherFlags: "Include flags needed to retrieve other necessary code" (string) + doF90Cpp: True if the preprocessor needs to be run (boolean) + additionalInstructions: additional instructions to run after checkout (string) Guide ---------- diff --git a/docs/usage/yaml_framework.rst b/docs/usage/yaml_framework.rst index 98b3bc0e..9acc6742 100644 --- a/docs/usage/yaml_framework.rst +++ b/docs/usage/yaml_framework.rst @@ -38,40 +38,43 @@ Where each dash indicates a list. &ReusableVariable Value -5. Users can apply a reusable variable on a block of values as well. For example, everything under that "section" is included in the reusable variable. +5. Users can apply a reusable variable on a block of values. For example, everything under "section" is associated with the reusable variable: .. code-block:: section: &ReusableVariable - key: value + key: value - key: value -6. In order to use them as a reference else where in either the same or other yamls, follow: +6. In order to use them as a reference else where in either the same or other yamls, use ``*``: .. code-block:: *ReusableVariable -7. If the reusable variable must be combined with other strings, the `!join` constructor is used. Example: +7. If the reusable variable must be combined with other strings, the **`!join`** constructor is used. Example: .. code-block:: - &version "2024" + &version "2024.01" &stem !join [FRE/, *version] +In this example, the reuasble variable ``stem`` will be parsed as ``FRE/2024.01``. + Model Yaml ---------- -The model yaml defines reusable variables, shared directories, switches, and post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: ``fre_properties``, ``build``, ``shared``, and ``experiments``. +The model yaml defines reusable variables, shared directories, switches, post-processing settings, and paths to compile and post-processing yamls. Required fields in the model yaml include: ``fre_properties``, ``build``, ``shared``, and ``experiments``. * **fre_properties**: Reusable variables - - list - - these values can be extracted from ``fre_properties`` in a group's XML + - list of variables + - these values can be extracted from ``fre_properties`` in a group's XML, if available - value type: string .. code-block:: - - &variable1 "value1" (srting) + - &variable1 "value1" (string) - &variable2 "value2" (string) * **build**: paths to information needed for compilation @@ -99,14 +102,16 @@ The model yaml defines reusable variables, shared directories, switches, and pos settings: &shared_settings key: "value" (string) switches: &shared_switches - key: value (boolean) + key: True/False (boolean) - - **Be sure to define directories, settings, and switches as reusable variables as well** + * **Be sure to define directories, settings, and switches as reusable variables as well** + they will be "inherited" in the post-processing yamls created * **experiments**: list of post-processing experiments + - subsections: ``name``, ``pp``, ``analysis`` + .. code-block:: experiments: @@ -118,12 +123,34 @@ The model yaml defines reusable variables, shared directories, switches, and pos Compile Yaml ---------- -The compile yaml defines compilation information including component names, repos, branches, necessary flags, and necessary overrides. +The compile yaml defines compilation information including component names, repos, branches, necessary flags, and necessary overrides. This is discussed more in the "Build FMS Model" section. -Platform Yaml +Platforms Yaml ---------- -The platform yaml defines information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. +The platform yaml contains user defined information for both bare-metal and container platforms. Information includes the platform name, the compiler used, necessary modules to load, an mk template, fc, cc, container build, and container run. This yaml file is not model specific. + + .. code-block:: + + platforms: + - name: the platform name + compiler: the compiler you are using + modulesInit: ["array of commands that are needed to load modules." , "each command must end with a newline character"] + modules: [array of modules to load including compiler] + fc: the name of the fortran compiler + cc: the name of the C compiler + mkTemplate: The location of the mkmf make template + modelRoot: The root directory of the model (where src, exec, experiments will go) + - container platform: container platform name + compiler: compiler you are using + RUNenv: Commands needed at the beginning of a RUN in dockerfile + modelRoot: The root directory of the model (where src, exec, experiments will go) INSIDE of the container (/apps) + fc: name of fortan compiler + cc: name of C compiler + container: True if this is a container platform + containerBuild: "podman" - the container build program + containerRun: "apptainer" - the container run program + Post-Processing Yaml ---------- -The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. The post-processing yaml can further define more ``fre_properties`` that may be experiment specific. If there are any repeated reusable variables, the ones set in this yaml will overwrite those set in the model yaml. +The post-processing yamls include information specific to experiments, such as directories to data and other scripts used, switches, and component information. The post-processing yaml can further define more ``fre_properties`` that may be experiment specific. If there are any repeated reusable variables, the ones set in this yaml will overwrite those set in the model yaml. This is discussed further in the "Postprocess FMS History Output" section. From 955ab1ed836fed4ec1b449bb52ee71bc6a8812b2 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 14 Nov 2024 10:24:59 -0500 Subject: [PATCH 036/111] fix some things --- docs/usage/compile.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/usage/compile.rst b/docs/usage/compile.rst index f84ae36f..4d66c637 100644 --- a/docs/usage/compile.rst +++ b/docs/usage/compile.rst @@ -3,10 +3,13 @@ Through the fre-cli, ``fre make`` can be used to create and run a checkout script, makefile, and compile a model. Fremake Canopy Supports: - - multiple targets; use ``-t`` flag to define each target + - multiple target use; ``-t`` flag to define each target (for multiple platform-target combinations) - bare-metal build - container creation - parallel checkouts for bare-metal build + - parallel model builds + - one yaml format + - additional library support if needed **Note: Users will not be able to create containers without access to podman. To get access, submit a helpdesk ticket.** From 0d9e41637462a131547facb86cc0e299c7a394d3 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 14 Nov 2024 10:28:42 -0500 Subject: [PATCH 037/111] uncomment things --- .github/workflows/create_test_conda_env.yml | 42 ++++++++++----------- .github/workflows/publish_conda.yml | 22 +++++------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 4b5e3096..25d382a5 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -29,26 +29,26 @@ jobs: # called as a module (-m pip) for explicitness $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . -# - name: Run pytest in fre-cli environment -# run: | -# # add spack installed binaries to front of path so that -# # conda's netcdf/hdf5 installs don't break compilation tests -# export path_save=$PATH -# export PATH="/opt/views/view/bin:$PATH" -# -# # run pytest -# pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ -# -# # restore original path and install genbadge to generate coverage badge based on xml -# export PATH="$path_save" -# pip install genbadge -# genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg -# genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg -# -# - name: Run pylint in fre-cli environment -# run: | -# # run pylint, ignored modules avoid warnings arising from code internal to those modules -# pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." + - name: Run pytest in fre-cli environment + run: | + # add spack installed binaries to front of path so that + # conda's netcdf/hdf5 installs don't break compilation tests + export path_save=$PATH + export PATH="/opt/views/view/bin:$PATH" + + # run pytest + pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ + + # restore original path and install genbadge to generate coverage badge based on xml + export PATH="$path_save" + pip install genbadge + genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg + genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg + + - name: Run pylint in fre-cli environment + run: | + # run pylint, ignored modules avoid warnings arising from code internal to those modules + pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." - name: Install Sphinx and Build Documentation run: | @@ -59,7 +59,7 @@ jobs: - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 -# if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} with: publish_branch: gh-pages github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index 798089ac..cc24012f 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -13,14 +13,14 @@ jobs: uses: actions/checkout@v4 with: submodules: 'recursive' -# - name: Add mkmf to PATH -# run: | -# echo $PWD/mkmf/bin >> $GITHUB_PATH -# - name: Run Conda to Build and Publish -# run: | -# conda config --append channels conda-forge -# conda config --append channels noaa-gfdl -# conda install conda-build anaconda-client conda-verify -# export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} -# conda config --set anaconda_upload yes -# conda build . + - name: Add mkmf to PATH + run: | + echo $PWD/mkmf/bin >> $GITHUB_PATH + - name: Run Conda to Build and Publish + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build anaconda-client conda-verify + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda config --set anaconda_upload yes + conda build . From d0c5ddd8f7ce8acf7afb58bfa672af2aed52c7a0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 14 Nov 2024 10:29:31 -0500 Subject: [PATCH 038/111] remove space --- .github/workflows/publish_conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index cc24012f..26b1825a 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout Files uses: actions/checkout@v4 with: - submodules: 'recursive' + submodules: 'recursive' - name: Add mkmf to PATH run: | echo $PWD/mkmf/bin >> $GITHUB_PATH From 96f79efa4ecd15d93f2108d1699a5bade0843077 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 14 Nov 2024 13:21:14 -0500 Subject: [PATCH 039/111] Update checkoutScript.py --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 93bf0403..491ca7c0 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -59,7 +59,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 else: #scenario 1 - subprocess.run(['git', 'clone', f'--branch={default_tag}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}']) + subprocess.run(['git', 'clone', '-b',f'{default_tag}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}']) ############################################# From 744c3a5519e45982f72afdc4df30e4fa5c168ce8 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 14 Nov 2024 14:21:48 -0500 Subject: [PATCH 040/111] documentation and version changes --- fre/pp/checkoutScript.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 491ca7c0..c0877331 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -36,8 +36,11 @@ def _checkoutTemplate(experiment, platform, target, branch=None): # branch and version parameters default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] + if default_tag == '2024.1': #hard coded solution to current discrepencies with fre --version + default_tag = '2024.01' + if branch is not None: - if os.path.isdir(name): #scenario 4 + if os.path.isdir(name): #scenario 4 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split('*') name_path_branch=subprocess.run(["git","branch"],capture_output=True, text=True).stdout @@ -47,9 +50,14 @@ def _checkoutTemplate(experiment, platform, target, branch=None): stop_report = f"Tag and branch of prexisting directory {directory}/{name} does not match fre --version or branch requested" sys.exit(stop_report) return 1 + print('directory exists, and branch requested matches branch in use') + else: #scenario 2 - subprocess.run(['git', 'clone', f'--branch={branch}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}']) + clone_output = subprocess.run(['git', 'clone', f'--branch={branch}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) + print('output of fre pp checkouts git clone command is as follows:',clone_output) else: + print('the default tag for directory ',directory,'/',name, ' is ', default_tag) + if os.path.isdir(name): #scenario 3 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split()[0] @@ -58,9 +66,11 @@ def _checkoutTemplate(experiment, platform, target, branch=None): stop_report = f"Tag of prexisting directory {diretory}/{name} does not match fre --version" sys.exit(stop_report) return 1 + print('directory exists, and its branch matches default tag') + else: #scenario 1 - subprocess.run(['git', 'clone', '-b',f'{default_tag}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}']) - + clone_output = subprocess.run(['git', 'clone', '-b',f'{default_tag}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) + print('output of fre pp checkouts git clone command is as follows:',clone_output) ############################################# From 0de08a832ed4c778803b1cf2398a15b48a5d3468 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 14 Nov 2024 14:32:20 -0500 Subject: [PATCH 041/111] order change --- fre/pp/checkoutScript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index c0877331..6632ed6a 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -38,7 +38,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] if default_tag == '2024.1': #hard coded solution to current discrepencies with fre --version default_tag = '2024.01' - + print('the default tag for directory ',directory,'/',name, ' is ', default_tag) if branch is not None: if os.path.isdir(name): #scenario 4 os.chdir(name) @@ -56,7 +56,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): clone_output = subprocess.run(['git', 'clone', f'--branch={branch}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) print('output of fre pp checkouts git clone command is as follows:',clone_output) else: - print('the default tag for directory ',directory,'/',name, ' is ', default_tag) + if os.path.isdir(name): #scenario 3 os.chdir(name) From 108a12340deb66da1370d0ad6d971d5eda754f69 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Thu, 14 Nov 2024 14:41:56 -0500 Subject: [PATCH 042/111] sanity check of recursive --- fre/pp/checkoutScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 6632ed6a..8d5ad7dd 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -69,7 +69,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): print('directory exists, and its branch matches default tag') else: #scenario 1 - clone_output = subprocess.run(['git', 'clone', '-b',f'{default_tag}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) + clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) print('output of fre pp checkouts git clone command is as follows:',clone_output) ############################################# From fe5fc2c21ede6468e7256c84af2a4e2b7c50dd8f Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Thu, 14 Nov 2024 14:54:55 -0500 Subject: [PATCH 043/111] more print statements --- fre/pp/checkoutScript.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 8d5ad7dd..3ded8384 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -16,9 +16,6 @@ ############################################# -package_dir = os.path.dirname(os.path.abspath(__file__)) - -############################################# def _checkoutTemplate(experiment, platform, target, branch=None): """ @@ -39,7 +36,8 @@ def _checkoutTemplate(experiment, platform, target, branch=None): if default_tag == '2024.1': #hard coded solution to current discrepencies with fre --version default_tag = '2024.01' print('the default tag for directory ',directory,'/',name, ' is ', default_tag) - if branch is not None: + if branch != None: + print(branch) if os.path.isdir(name): #scenario 4 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split('*') @@ -50,11 +48,11 @@ def _checkoutTemplate(experiment, platform, target, branch=None): stop_report = f"Tag and branch of prexisting directory {directory}/{name} does not match fre --version or branch requested" sys.exit(stop_report) return 1 - print('directory exists, and branch requested matches branch in use') + print('scenario 4: directory exists, and branch requested matches branch in use') else: #scenario 2 - clone_output = subprocess.run(['git', 'clone', f'--branch={branch}', '--recursive', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) - print('output of fre pp checkouts git clone command is as follows:',clone_output) + clone_output = subprocess.run(['git', 'clone','--recursive', f'--branch={branch}', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) + print('scenario 2: output of fre pp checkouts git clone command is as follows:',clone_output) else: @@ -66,11 +64,11 @@ def _checkoutTemplate(experiment, platform, target, branch=None): stop_report = f"Tag of prexisting directory {diretory}/{name} does not match fre --version" sys.exit(stop_report) return 1 - print('directory exists, and its branch matches default tag') + print('scenario 3: directory exists, and its branch matches default tag') else: #scenario 1 clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) - print('output of fre pp checkouts git clone command is as follows:',clone_output) + print('scenario 1: output of fre pp checkouts git clone command is as follows:',clone_output) ############################################# From 4b3cde4ad219051fd36c14cfd088f9fa0209953e Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Fri, 15 Nov 2024 11:43:54 -0500 Subject: [PATCH 044/111] update esm4p2 and null model yamls for c5 update --- fre/make/tests/ESM4_example/compile.yaml | 2 +- fre/make/tests/ESM4_example/platforms.yaml | 10 +--------- fre/make/tests/null_example/platforms.yaml | 10 +--------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/fre/make/tests/ESM4_example/compile.yaml b/fre/make/tests/ESM4_example/compile.yaml index ee51658d..c4e63597 100644 --- a/fre/make/tests/ESM4_example/compile.yaml +++ b/fre/make/tests/ESM4_example/compile.yaml @@ -5,7 +5,7 @@ compile: src: - component: "FMS" repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-Duse_libMPI -Duse_netCDF -DMAXFIELDMETHODS_=500" + cppdefs: "-Duse_libMPI -Duse_netCDF -DMAXFIELDMETHODS_=500 -DHAVE_GETTID" branch: *FMS_GIT_TAG - component: "atmos_phys" requires: ["FMS"] diff --git a/fre/make/tests/ESM4_example/platforms.yaml b/fre/make/tests/ESM4_example/platforms.yaml index 14d4dfff..2114f9da 100644 --- a/fre/make/tests/ESM4_example/platforms.yaml +++ b/fre/make/tests/ESM4_example/platforms.yaml @@ -1,16 +1,8 @@ platforms: - - name: ncrc5.intel - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL,".mk"] - modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] + modules: [ !join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] fc: ftn cc: cc mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] diff --git a/fre/make/tests/null_example/platforms.yaml b/fre/make/tests/null_example/platforms.yaml index fdfa9d4f..d2a3c97f 100644 --- a/fre/make/tests/null_example/platforms.yaml +++ b/fre/make/tests/null_example/platforms.yaml @@ -1,16 +1,8 @@ platforms: - - name: ncrc5.intel - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] - modelRoot: ${HOME}/fremake_canopy/test - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] + modules: [!join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] fc: ftn cc: cc mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] From 2d28f49a7aeb211763ecbfe963370c36d9acd700 Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Fri, 15 Nov 2024 14:18:58 -0500 Subject: [PATCH 045/111] Update frepp.py so branch does not default to main --- fre/pp/frepp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index b3456e31..bc83ebd3 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -114,9 +114,9 @@ def configure_yaml(context,yamlfile,experiment,platform,target): required=True) @click.option("-b", "--branch", show_default=True, - default="main", type=str, + default=None, type=NoneType, help="Name of fre2/workflows/postproc branch to clone; " \ - "defaults to 'main'. Not intended for production use, " \ + "defaults to None. Not intended for production use, " \ "but needed for branch testing." ) @click.pass_context def checkout(context, experiment, platform, target, branch='main'): From 3eee452fda4d8273f21255da11c399083c57f0fb Mon Sep 17 00:00:00 2001 From: kiihne-noaa Date: Fri, 15 Nov 2024 14:22:31 -0500 Subject: [PATCH 046/111] Update checkoutScript.py documentation --- fre/pp/checkoutScript.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 3ded8384..b665ccb4 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -1,8 +1,10 @@ #!/usr/bin/env python -# Author: Bennett Chang -# Description: - +# Author: Avery Kiihne +''' +Description: Checkout script which accounts for 4 different scenarios: 1. branch not given, folder does not exist, +2. branch given, folder does not exist, 3. branch not given, folder exists, 4. branch given and folder exists +''' import os import sys import subprocess From ec650ba63f2b4bdc2dcb95093bed890909780617 Mon Sep 17 00:00:00 2001 From: Avery Kiihne Date: Fri, 15 Nov 2024 14:39:53 -0500 Subject: [PATCH 047/111] attempt to change default away from main --- fre/pp/frepp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index bc83ebd3..fd99e74b 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -114,12 +114,12 @@ def configure_yaml(context,yamlfile,experiment,platform,target): required=True) @click.option("-b", "--branch", show_default=True, - default=None, type=NoneType, + default=None, type='NoneType', help="Name of fre2/workflows/postproc branch to clone; " \ "defaults to None. Not intended for production use, " \ "but needed for branch testing." ) @click.pass_context -def checkout(context, experiment, platform, target, branch='main'): +def checkout(context, experiment, platform, target, branch=None): # pylint: disable=unused-argument """ - Execute fre pp checkout """ context.forward(checkoutTemplate) @@ -198,12 +198,12 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo required=True) @click.option("-b", "--branch", show_default=True, - default="main", type=str, + default=None, type='NoneType', help="Name of fre2/workflows/postproc branch to clone; " \ - "defaults to 'main'. Not intended for production use, " \ + "defaults to None. Not intended for production use, " \ "but needed for branch testing." ) @click.pass_context -def wrapper(context, experiment, platform, target, config_file, branch='main'): +def wrapper(context, experiment, platform, target, config_file, branch=None): # pylint: disable=unused-argument """ - Execute fre pp steps in order """ context.forward(runFre2pp) From c29e3116c1061c2c7ac346677a4530229c24d08b Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Fri, 15 Nov 2024 15:51:47 -0500 Subject: [PATCH 048/111] add a function to generate a container build script if --execute not given --- fre/make/fremake.py | 3 +- fre/make/gfdlfremake/buildBaremetal.py | 6 ++-- fre/make/gfdlfremake/buildDocker.py | 49 ++++++++++++++++++-------- fre/make/runFremake.py | 13 +++++-- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 21cd7bfb..6dfb9a34 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -66,7 +66,8 @@ def make_cli(): "--no-parallel-checkout", is_flag = True, help = no_parallel_checkout_opt_help) -@click.option("--execute", +@click.option("-e", + "--execute", is_flag = True, default = False, help = "Use this to run the created compilation script.") diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index fdb4e2d8..a4ab216b 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -10,14 +10,14 @@ def fremake_parallel(fremakeBuildList): """ Brief: Called for parallel execution purposes. Runs the builds. - Param: + Param: - fremakeBuildList : fremakeBuild object list passes by pool.map """ fremakeBuildList.run() class buildBaremetal(): """ - Brief: Creates the build script to compile the model + Brief: Creates the build script to compile the model Param: - self : The buildScript object - exp : The experiment name @@ -50,6 +50,8 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jo if self.modules != "": self.setup.extend(modulesInit) #extend - this is a list self.setup.append("module load "+self.modules+" \n") # Append -this is a single string + # make sure our modules are loaded correctly + self.setup.append("test $? != 0 && echo \"**********Modules failed to load, check platform yaml\" && exit 1\n") ## Create the build directory os.system("mkdir -p "+self.bld) diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index 6d33d0d2..cb48b48c 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -9,12 +9,12 @@ class container(): """ Brief: Opens the Dockerfile for writing - Param: + Param: - self : The dockerfile object - base : The docker base image to start from - libs : Additional libraries defined by user - exp : The experiment name - - RUNenv : The commands that have to be run at + - RUNenv : The commands that have to be run at the beginning of a RUN in the dockerfile to set up the environment """ @@ -58,7 +58,7 @@ def __init__(self,base,exp,libs,RUNenv,target): def writeDockerfileCheckout(self, cScriptName, cOnDisk): """ Brief: writes to the checkout part of the Dockerfile and sets up the compile - Param: + Param: - self : The dockerfile object - cScriptName : The name of the checkout script in the container - cOnDisk : The relative path to the checkout script on disk @@ -74,7 +74,7 @@ def writeDockerfileCheckout(self, cScriptName, cOnDisk): def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath): """ Brief: Copies the Makefile into the bldDir in the dockerfile - Param: + Param: - self : The dockerfile object - makefileOnDiskPath : The path to Makefile on the local disk - linklineonDiskPath : The path to the link line script on the local disk @@ -98,8 +98,8 @@ def writeDockerfileMakefile(self, makefileOnDiskPath, linklineonDiskPath): def writeDockerfileMkmf(self, c): """ - Brief: Adds components to the build part of the Dockerfile - Param: + Brief: Adds components to the build part of the Dockerfile + Param: - self : The dockerfile object - c : Component from the compile yaml """ @@ -141,14 +141,14 @@ def writeDockerfileMkmf(self, c): def writeRunscript(self,RUNenv,containerRun,runOnDisk): """ Brief: Writes a runscript to set up spack loads/environment - in order to run the executable in the container; + in order to run the executable in the container; runscript copied into container - Param: + Param: - self : The dockerfile object - - RUNEnv : The commands that have to be run at + - RUNEnv : The commands that have to be run at the beginning of a RUN in the dockerfile - - containerRun : The container platform used with `exec` - to run the container; apptainer + - containerRun : The container platform used with `exec` + to run the container; apptainer or singularity used - runOnDisk : The path to the run script on the local disk """ @@ -187,14 +187,35 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk): def build(self,containerBuild,containerRun): """ Brief: Builds the container image for the model - Param: + Param: - self : The dockerfile object - - containerBuild : The tool used to build the container; + - containerBuild : The tool used to build the container; docker or podman used - - containerRun : The container platform used with `exec` to + - containerRun : The container platform used with `exec` to run the container; apptainer or singularity used """ os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()) os.system("rm -f "+self.e+".tar "+self.e+".sif") os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()) os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar") + + def createBuildScript(self,containerBuild,containerRun): + """ + Brief: Writes out the build commands for the created dockerfile in a script, uses the same commands as the routine above + Param: + - self : The dockerfile object + - containerBuild : The tool used to build the container; + docker or podman used + - containerRun : The container platform used with `exec` to + run the container; apptainer or singularity used + """ + self.userScript = ["#!/bin/bash\n"] + self.userScript.append(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()+"\n") + self.userScript.append("rm -f "+self.e+".tar "+self.e+".sif\n") + self.userScript.append(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()+"\n") + self.userScript.append(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar\n") + self.userScriptFile = open("createContainer.sh","w") + self.userScriptFile.writelines(self.userScript) + os.chmod("createContainer.sh", 0o744) + self.userScriptPath = os.getcwd()+"/createContainer.sh" + diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index f57c2af0..0e4cfecd 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -142,8 +142,11 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec fremakeBuild.writeBuildComponents(c) fremakeBuild.writeScript() fremakeBuildList.append(fremakeBuild) - ## Run the build - fremakeBuild.run() + ## Run the build if --execute option given, otherwise print out compile script path + if execute: + fremakeBuild.run() + else: + print("Compile script created at "+ bldDir+"/compile.sh\n\n") else: ###################### container stuff below ####################################### ## Run the checkout script @@ -187,9 +190,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") - ## Run the dockerfile; build the container + ## Run the dockerfile and create the container if execute option is given + ## otherwise create a build script and print out its path if execute: dockerBuild.build(containerBuild,containerRun) + else: + dockerBuild.createBuildScript(containerBuild, containerRun) + print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") #freCheckout.cleanup() #buildDockerfile(fremakeYaml,image) From 26956ba82aa1ccb9bc234c3a4a82ffb4ac39dc9a Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Fri, 15 Nov 2024 15:55:24 -0500 Subject: [PATCH 049/111] delete yamls that are in the fre-examples repo (and not used here) --- fre/make/tests/ESM4_example/compile.yaml | 124 ------------------- fre/make/tests/ESM4_example/esm4.yaml | 86 ------------- fre/make/tests/ESM4_example/platforms.yaml | 18 --- fre/make/tests/SHiELD_example/SHiELD.yaml | 10 -- fre/make/tests/SHiELD_example/compile.yaml | 38 ------ fre/make/tests/SHiELD_example/platforms.yaml | 26 ---- fre/make/tests/null_example/compile.yaml | 2 +- 7 files changed, 1 insertion(+), 303 deletions(-) delete mode 100644 fre/make/tests/ESM4_example/compile.yaml delete mode 100644 fre/make/tests/ESM4_example/esm4.yaml delete mode 100644 fre/make/tests/ESM4_example/platforms.yaml delete mode 100644 fre/make/tests/SHiELD_example/SHiELD.yaml delete mode 100644 fre/make/tests/SHiELD_example/compile.yaml delete mode 100644 fre/make/tests/SHiELD_example/platforms.yaml diff --git a/fre/make/tests/ESM4_example/compile.yaml b/fre/make/tests/ESM4_example/compile.yaml deleted file mode 100644 index c4e63597..00000000 --- a/fre/make/tests/ESM4_example/compile.yaml +++ /dev/null @@ -1,124 +0,0 @@ -compile: - experiment: "esm4" - container_addlibs: - baremetal_linkerflags: - src: - - component: "FMS" - repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-Duse_libMPI -Duse_netCDF -DMAXFIELDMETHODS_=500 -DHAVE_GETTID" - branch: *FMS_GIT_TAG - - component: "atmos_phys" - requires: ["FMS"] - repo: "https://gitlab.gfdl.noaa.gov/FMS/atmos_phys.git" - branch: *ATM_PHYS_GIT_TAG - otherFlags: *FMSincludes - - component: "atmos_dyn" - requires: ["FMS", "atmos_phys"] - repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" - cppdefs: "-DSPMD -DCLIMATE_NUDGE -DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - branch: *ATM_FV3_GIT_TAG - paths: [ "atmos_dyn/driver/GFDL", - "atmos_dyn/model", - "atmos_dyn/model_nh_null", - "atmos_dyn/GFDL_tools", - "atmos_dyn/driver/SHiELD/cloud_diagnosis.F90", - "atmos_dyn/driver/SHiELD/gfdl_cloud_microphys.F90", - "atmos_dyn/tools" ] - - component: "atmos_drivers" - requires: ["FMS", "atmos_phys", "atmos_dyn"] - repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" - cppdefs: "-DSPMD -DCLIMATE_NUDGE" - branch: *ATM_DRV_GIT_TAG - otherFlags: *FMSincludes - paths: ["atmos_drivers/coupled"] - - component: "lm4p" - requires: ["FMS"] - repo: "https://gitlab.gfdl.noaa.gov/FMS/lm4p.git" - branch: *LAND_GIT_TAG - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "mom6" - requires: ["FMS"] - repo: ["https://github.com/NOAA-GFDL/MOM6-examples.git", - "https://github.com/NOAA-GFDL/ocean_BGC.git" ] - branch: [ "dev/gfdl", *OCEAN_BGC_GIT_TAG ] # cant use property for mom6 since its a commit hash instead of a branch - otherFlags: !join [ *FMSincludes, " ", *MOMincludes ] - additionalInstructions: | - pushd mom6/MOM6-examples - git checkout 40e3937 # this is just the value of MOM6_GIT_TAG property, can't seem to use variable - popd - # this step might be covered by initial checkout since the default is recursive - git submodule update --recursive --init mom6/MOM6-examples/src/MOM6 mom6/MOM6-examples/src/SIS2 mom6/MOM6-examples/src/icebergs - # checkout dev/gfdl on the icebergs submodule - pushd mom6/MOM6-examples/src/icebergs - git checkout dev/gfdl - popd - # this is different than the MOM6_GIT_TAG above since its pulling a submodule not the whole repo - if [[ $MOM6_GIT_FIX ]]; then - echo WARNING: Checking out MOM6_GIT_FIX, set to: $MOM6_GIT_FIX - pushd mom6/MOM6-examples/src/MOM6/ - git checkout $MOM6_GIT_FIX - popd - fi - if [[ $SIS2_GIT_FIX ]]; then - echo WARNING: Checking out SIS2_GIT_FIX, set to: $SIS2_GIT_FIX - pushd mom6/MOM6-examples/src/SIS2/ - git checkout $SIS2_GIT_FIX - popd - fi - # link in dataset - pushd mom6/MOM6-examples - # skip the check for orion - #set platform_domain = `perl -T -e "use Net::Domain(hostdomain) ; print hostdomain"` - #if ("${platform_domain}" =~ *"MsState"* ) then - # ln -s /work/noaa/gfdlscr/pdata/gfdl/gfdl_O/datasets/ .datasets - #else - ln -s /gpfs/f5/gfdl_o/world-shared/datasets .datasets - #endif - popd - test -e mom6/.datasets - if [[ $status != 0 ]]; then - echo ""; echo "" ; echo " WARNING: .datasets link in MOM6 examples directory is invalid"; echo ""; echo "" - fi - cppdefs: "-DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2" - paths: [ "mom6/MOM6-examples/src/MOM6/config_src/infra/FMS2", - "mom6/MOM6-examples/src/MOM6/config_src/memory/dynamic_symmetric", - "mom6/MOM6-examples/src/MOM6/config_src/drivers/FMS_cap", - "mom6/MOM6-examples/src/MOM6/src/ALE", - "mom6/MOM6-examples/src/MOM6/src/core", - "mom6/MOM6-examples/src/MOM6/src/diagnostics", - "mom6/MOM6-examples/src/MOM6/src/equation_of_state", - "mom6/MOM6-examples/src/MOM6/src/framework", - "mom6/MOM6-examples/src/MOM6/src/ice_shelf", - "mom6/MOM6-examples/src/MOM6/src/initialization", - "mom6/MOM6-examples/src/MOM6/src/ocean_data_assim", - "mom6/MOM6-examples/src/MOM6/src/parameterizations", - "mom6/MOM6-examples/src/MOM6/src/tracer", - "mom6/MOM6-examples/src/MOM6/src/user", - "mom6/MOM6-examples/src/MOM6/config_src/external/ODA_hooks", - "mom6/MOM6-examples/src/MOM6/config_src/external/database_comms", - "mom6/MOM6-examples/src/MOM6/config_src/external/drifters", - "mom6/MOM6-examples/src/MOM6/config_src/external/stochastic_physics", - "mom6/MOM6-examples/src/MOM6/config_src/external/stochastic_physics", - "mom6/ocean_BGC/generic_tracers", - "mom6/ocean_BGC/mocsy/src" ] - - component: "sis2" - requires: ["FMS", "mom6"] - repo: "https://github.com/NOAA-GFDL/ice_param.git" - branch: "2024.01" - cppdefs: "-DUSE_FMS2_IO" - otherFlags: !join [ *FMSincludes, " ", *MOMincludes ] - paths: [ "mom6/MOM6-examples/src/SIS2/config_src/dynamic_symmetric", - "mom6/MOM6-examples/src/SIS2/config_src/external/Icepack_interfaces", - "mom6/MOM6-examples/src/SIS2/src", - "mom6/MOM6-examples/src/icebergs/src", - "sis2" ] - - component: "coupler" - requires: ["FMS", "atmos_dyn", "atmos_drivers", "atmos_phys", "lm4p", "sis2", "mom6"] - repo: "https://github.com/NOAA-GFDL/FMScoupler" - branch: "2024.01" - otherFlags: !join [ *FMSincludes, " ", *MOMincludes ] - paths: [ "coupler/shared", - "coupler/full" ] - diff --git a/fre/make/tests/ESM4_example/esm4.yaml b/fre/make/tests/ESM4_example/esm4.yaml deleted file mode 100644 index 436672ab..00000000 --- a/fre/make/tests/ESM4_example/esm4.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# esm4.2, based off ESM4p2_piControl_spinup_J_rts.xml -# this needs -npc (non-parallel checkout) flag during checkout script creation for additional checkouts to work properly -fre_properties: - - &RELEASE "2024.01" - - &FMS_GIT_TAG "2024.01" - - &ATM_PHYS_GIT_TAG "2024.01-alpha6" - - &ATM_FV3_GIT_TAG "2023.03" - - &ATM_DRV_GIT_TAG "2023.04" - - &LAND_GIT_TAG "2024.01" - - &ICE_PARAM_GIT_TAG "2023.04" - - &ESM4_VERSION "2024.01" - - &OCEAN_BGC_GIT_TAG "dev4.2_benthic" - - &MOM6_DATE "20231130" - - &MOM6_GIT_TAG "40e3937" - - &MOM6_GIT_FIX "" - - &SIS2_GIT_FIX "" - # compile - - &FRE_STEM !join [fre/FMS, *RELEASE, _mom6_, *MOM6_DATE] - - &INTEL "intel-classic" - - &FMSincludes "-IFMS/include" - - &MOMincludes "-Imom6/MOM6-examples/src/MOM6/src/framework" - # post processing - - &OM4_ANALYSIS "ice_ocean_SIS2/OM4_05" - - &PROD_SIMTIME "5" # simulation length in years - - &PP_CMIP_CHUNK_A "5yr" # smaller chunk length for pp/analysis - - &PP_CMIP_CHUNK_B "10yr" # bigger chunk length for pp/analysis - - &PP_START_YEAR "0001" # starting year - - &PROD_RUNTIME "16:00:00" # Maximum wall clock per simulation - - &PROD_SEGTIME "03:00:00" # Wall clock per segment (usually 1 year) - # MDBI settings - - &EXP_CPLD_START "0001" # simulation start year - - &EXP_CPLD_END "1000" # simulation end year - # this property wasn't actually used in the xml, but seems to be intended for specifying reference files to verify regression tests - - &reference_tag "FMS2022.03_mom6_20220703" - # these properties modify the build - - &BUILD_DATE "" # included as part of the src directory path - - &MODIFIER "" # appended to compile experiment name, can likely remove since compile experiments are no longer a thing - - &PLATINFO "" # used for awg_input - - &LIBS_ROOT "esm4.2_compile$(MODIFIER)" # used below - - &SRC_DIR !join [ $root/, *BUILD_DATE, /, *LIBS_ROOT, /src] # this ends up being used for input paths - - &MOM6_EXAMPLES !join [ $root/, *BUILD_DATE, /, *LIBS_ROOT, /src/mom6] # also for input paths - # input paths - - &AWG_INPUT_HOME_GAEA "awg_include" - - &AWG_INPUT_HOME_NESCC "awg_include" - - &AWG_INPUT_HOME_GFDL !join [ /nbhome/$USER/, *FRE_STEM, *PLATINFO, /$(name)/mdt_xml/awg_include] - - &USER_FILES_F2toF5 "/gpfs/f5/gfdl_f/world-shared/Niki.Zadeh/archive/input/f2_user_files_in_xmls" - - &append_to_setup_csh "" # The start year of forcing dataset. FRE hack to set fyear - -build: - compileYaml: "compile.yaml" - platformYaml: "platforms.yaml" - -shared: - # directories shared across tools - # shamelessly stolen from am5 example - directories: &shared_directories - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/xtmp/$USER/ptmp" - fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - - # shared pp settings - # also shamelessly stolen from am5 example - postprocess: - settings: &shared_settings - history_segment: "P1Y" - site: "ppan" - switches: &shared_switches - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True - -experiments: - - name: "ESM4p2_piControl_spinup_J" - pp: - - name: "ESM4p2_piControl_spinup_Jb" - pp: - - name: "ESM4p2_piControl_spinup_J_redoyr450_btmdiags" - pp: - - name: "ESM4p2_piControl_spinup_J_redoyr450" - pp: diff --git a/fre/make/tests/ESM4_example/platforms.yaml b/fre/make/tests/ESM4_example/platforms.yaml deleted file mode 100644 index 2114f9da..00000000 --- a/fre/make/tests/ESM4_example/platforms.yaml +++ /dev/null @@ -1,18 +0,0 @@ -platforms: - - name: ncrc5.intel23 - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [ !join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] - fc: ftn - cc: cc - mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] - modelRoot: ${HOME}/fremake_canopy/test - - name: hpcme.2023 - compiler: intel - RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.14.0"] - modelRoot: /apps - fc: mpiifort - cc: mpiicc - container: True - containerBuild: "podman" - containerRun: "apptainer" diff --git a/fre/make/tests/SHiELD_example/SHiELD.yaml b/fre/make/tests/SHiELD_example/SHiELD.yaml deleted file mode 100644 index 9f063d0a..00000000 --- a/fre/make/tests/SHiELD_example/SHiELD.yaml +++ /dev/null @@ -1,10 +0,0 @@ -platformYaml: platforms.yaml -compileYaml: compile.yaml -fv3_release: main -phy_release: main -fms_release: "2023.02" -drivers_release: main -coupler_release: "2023.02" -FMSincludes: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" -momIncludes: "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" -INTEL: intel-classic diff --git a/fre/make/tests/SHiELD_example/compile.yaml b/fre/make/tests/SHiELD_example/compile.yaml deleted file mode 100644 index a83bb1ce..00000000 --- a/fre/make/tests/SHiELD_example/compile.yaml +++ /dev/null @@ -1,38 +0,0 @@ -experiment: shield_nh -compileInclude: "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" -container_addlibs: ["bacio","sp","w3emc","w3nco"] -baremetal_addlibs: ["-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/bacio-2.4.1-wrykbu2/lib -lbacio_4", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/bacio-2.4.1-wrykbu2/lib -lbacio_8", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/sp-2.5.0-7bumbmx/lib64 -lsp_d", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/w3emc-2.10.0-zmuykep/lib64 -lw3emc_d", "-L/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/intel/2023.1.0/w3nco-2.4.1-76qm6h2/lib -lw3nco_d"] -src: - - component: "FMS" - repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-Duse_libMPI -Duse_netCDF -Duse_LARGEFILE -DHAVE_SCHED_GETAFFINITY -DINTERNAL_FILE_NML -DGFS_PHYS -DGFS_CONSTANTS -DHAVE_GETTID" - branch: "$(fms_release)" - - component: "SHiELD_physics" - requires: ["FMS"] - repo: "https://github.com/NOAA-GFDL/SHiELD_physics.git" - branch: "$(phy_release)" - paths: [SHiELD_physics/gsmphys, - SHiELD_physics/GFS_layer, - SHiELD_physics/IPD_layer] - cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND" - otherFlags: "$(FMSincludes)" - - component: "fv3" - requires: ["FMS", "SHiELD_physics"] - repo: ["https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git", - "https://github.com/NOAA-GFDL/atmos_drivers.git"] - cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND" - branch: ["$(fv3_release)","$(drivers_release)"] - paths: [SHiELD_physics/FV3GFS/, - fv3/atmos_drivers/SHiELD/atmos_model.F90, - fv3/GFDL_atmos_cubed_sphere/driver/SHiELD/atmosphere.F90, - fv3/GFDL_atmos_cubed_sphere/tools/, - fv3/GFDL_atmos_cubed_sphere/model/, - fv3/GFDL_atmos_cubed_sphere/GFDL_tools/fv_diag_column.F90] - otherFlags: "$(FMSincludes)" - - component: "FMScoupler" - requires: ["FMS", "SHiELD_physics", "fv3"] - repo: "https://github.com/NOAA-GFDL/FMScoupler.git" - cppdefs: "-Duse_libMPI -Duse_netCDF -DHAVE_SCHED_GETAFFINITY -DSPMD -Duse_LARGEFILE -DGFS_PHYS -DUSE_GFSL63 -DNEW_TAUCTMAX -DNEMS_GSM -DINTERNAL_FILE_NML -DMOIST_CAPPA -DUSE_COND" - branch: "$(coupler_release)" - paths: ["FMScoupler/SHiELD/coupler_main.F90"] - otherFlags: "$(FMSincludes)" diff --git a/fre/make/tests/SHiELD_example/platforms.yaml b/fre/make/tests/SHiELD_example/platforms.yaml deleted file mode 100644 index 9f72043b..00000000 --- a/fre/make/tests/SHiELD_example/platforms.yaml +++ /dev/null @@ -1,26 +0,0 @@ -platforms: - - name: ncrc5.intel - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: ["$(INTEL)/2022.2.1","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/SHiELDtest - - name: ncrc5.intel23 - compiler: intel - modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: ["$(INTEL)/2023.1.0","fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] - fc: ftn - cc: cc - mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk" - modelRoot: ${HOME}/fremake_canopy/SHiELDtest - - name: hpcme.2023 - compiler: intel - RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load netcdf-fortran@4.5.4", "spack load hdf5@1.12.1"] - modelRoot: /apps - fc: mpiifort - cc: mpiicc - container: True - containerBuild: "podman" - containerRun: "apptainer" diff --git a/fre/make/tests/null_example/compile.yaml b/fre/make/tests/null_example/compile.yaml index ab5052a0..85c7862d 100644 --- a/fre/make/tests/null_example/compile.yaml +++ b/fre/make/tests/null_example/compile.yaml @@ -5,7 +5,7 @@ compile: src: - component: "FMS" repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" # gettid flag is platform specific + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML" otherFlags: "-fallow-argument-mismatch" # only needed for gcc branch: *branch - component: "atmos_null" From 670cf66f29a752636bb4bbc290e4d2098ed6fe5f Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:50:15 -0500 Subject: [PATCH 050/111] add option to test_fre_make_run_fremake.py --- fre/make/tests/compilation/test_fre_make_run_fremake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index be91a547..a92cf6a7 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -16,6 +16,6 @@ def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' - runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) + runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, True, False) assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() From ed44e680752854366ce31e3df03d285ada63a555 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:59:47 -0500 Subject: [PATCH 051/111] remove module check --- fre/make/gfdlfremake/buildBaremetal.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index a4ab216b..48205d6b 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -50,8 +50,6 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jo if self.modules != "": self.setup.extend(modulesInit) #extend - this is a list self.setup.append("module load "+self.modules+" \n") # Append -this is a single string - # make sure our modules are loaded correctly - self.setup.append("test $? != 0 && echo \"**********Modules failed to load, check platform yaml\" && exit 1\n") ## Create the build directory os.system("mkdir -p "+self.bld) From b980f031d3b5e74f5e516a4a39f139dfdffa3cf9 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:13:28 -0500 Subject: [PATCH 052/111] fix click entry point, explicitly raise filenotfounderror when the yaml doesnt exist in combine_model functions within combine_yamls classes, add test that would catch a problematic click entrypoint function. --- fre/pp/frepp.py | 4 ++-- fre/tests/test_fre_pp_cli.py | 12 ++++++++++++ fre/yamltools/combine_yamls.py | 20 ++++++++++++-------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index b3456e31..a1c7f8c3 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -2,7 +2,7 @@ import click from .checkoutScript import checkoutTemplate -from .configure_script_yaml import yamlInfo +from .configure_script_yaml import _yamlInfo from .configure_script_xml import convert from .validate import validate_subtool from .install import install_subtool @@ -100,7 +100,7 @@ def install(context, experiment, platform, target): def configure_yaml(context,yamlfile,experiment,platform,target): # pylint: disable=unused-argument """ - Execute fre pp configure """ - context.forward(yamlInfo) + context.forward(_yamlInfo) @pp_cli.command() @click.option("-e", "--experiment", type=str, diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index 399c8e3e..5897d713 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -71,6 +71,18 @@ def test_cli_fre_pp_configure_yaml_opt_dne(): result = runner.invoke(fre.fre, args=["pp", "configure-yaml", "optionDNE"]) assert result.exit_code == 2 +def test_cli_fre_pp_configure_yaml_fail1(): + ''' fre pp configure-yaml ''' + result = runner.invoke(fre.fre, args = [ "pp", "configure-yaml", + "-e", "FOO", + "-p", "BAR", + "-t", "BAZ", + "-y", "BOO" ] ) + assert all( [ result.exit_code == 1, + isinstance(result.exception, FileNotFoundError ) + ] ) + + #-- fre pp install def test_cli_fre_pp_install(): ''' fre pp install ''' diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index b2b6540f..d1879ab7 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -143,10 +143,12 @@ def combine_model(self): f1.write(f'name: &name "{self.name}"\n') f1.write(f'platform: &platform "{self.platform}"\n') f1.write(f'target: &target "{self.target}"\n\n') - with open(self.yml,'r',encoding='UTF-8') as f2: - f1.write("### MODEL YAML SETTINGS ###\n") - shutil.copyfileobj(f2,f1) - + try: + with open(self.yml,'r',encoding='UTF-8') as f2: + f1.write("### MODEL YAML SETTINGS ###\n") + shutil.copyfileobj(f2,f1) + except Exception as exc: + raise FileNotFoundError(f'{self.yml} not found') from exc print(f" model yaml: {self.yml}") def combine_compile(self): @@ -231,10 +233,12 @@ def combine_model(self): f1.write(f'name: &name "{self.name}"\n') f1.write(f'platform: &platform "{self.platform}"\n') f1.write(f'target: &target "{self.target}"\n\n') - with open(self.yml,'r',encoding='UTF-8') as f2: - f1.write("### MODEL YAML SETTINGS ###\n") - shutil.copyfileobj(f2,f1) - + try: + with open(self.yml,'r',encoding='UTF-8') as f2: + f1.write("### MODEL YAML SETTINGS ###\n") + shutil.copyfileobj(f2,f1) + except Exception as exc: + raise FileNotFoundError(f'{self.yml} not found') from exc print(f" model yaml: {self.yml}") def combine_experiment(self): From 443d2bf0ed9d271a819131fccba678bb0ee5fb24 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:17:23 -0500 Subject: [PATCH 053/111] try resetting os.environ HOME back to default when makefile creation test is done --- fre/make/tests/test_create_makefile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fre/make/tests/test_create_makefile.py b/fre/make/tests/test_create_makefile.py index 36188b33..be32e9e2 100644 --- a/fre/make/tests/test_create_makefile.py +++ b/fre/make/tests/test_create_makefile.py @@ -51,7 +51,8 @@ def test_bm_makefile_creation(): Check the makefile is created when a bare-metal platform is used """ # Set output directory as home for fre make output - os.environ["HOME"]=str(Path(out)) + def_home = str(os.environ["HOME"]) + os.environ["HOME"]=out#str(Path(out)) bm_plat = BM_PLATFORM[0] targ = TARGET[0] @@ -60,6 +61,8 @@ def test_bm_makefile_creation(): createMakefile.makefile_create(yamlfile_path,BM_PLATFORM,TARGET) assert Path(f"{out}/fremake_canopy/test/{EXPERIMENT}/{bm_plat}-{targ}/exec/Makefile").exists() + os.environ["HOME"]=def_home + assert os.environ["HOME"] == def_home def test_container_makefile_creation(): """ From b311a360adb867e8f7ed6e47bd69961ead040ddf Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:31:35 -0500 Subject: [PATCH 054/111] remove ch.dir from comments for searchability on github --- fre/app/regrid_xy/regrid_xy.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fre/app/regrid_xy/regrid_xy.py b/fre/app/regrid_xy/regrid_xy.py index 8721b36a..7b363e32 100755 --- a/fre/app/regrid_xy/regrid_xy.py +++ b/fre/app/regrid_xy/regrid_xy.py @@ -164,7 +164,7 @@ def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None, """ ## rose config load check - config_name = os.getcwd() + config_name = os.getcwd() #REMOVE ME TODO config_name += '/rose-app-run.conf' #config_name += '/rose-app.conf' print(f'config_name = {config_name}') @@ -235,8 +235,6 @@ def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None, # grid_spec file management - #starting_dir = os.getcwd() - #os.chdir(work_dir) # i hate it if '.tar' in grid_spec: untar_sp = \ subprocess.run( ['tar', '-xvf', grid_spec, '-C', input_dir], @@ -463,7 +461,6 @@ def regrid_xy(input_dir = None, output_dir = None, begin = None, tmp_dir = None, continue # end of comp loop, exit or next one. - #os.chdir(starting_dir) # not clear this is necessary. print('done running regrid_xy()') return 0 From 2da499c7c225ee80a753e41e6fb8aefc6a5b96fd Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:32:26 -0500 Subject: [PATCH 055/111] swap entrypoint convention for fre pp validate --- fre/pp/frepp.py | 4 ++-- fre/pp/validate.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index a1c7f8c3..fab4de0b 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -4,7 +4,7 @@ from .checkoutScript import checkoutTemplate from .configure_script_yaml import _yamlInfo from .configure_script_xml import convert -from .validate import validate_subtool +from .validate import _validate_subtool from .install import install_subtool from .run import pp_run_subtool from .status import status_subtool @@ -64,7 +64,7 @@ def run(context, experiment, platform, target): def validate(context, experiment, platform, target): # pylint: disable=unused-argument """ - Validate PP configuration""" - context.forward(validate_subtool) + context.forward(_validate_subtool) # fre pp install @pp_cli.command() diff --git a/fre/pp/validate.py b/fre/pp/validate.py index 9c07340f..a2f7e1b3 100644 --- a/fre/pp/validate.py +++ b/fre/pp/validate.py @@ -5,12 +5,12 @@ import subprocess import click -def _validate_subtool(experiment, platform, target): +def validate_subtool(experiment, platform, target): """ Validate the Cylc workflow definition located in ~/cylc-src/____ """ - + go_back_here = os.getcwd() directory = os.path.expanduser('~/cylc-src/' + experiment + '__' + platform + '__' + target) # Change the current working directory @@ -23,8 +23,9 @@ def _validate_subtool(experiment, platform, target): # Validate the Cylc configuration cmd = "cylc validate ." subprocess.run(cmd, shell=True, check=True) + os.chdir(go_back_here) @click.command() -def validate_subtool(experiment, platform, target): +def _validate_subtool(experiment, platform, target): ''' entry point to validate for click ''' - return _validate_subtool(experiment, platform, target) + return validate_subtool(experiment, platform, target) From ece0d33f7fbdbc2fb6cf88a2df67237584d826ac Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:33:01 -0500 Subject: [PATCH 056/111] tweak makfile test to restore homedir to what it once was before the test runs. for later checkout test success. --- fre/make/tests/test_create_makefile.py | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fre/make/tests/test_create_makefile.py b/fre/make/tests/test_create_makefile.py index be32e9e2..1fe2001c 100644 --- a/fre/make/tests/test_create_makefile.py +++ b/fre/make/tests/test_create_makefile.py @@ -7,7 +7,7 @@ from fre.make import createMakefile # SET-UP -test_dir = Path("fre/make/tests") +TEST_DIR = Path("fre/make/tests") NM_EXAMPLE = Path("null_example") YAMLFILE = "null_model.yaml" BM_PLATFORM = ["ncrc5.intel23"] @@ -16,35 +16,35 @@ EXPERIMENT = "null_model_full" # Create output location -out = f"{test_dir}/makefile_out" -if Path(out).exists(): +OUT = f"{TEST_DIR}/makefile_out" +if Path(OUT).exists(): # remove - shutil.rmtree(out) + shutil.rmtree(OUT) # create output directory - Path(out).mkdir(parents=True,exist_ok=True) + Path(OUT).mkdir(parents=True,exist_ok=True) else: - Path(out).mkdir(parents=True,exist_ok=True) + Path(OUT).mkdir(parents=True,exist_ok=True) # Set output directory as home for fre make output -#os.environ["HOME"]=str(Path(out)) +#os.environ["HOME"]=str(Path(OUT)) def test_modelyaml_exists(): """ Check the model yaml exists """ - assert Path(f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}").exists() + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists() def test_compileyaml_exists(): """ Check the compile yaml exists """ - assert Path(f"{test_dir}/{NM_EXAMPLE}/compile.yaml").exists() + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/compile.yaml").exists() def test_platformyaml_exists(): """ Check the platform yaml exists """ - assert Path(f"{test_dir}/{NM_EXAMPLE}/platforms.yaml").exists() + assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/platforms.yaml").exists() def test_bm_makefile_creation(): """ @@ -52,16 +52,16 @@ def test_bm_makefile_creation(): """ # Set output directory as home for fre make output def_home = str(os.environ["HOME"]) - os.environ["HOME"]=out#str(Path(out)) + os.environ["HOME"]=OUT#str(Path(OUT)) bm_plat = BM_PLATFORM[0] targ = TARGET[0] - yamlfile_path = f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}" + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" createMakefile.makefile_create(yamlfile_path,BM_PLATFORM,TARGET) - assert Path(f"{out}/fremake_canopy/test/{EXPERIMENT}/{bm_plat}-{targ}/exec/Makefile").exists() - os.environ["HOME"]=def_home + assert Path(f"{OUT}/fremake_canopy/test/{EXPERIMENT}/{bm_plat}-{targ}/exec/Makefile").exists() + os.environ["HOME"] = def_home assert os.environ["HOME"] == def_home def test_container_makefile_creation(): @@ -69,7 +69,7 @@ def test_container_makefile_creation(): Check the makefile is created when the container platform is used """ container_plat = CONTAINER_PLATFORM[0] - yamlfile_path = f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}" + yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" createMakefile.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET) assert Path(f"tmp/{container_plat}/Makefile").exists() From 50999f3a37f645dc8ef940fd741c6ed732483b9a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:33:43 -0500 Subject: [PATCH 057/111] fre pp chekcout test after fixing makefile test home directory path change --- fre/tests/test_fre_pp_cli.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index 5897d713..df3555b8 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -39,6 +39,16 @@ def test_cli_fre_pp_checkout_opt_dne(): result = runner.invoke(fre.fre, args=["pp", "checkout", "optionDNE"]) assert result.exit_code == 2 +def test_cli_fre_pp_checkout_case(): + ''' fre pp checkout -e FOO -p BAR -t BAZ''' + result = runner.invoke(fre.fre, args=["pp", "checkout", + "-e", "FOO", + "-p", "BAR", + "-t", "BAZ"] ) + directory = os.path.expanduser("~/cylc-src")+'/FOO__BAR__BAZ' + assert all( [ result.exit_code == 0, + Path(directory).exists()] ) + #-- fre pp configure-xml def test_cli_fre_pp_configure_xml(): ''' fre pp configure-xml ''' From 9fc7fe5a967c3a18c15eb7226e09e573eb56c504 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:40:22 -0500 Subject: [PATCH 058/111] remove chdir dependency in fre pp checkout. fix click entrypoint convention --- fre/pp/checkoutScript.py | 14 +++++++------- fre/pp/frepp.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 57036634..71dd809d 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -16,7 +16,7 @@ ############################################# -def _checkoutTemplate(experiment, platform, target, branch='main'): +def checkoutTemplate(experiment, platform, target, branch='main'): """ Checkout the workflow template files from the repo """ @@ -25,7 +25,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): os.makedirs(directory, exist_ok=True) # Change the current working directory - os.chdir(directory) + #os.chdir(directory) # Set the name of the directory name = f"{experiment}__{platform}__{target}" @@ -34,7 +34,7 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): click.echo("cloning experiment into directory " + directory + "/" + name) clonecmd = ( f"git clone -b {branch} --single-branch --depth=1 --recursive " - f"https://github.com/NOAA-GFDL/fre-workflows.git {name}" ) + f"https://github.com/NOAA-GFDL/fre-workflows.git {directory}/{name}" ) preexist_error = f"fatal: destination path '{name}' exists and is not an empty directory." click.echo(clonecmd) cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) @@ -43,8 +43,8 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): argstring = f" -e {experiment} -p {platform} -t {target}" stop_report = ( "Error in checkoutTemplate: the workflow definition specified by -e/-p/-t already" - f" exists at the location ~/cylc-src/{name}!\n" - f"In the future, we will confirm that ~/cylc-src/{name} is usable and will check " + f" exists at the location {directory}/{name}!\n" + f"In the future, we will confirm that {directory}/{name} is usable and will check " "whether it is up-to-date.\n" "But for now, if you wish to proceed, you must delete the workflow definition.\n" "To start over, try:\n" @@ -62,12 +62,12 @@ def _checkoutTemplate(experiment, platform, target, branch='main'): ############################################# @click.command() -def checkoutTemplate(experiment, platform, target, branch="main"): +def _checkoutTemplate(experiment, platform, target, branch="main"): ''' Wrapper script for calling checkoutTemplate - allows the decorated version of the function to be separate from the undecorated version ''' - return _checkoutTemplate(experiment, platform, target, branch) + return checkoutTemplate(experiment, platform, target, branch) if __name__ == '__main__': diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index fab4de0b..dfdaa4aa 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -1,7 +1,7 @@ ''' fre pp ''' import click -from .checkoutScript import checkoutTemplate +from .checkoutScript import _checkoutTemplate from .configure_script_yaml import _yamlInfo from .configure_script_xml import convert from .validate import _validate_subtool @@ -122,7 +122,7 @@ def configure_yaml(context,yamlfile,experiment,platform,target): def checkout(context, experiment, platform, target, branch='main'): # pylint: disable=unused-argument """ - Execute fre pp checkout """ - context.forward(checkoutTemplate) + context.forward(_checkoutTemplate) @pp_cli.command() @click.option('-x', '--xml', From 32a24a6c7c22f0fc95b12c129718d83acc6fbe33 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 16:47:34 -0500 Subject: [PATCH 059/111] add missing os module import --- fre/tests/test_fre_pp_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index df3555b8..e2892a52 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -1,5 +1,7 @@ ''' test "fre pp" calls ''' +import os + from click.testing import CliRunner from fre import fre From 0b3bb5058bf5b4fc28856f444e723e6de4b8d12d Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 22 Nov 2024 17:06:08 -0500 Subject: [PATCH 060/111] add in missing pathlib.Path import --- fre/tests/test_fre_pp_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index e2892a52..de47ca1f 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -1,6 +1,7 @@ ''' test "fre pp" calls ''' import os +from pathlib import Path from click.testing import CliRunner From 6709adaee042a9a19c112b8c01ba32e029613575 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 25 Nov 2024 16:15:04 -0500 Subject: [PATCH 061/111] #216 Fix compile log and change some format to f-strings --- fre/make/gfdlfremake/buildBaremetal.py | 66 +++++++++++++++++--------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index fdb4e2d8..09e707ab 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -40,22 +40,22 @@ def __init__(self,exp,mkTemplatePath,srcDir,bldDir,target,modules,modulesInit,jo self.template = mkTemplatePath self.modules = "" for m in modules: - self.modules = self.modules +" "+ m + self.modules = f"{self.modules} {m}" ## Set up the top portion of the compile script self.setup=[ "#!/bin/sh -fx \n", - "bld_dir="+self.bld+"/ \n", - "src_dir="+self.src+"/ \n", - "mkmf_template="+self.template+" \n"] + f"bld_dir={self.bld}/ \n", + f"src_dir={self.src}/ \n", + f"mkmf_template={self.template} \n"] if self.modules != "": self.setup.extend(modulesInit) #extend - this is a list - self.setup.append("module load "+self.modules+" \n") # Append -this is a single string + self.setup.append(f"module load {self.modules} \n") # Append -this is a single string ## Create the build directory - os.system("mkdir -p "+self.bld) + os.system(f"mkdir -p {self.bld}") ## Create the compile script - self.f=open(self.bld+"/compile.sh","w") + self.f=open(f"{self.bld}/compile.sh","w") self.f.writelines(self.setup) def writeBuildComponents(self, c): @@ -69,7 +69,7 @@ def writeBuildComponents(self, c): comp = c["component"] # Make the component directory - self.f.write("\n mkdir -p $bld_dir/"+comp+"\n") + self.f.write(f"\n mkdir -p $bld_dir/{comp}\n") # Get the paths needed for compiling pstring = "" @@ -77,16 +77,22 @@ def writeBuildComponents(self, c): pstring = pstring+"$src_dir/"+paths+" " # Run list_paths - self.f.write(" list_paths -l -o $bld_dir/"+comp+"/pathnames_"+comp+" "+pstring+"\n") - self.f.write(" cd $bld_dir/"+comp+"\n") + self.f.write(f" list_paths -l -o $bld_dir/{comp}/pathnames_{comp} {pstring}\n") + self.f.write(f" cd $bld_dir/{comp}\n") # Create the mkmf line # If this lib doesnt have any code dependencies and # it requires the preprocessor (no -o and yes --use-cpp) if c["requires"] == [] and c["doF90Cpp"]: - self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n") + self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir " + "-p lib"+comp+".a -t $mkmf_template --use-cpp " + "-c \""+c["cppdefs"]+"\" "+c["otherFlags"] + +" $bld_dir/"+comp+"/pathnames_"+comp+" \n") elif c["requires"] == []: # If this lib doesnt have any code dependencies (no -o) - self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n") + self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir " + "-p lib"+comp+".a -t $mkmf_template -c \"" + +c["cppdefs"]+"\" "+c["otherFlags"] + +" $bld_dir/"+comp+"/pathnames_"+comp+" \n") else: #Has requirements #Set up the requirements as a string to inclue after the -o reqstring = "" @@ -95,9 +101,15 @@ def writeBuildComponents(self, c): #Figure out if we need the preprocessor if c["doF90Cpp"]: - self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template --use-cpp -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n") + self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir " + "-p lib"+comp+".a -t $mkmf_template --use-cpp " + "-c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" " + +c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n") else: - self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir -p lib"+comp+".a -t $mkmf_template -c \""+c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"]+" $bld_dir/"+comp+"/pathnames_"+comp+" \n") + self.f.write(" mkmf -m Makefile -a $src_dir -b $bld_dir " + "-p lib"+comp+".a -t $mkmf_template -c \"" + +c["cppdefs"]+"\" -o \""+reqstring+"\" "+c["otherFlags"] + +" $bld_dir/"+comp+"/pathnames_"+comp+" \n") ##TODO: add targets input def writeScript(self): @@ -106,8 +118,8 @@ def writeScript(self): Param: - self : The buildScript object """ - self.f.write("cd "+self.bld+"\n") - self.f.write(self.make+"\n") + self.f.write(f"cd {self.bld}\n") + self.f.write(f"{self.make}\n") self.f.close() # Make compile script executable @@ -120,10 +132,18 @@ def run(self): Param: - self : The dockerfile object """ -###### TODO make the Makefile - command = [self.bld+"/compile.sh","|","tee",self.bld+"/log.compile"] - try: - subprocess.run(args=command, check=True) - except: - print("There was an error running "+self.bld+"/compile.sh") - raise + command = [self.bld+"/compile.sh"] + + # Run compile script + p1 = subprocess.Popen(command, stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + + # Direct output to log file as well + p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout) + + # Allow p1 to receive SIGPIPE is p2 exits + p1.stdout.close() + p2.communicate() + + if p1.returncode != 0: + print("\nThere was an error running {self.bld}/compile.sh") + print(f"Check the log file: {self.bld}/log.compile") From 6ae49957e5c8ba47d3346f418e5d6052dc34fa93 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 25 Nov 2024 18:10:30 -0500 Subject: [PATCH 062/111] #216 Add f-string --- fre/make/gfdlfremake/buildBaremetal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index 09e707ab..a6a9c208 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -145,5 +145,5 @@ def run(self): p2.communicate() if p1.returncode != 0: - print("\nThere was an error running {self.bld}/compile.sh") + print(f"\nThere was an error running {self.bld}/compile.sh") print(f"Check the log file: {self.bld}/log.compile") From 211d9a1880e2c66cbc332387da6ea5cfe83222f3 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 20:03:00 -0500 Subject: [PATCH 063/111] Safer method of specifying None as default --- fre/pp/frepp.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index fd99e74b..cd1fd2c0 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -113,11 +113,8 @@ def configure_yaml(context,yamlfile,experiment,platform,target): help="Target name", required=True) @click.option("-b", "--branch", - show_default=True, - default=None, type='NoneType', - help="Name of fre2/workflows/postproc branch to clone; " \ - "defaults to None. Not intended for production use, " \ - "but needed for branch testing." ) + required=False, + help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.pass_context def checkout(context, experiment, platform, target, branch=None): # pylint: disable=unused-argument @@ -198,10 +195,7 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo required=True) @click.option("-b", "--branch", show_default=True, - default=None, type='NoneType', - help="Name of fre2/workflows/postproc branch to clone; " \ - "defaults to None. Not intended for production use, " \ - "but needed for branch testing." ) + help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.pass_context def wrapper(context, experiment, platform, target, config_file, branch=None): # pylint: disable=unused-argument From 558429f6ee3157c6e8d4ba3011b3aa31b83fd802 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 20:16:42 -0500 Subject: [PATCH 064/111] The evils of changing directory- you must go back at least --- fre/pp/checkoutScript.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index b665ccb4..a2cd990f 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -28,6 +28,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): os.makedirs(directory, exist_ok=True) # Change the current working directory + go_back_here = os.getcwd() os.chdir(directory) # Set the name of the directory @@ -71,6 +72,7 @@ def _checkoutTemplate(experiment, platform, target, branch=None): else: #scenario 1 clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) print('scenario 1: output of fre pp checkouts git clone command is as follows:',clone_output) + os.chdir(go_back_here) ############################################# From bda67b262a1dc870f62d87253649268a9300044c Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 20:17:54 -0500 Subject: [PATCH 065/111] Improve error message clarity (sort of) --- fre/yamltools/combine_yamls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index 4584c115..94531258 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -94,7 +94,7 @@ def experiment_check(mainyaml_dir,comb,experiment): ey=Path(os.path.join(mainyaml_dir,e)) ey_path.append(ey) else: - raise ValueError("Incorrect experiment yaml path given; does not exist.") + raise ValueError(f"Incorrect experiment yaml path given ({e}); does not exist.") else: raise ValueError("No experiment yaml path given!") From 822900b84049bb997c39812576b1fcd1f184c628 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 21:08:30 -0500 Subject: [PATCH 066/111] Print 2024.1 as 2024.01 when called as fre --version --- fre/fre.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fre/fre.py b/fre/fre.py index 9dd65eb0..573687c7 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -10,6 +10,16 @@ import click from .lazy_group import LazyGroup +# Horrible way to turn xxxx.y into xxxx.0y +import importlib.metadata +version_unexpanded = importlib.metadata.version('fre-cli') +version_unexpanded_split = version_unexpanded.split('.') +if len(version_unexpanded_split[1]) == 1: + version_minor = "0" + version_unexpanded_split[1] +else: + version_minor = version_unexpanded_split[1] +version = version_unexpanded_split[0] + '.' + version_minor + @click.group( cls = LazyGroup, lazy_subcommands = {"pp": ".pp.frepp.pp_cli", @@ -27,10 +37,10 @@ fg='cyan') ) + @click.version_option( package_name = "fre-cli", - message = click.style("%(package)s | %(version)s", - fg = (155,255,172) ) + version=version ) def fre(): From 7492c8060357f7428639572e532d96d460869f06 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 21:10:25 -0500 Subject: [PATCH 067/111] Comment out version specifying in __init__.py for now --- fre/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fre/__init__.py b/fre/__init__.py index 4711e312..ff9bed68 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -1 +1,2 @@ -__version__ = '2024.01' +# aspirational.. like fms-yaml-tools and others specify version +#__version__ = '2024.01' From 0f07d19281b10be98ed771a9a9c99d4c38bc668a Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 21:18:47 -0500 Subject: [PATCH 068/111] Remove hard-coded 2024.1 to 2024.01 translation --- fre/pp/checkoutScript.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index a2cd990f..f8d10075 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -36,9 +36,6 @@ def _checkoutTemplate(experiment, platform, target, branch=None): # branch and version parameters default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] - if default_tag == '2024.1': #hard coded solution to current discrepencies with fre --version - default_tag = '2024.01' - print('the default tag for directory ',directory,'/',name, ' is ', default_tag) if branch != None: print(branch) if os.path.isdir(name): #scenario 4 From 3770d339ce9a68351d5e3c44bfe4b28e2356de77 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 21:27:41 -0500 Subject: [PATCH 069/111] Prepend the yaml directory to the sub-yamls when combining --- fre/yamltools/combine_yamls.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index 94531258..b37687f5 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -90,6 +90,8 @@ def experiment_check(mainyaml_dir,comb,experiment): if expyaml is not None: ey_path=[] for e in expyaml: + # prepend the directory containing the yaml + e = Path(mainyaml_dir, e) if Path(e).exists(): ey=Path(os.path.join(mainyaml_dir,e)) ey_path.append(ey) @@ -101,6 +103,8 @@ def experiment_check(mainyaml_dir,comb,experiment): if analysisyaml is not None: ay_path=[] for a in analysisyaml: + # prepend the directory containing the yaml + a = Path(mainyaml_dir, a) if Path(a).exists(): ay=Path(os.path.join(mainyaml_dir,a)) ay_path.append(ay) From 35ae479f51418e3c62aa4f9c334f5235d77f3c9d Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 21:29:21 -0500 Subject: [PATCH 070/111] Removed old unused bash script --- fre/pp/wrapperscript | 320 ------------------------------------------- 1 file changed, 320 deletions(-) delete mode 100755 fre/pp/wrapperscript diff --git a/fre/pp/wrapperscript b/fre/pp/wrapperscript deleted file mode 100755 index f98d9cd6..00000000 --- a/fre/pp/wrapperscript +++ /dev/null @@ -1,320 +0,0 @@ -#!/bin/bash -set -euo pipefail -set -x - -# https://stackoverflow.com/questions/402377/using-getopts-to-process-long-and-short-command-line-options -TEMP=$(getopt -o x:p:P:T:t:shvc:D:d: --long xml:,platform:,target:,time:,help,mppnccombine-opts:,mail-list: -n 'frepp' -- "$@") -eval set -- "$TEMP" - -# defaults -xml= -platform= -target= -time= -help= - -# arg parsing -while true; do - case "$1" in - # required - -x | --xml ) xml="$2"; shift 2 ;; - -p | -P | --platform ) platform="$2"; shift 2 ;; - -T | --target ) target="$2"; shift 2 ;; - -t | --time ) time="$2"; shift 2 ;; - - # optional - -h | --help ) help=true; shift ;; - - # ignored - -v ) shift ;; - -c ) shift 2 ;; - -D ) shift 2 ;; - -d ) shift 2 ;; - -s ) shift ;; - --mppnccombine-opts ) shift 2 ;; - --mail-list ) shift 2 ;; - - -- ) shift; break ;; - * ) break ;; - esac -done -if [[ -n ${1-} ]]; then - expname=$1 -else - expname= -fi - -# If $FRE_DUALPP is set, then take two different actions -# 1. Append "_canopy" to pp, analysis, and history_refined directories created through the XML converter -# 2. Submit Bronx frepp as well -set +u -if [[ $FRE_DUALPP ]]; then - dual=true -else - dual=false -fi -set -u - -# Help -usage="Usage: frepp --xml=XML --platform=PLATFORM --target=TARGET --time=YYYY EXP" -if [[ $help ]]; then - echo $usage - cat << EOF -################################################################################ -FRE Canopy frepp wrapper to start Canopy postprocessing workflow with -traditional Bronx frepp usage. - -Cylc implementation current settings used by this wrapper: -1. Workflow name is ____ -e.g. use cylc commands such as: - -cylc workflow-state ____ - -This is somewhat overly verbose and also not verbose enough -(i.e. does not include FRE STEM). -If you have suggestions please let the FRE team know. - -2. Will not use unique run directories. -If the run directory exists you will need to remove it before re-installing. - -################################################################################ -What does this script do? -1. If workflow run-dir was previously installed, - start postprocessing for a history file segment: - -- Check if the workflow is running -- Check the task states -- Start cylc scheduler -- Trigger requested processing (-t YYYY) -- Exit - -2. Otherwise, if workflow src-dir does not exist, - configure the postprocessing: - -- Checkout a fresh PP template -- Run the XML converter - -3. Then, install and start the postprocessing for a history file segment -- Run the validation scripts -- Install the workflow -- Start cylc scheduler -- Trigger requested processing (-t YYYY) - -################################################################################ -Recovery steps and scenarios: -1. Something is terribly wrong with PP and you want to reconfigure and try again -- Stop cylc scheduler with "cylc stop --kill " -- Remove run directory with "cylc clean " -- Edit the configuration files in ~/cylc-src/ -- Run frepp again to reinstall and run the updated PP configuration. - -2. Something is terribly wrong and you want a complete fresh start, - or you want an update from the pp template repo. -- Stop cylc scheduler with "cylc stop --kill" -- Remove run directory with "cylc clean " -- Remove src directory with "rm -rf ~/cylc-src/" -- Run frepp again to recheckout pp template, run xml converter, and install/run - -################################################################################ -Specific suggestions to recover from task failures: - -1. refineDiag script failures are likely with a XML-converted configs - for two reasons, so you will probably need to either adjust or remove them. - To disable refineDiag, - - set DO_REFINEDIAG=False, and - - comment out HISTORY_DIR_REFINED - -a. It may use something in the XML, using an xml shell variable that does not - exist now. In these cases, you could rewrite the refineDiag script to - not use the xmlDir shell variable or not use the script. - For "refineDiag_atmos_cmip6.csh", it was included in the postprocessing - template checkout with a small modification. Use this location: - '\$CYLC_WORKFLOW_RUN_DIR/etc/refineDiag/refineDiag_atmos_cmip6.csh'. - - set REFINEDIAG_SCRIPTS to that location - -b. It may be a refineDiag script that does not generate .nc files - as it was expected to do. FRE Bronx allows these side-effect refineDiags, - and instead a new mechanism was invented for these scripts that - do not generate netcdf output: - - set DO_PREANALYSIS=True, and - - PREANALYSIS_SCRIPT="/paath/to/script". - -2. Many PP components in Bronx XMLs are doomed (in terms of failing to - produce output and job failing) caused by using history files that do not - exist, but do not cause problems for the other components. Currently, - the Canopy pp template is not robust in terms of this error mode, - so it's best to not process history files that do not exist. - - In the future, diag manager metadata output will provide a catalog - of history output that the validators will check against. For now, - a simple checker exists, but you must manually generate the - history output list ("history-manifest" file). - - Generate the file with a simple listing of the history tarfile. - You can append a history_refined tarfile as well. Then, the validator - will report on PP components you have specified - (PP_COMPONENTS) but that do not exist in the history-manifest file. - - tar -tf /path/to/history/YYYYMMDD.nc.tar | sort > history-manifest - - To run the configuration validation: - -cd ~/cylc-src/ -rose macro --validate - - It is a good idea to not include pp components (PP_COMPONENTS) that - include history files that do not exist. - - In all cases it is recommended to remove validation errors. - See README.md for general configuration instructions. -EOF - exit 0 -fi - -# check for all options -if [[ $xml ]]; then - xml=$(readlink -f $xml) - if [[ -f $xml ]]; then - echo "using $xml" - else - echo "XML '$xml' does not exist" - exit 1 - fi -else - echo $usage - exit 1 -fi - -if [[ $platform ]]; then - echo "using $platform" -else - echo $usage - exit 1 -fi - -if [[ $target ]]; then - echo "using $target" -else - echo $usage - exit 1 -fi - -if [[ $time ]]; then - echo "using $time" -else - echo $usage - exit 1 -fi - -if [[ $expname ]]; then - echo "using $expname" -else - echo $usage - exit 1 -fi - -cylc --version -if cylc cycle-point $time; then - time_iso=$(cylc cycle-point $time --template CCYYMMDDT0000Z) -else - echo "Time '$time' not a valid ISO8601 date" - exit 1 -fi - -# Start bronx dual-pp -if [[ $dual == true ]]; then - $FRE_COMMANDS_HOME/bin/frepp -x $xml -P $platform -T $target -t $time -D '' $expname -v -s -fi - -# Set the cylc workflow name to __ -# use the default workflow source convention -name=${expname}__${platform}__$target -rundir="$HOME/cylc-run/$name" -srcdir="$HOME/cylc-src/$name" -echo Workflow name: $name -echo Run directory: $rundir -echo Src directory: $srcdir - -# Start postprocessing for a history file segment (workflow was previously installed) -if [[ -d $rundir ]]; then - echo "Run directory '$rundir' exists, so will now try to start it" - cylc scan - cylc workflow-state $name - if cylc workflow-state $name | grep failed; then - cat << EOF -################################################################################ -Unfortunately, there are failed tasks, probably caused by refineDiag errors -or try to use a history file that does not exist. - -While Cylc workflows can be configured to handle failure gracefully, -this workflow is not yet set to do this, so currently it's recommended -to reconfigure your postprocessing to remove task errors. - -For some suggestions to recover from the above most common errors, see: - -frepp --help -################################################################################ -EOF - fi - # sometimes this hangs for unknown reasons - # So for now we'll add --debug to try to diagnose it, and - # use /bin/timeout to exit after 10 min - timeout 10m cylc play --debug $name - sleep 20 - cylc trigger $name//$time_iso/pp-starter - exit 0 -fi - -# Checkout postprocessing template and configure -if [[ ! -d $srcdir ]]; then - echo "Workflow source directory '$srcdir' does not exist, so will now try to checkout template" - - # checkout - mkdir -p $HOME/cylc-src - cd $HOME/cylc-src - # try to reduce checkout size with depth=1 - #git clone --depth=1 --recursive git@gitlab.gfdl.noaa.gov:fre2/workflows/postprocessing.git $name - git clone --depth=1 --recursive https://gitlab.gfdl.noaa.gov/fre2/workflows/postprocessing.git $name - - # xml converter - cd $srcdir - if [[ $dual == true ]]; then - bin/fre-bronx-to-canopy.py -x $xml -p $platform -t $target -e $expname -v --dual - else - bin/fre-bronx-to-canopy.py -x $xml -p $platform -t $target -e $expname -v - fi -fi - -# validate configuration -cd $srcdir -if ! rose macro --validate; then - cat << EOF -################################################################################ -Configuration may not be valid. - -In general, Canopy configurations should pass all available validation scripts. -To run them, - -cd $HOME/cylc-src/$name -rose macro --validate - -Most validation errors reflect configurations problems that should be corrected. -The exceptions are: -1. PP_DIR will be created if it does not exist -2. HISTORY_DIR_REFINED will be created if it does not exist, - assuming DO_REFINEDIAG is also set - -See README.md for general configuration instructions. -################################################################################ -EOF -fi -cylc validate . - -# Install -cylc install --no-run-name $name - -# Start -cylc play $name -sleep 20 -cylc trigger $name//$time_iso/pp-starter -exit 0 From de6c8c7a811a8b62c64fa9a3949b8cc21c612dba Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 22:26:25 -0500 Subject: [PATCH 071/111] logic for fre pp install to compare configurations --- fre/pp/install.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/fre/pp/install.py b/fre/pp/install.py index 9ffc00ee..1b8c59f3 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -1,6 +1,8 @@ #!/usr/bin/env python ''' fre pp install ''' +from pathlib import Path +import os import subprocess import click @@ -13,8 +15,26 @@ def _install_subtool(experiment, platform, target): """ name = experiment + '__' + platform + '__' + target - cmd = f"cylc install --no-run-name {name}" - subprocess.run(cmd, shell=True, check=True) + # if the cylc-run directory already exists, + # then check whether the cylc expanded definition (cylc config) + # is identical. If the same, good. If not, bad. + source_dir = Path(os.path.expanduser("~/cylc-src"), name) + install_dir = Path(os.path.expanduser("~/cylc-run"), name) + if os.path.isdir(install_dir): + installed_def = subprocess.run(["cylc", "config", name],capture_output=True).stdout + go_back_here = os.getcwd() + os.chdir(source_dir) + source_def = subprocess.run(['cylc', 'config', '.'], capture_output=True).stdout + if installed_def == source_def: + print(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") + else: + print(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") + print(f"ERROR: Please manually remove or archive your workflow run directory '{install_dir}' and then run again") + raise Exception + else: + print(f"NOTE: About to install workflow into ~/cylc-run/{name}") + cmd = f"cylc install --no-run-name {name}" + subprocess.run(cmd, shell=True, check=True) @click.command() def install_subtool(experiment, platform, target): From 6bbf2cd575aae6b96e5631b5c5af1ab00671a495 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 22:32:13 -0500 Subject: [PATCH 072/111] please help me learn python exceptions --- fre/pp/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/install.py b/fre/pp/install.py index 1b8c59f3..83cf94f6 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -30,7 +30,7 @@ def _install_subtool(experiment, platform, target): else: print(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") print(f"ERROR: Please manually remove or archive your workflow run directory '{install_dir}' and then run again") - raise Exception + exit(1) else: print(f"NOTE: About to install workflow into ~/cylc-run/{name}") cmd = f"cylc install --no-run-name {name}" From dde48622bc8f7800e4c281e8286aef7c5a72ddc3 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 22:38:31 -0500 Subject: [PATCH 073/111] fre pp wrapper to print one round of 'fre pp status' and exit --- fre/pp/wrapper.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index 0f0aad39..f5dbb1ee 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -73,13 +73,10 @@ def runFre2pp(experiment, platform, target, config_file, branch): except Exception as err: raise - #send off a watcher script that reports on how it's going - for n in range(1,12): - try: - _status_subtool(experiment, platform, target) - except Exception as err: - raise - time.sleep(300) + try: + _status_subtool(experiment, platform, target) + except Exception as err: + raise if __name__ == '__main__': runFre2pp() From 01023ef42de71d52ee60899c77e90414639d45e3 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 23:11:39 -0500 Subject: [PATCH 074/111] Align some click bindings --- fre/pp/install.py | 4 ++-- fre/pp/run.py | 4 ++-- fre/pp/status.py | 4 ++-- fre/pp/wrapper.py | 41 ++++++++++++----------------------------- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/fre/pp/install.py b/fre/pp/install.py index 83cf94f6..30b1cc9d 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -6,7 +6,7 @@ import subprocess import click -def _install_subtool(experiment, platform, target): +def install_subtool(experiment, platform, target): """ Install the Cylc workflow definition located in ~/cylc-src/____ @@ -37,6 +37,6 @@ def _install_subtool(experiment, platform, target): subprocess.run(cmd, shell=True, check=True) @click.command() -def install_subtool(experiment, platform, target): +def _install_subtool(experiment, platform, target): ''' entry point to install for click ''' return _install_subtool(experiment, platform, target) diff --git a/fre/pp/run.py b/fre/pp/run.py index 57f2c427..724acc7a 100644 --- a/fre/pp/run.py +++ b/fre/pp/run.py @@ -4,7 +4,7 @@ import subprocess import click -def _pp_run_subtool(experiment, platform, target): +def pp_run_subtool(experiment, platform, target): """ Start or restart the Cylc workflow identified by: ____ @@ -15,6 +15,6 @@ def _pp_run_subtool(experiment, platform, target): subprocess.run(cmd, shell=True, check=True) @click.command() -def pp_run_subtool(experiment, platform, target): +def _pp_run_subtool(experiment, platform, target): ''' entry point to run for click ''' return _pp_run_subtool(experiment, platform, target) diff --git a/fre/pp/status.py b/fre/pp/status.py index 6e2c07e1..ebfa374d 100644 --- a/fre/pp/status.py +++ b/fre/pp/status.py @@ -4,7 +4,7 @@ import subprocess import click -def _status_subtool(experiment, platform, target): +def status_subtool(experiment, platform, target): """ Report workflow state for the Cylc workflow ____ @@ -16,6 +16,6 @@ def _status_subtool(experiment, platform, target): @click.command() -def status_subtool(experiment, platform, target): +def _status_subtool(experiment, platform, target): ''' entry point to status for click ''' return _status_subtool(experiment, platform, target) diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index 6b41bbb4..49ceb12d 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -18,9 +18,9 @@ # Import from the local packages from .checkoutScript import checkoutTemplate from .configure_script_yaml import yamlInfo -from .install import _install_subtool -from .run import _pp_run_subtool -from .status import _status_subtool +from .install import install_subtool +from .run import pp_run_subtool +from .status import status_subtool @click.command() def runFre2pp(experiment, platform, target, config_file, branch): @@ -32,32 +32,15 @@ def runFre2pp(experiment, platform, target, config_file, branch): config_file = os.path.abspath(config_file) - #env_setup - #todo: check for experiment existing, call frepp_stop to clean experiment, - try: - checkoutTemplate(experiment, platform, target, branch) - except Exception as err: - raise - - try: - yamlInfo(config_file, experiment, platform, target) - except Exception as err: - raise - - try: - _install_subtool(experiment, platform, target) - except: - raise - - try: - _pp_run_subtool(experiment, platform, target) - except Exception as err: - raise - - try: - _status_subtool(experiment, platform, target) - except Exception as err: - raise + checkoutTemplate(experiment, platform, target, branch) + + yamlInfo(config_file, experiment, platform, target) + + install_subtool(experiment, platform, target) + + pp_run_subtool(experiment, platform, target) + + status_subtool(experiment, platform, target) if __name__ == '__main__': runFre2pp() From bdfa7f5054991bbc0ff16c761c04cc515aa6f2a2 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 23:39:46 -0500 Subject: [PATCH 075/111] Added --time option to fre pp wrapper, and standalone fre pp trigger --- fre/pp/frepp.py | 29 ++++++++++++++++++++++++++--- fre/pp/trigger.py | 20 ++++++++++++++++++++ fre/pp/wrapper.py | 7 +++++-- 3 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 fre/pp/trigger.py diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index c5a268ff..12132f94 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -7,6 +7,7 @@ from .validate import _validate_subtool from .install import install_subtool from .run import pp_run_subtool +from .trigger import _trigger from .status import status_subtool from .wrapper import runFre2pp @@ -187,21 +188,43 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo @click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", "--target", type=str, +@click.option("-T", "--target", type=str, help="Target name", required=True) @click.option("-c", "--config-file", type=str, help="Path to a configuration file in either XML or YAML", required=True) +@click.option("-t", "--time", + required=False, + help="Time whose history files are ready") @click.option("-b", "--branch", - show_default=True, + required=False, help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.pass_context -def wrapper(context, experiment, platform, target, config_file, branch=None): +def wrapper(context, experiment, platform, target, config_file, time=None, branch=None): # pylint: disable=unused-argument """ - Execute fre pp steps in order """ context.forward(runFre2pp) +@pp_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +@click.option("-p", "--platform", type=str, + help="Platform name", + required=True) +@click.option("-T", "--target", type=str, + help="Target name", + required=True) +@click.option("-t", "--time", + required=True, + help="Time whose history files are ready") +@click.pass_context +def trigger(context, experiment, platform, target, time): + # pylint: disable=unused-argument + """ - Start postprocessing for a particular time """ + context.forward(_trigger) + if __name__ == "__main__": ''' entry point for click to fre pp commands ''' pp_cli() diff --git a/fre/pp/trigger.py b/fre/pp/trigger.py new file mode 100644 index 00000000..1339e5cc --- /dev/null +++ b/fre/pp/trigger.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +''' fre pp trigger ''' + +import subprocess +import click + +def trigger(experiment, platform, target, time): + """ + Trigger the pp-starter task for the time indicated + """ + + name = experiment + '__' + platform + '__' + target + cmd = f"cylc trigger {name}//{time}/pp-starter" + subprocess.run(cmd, shell=True, check=True, timeout=30) + + +@click.command() +def _trigger(experiment, platform, target, time): + ''' entry point to trigger for click ''' + return trigger(experiment, platform, target, time) diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index 49ceb12d..895f7ee3 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -20,14 +20,14 @@ from .configure_script_yaml import yamlInfo from .install import install_subtool from .run import pp_run_subtool +from .trigger import trigger from .status import status_subtool @click.command() -def runFre2pp(experiment, platform, target, config_file, branch): +def runFre2pp(experiment, platform, target, config_file, branch, time): ''' Wrapper script for calling a FRE2 pp experiment with the canopy-style infrastructure and fre-cli - time=0000 ''' config_file = os.path.abspath(config_file) @@ -40,6 +40,9 @@ def runFre2pp(experiment, platform, target, config_file, branch): pp_run_subtool(experiment, platform, target) + if time: + trigger(experiment, platform, target, time) + status_subtool(experiment, platform, target) if __name__ == '__main__': From c42687bcf44f1f78254b1ac88d3aeafdf01a4c5f Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Mon, 25 Nov 2024 23:55:42 -0500 Subject: [PATCH 076/111] Somewhat improved error message --- fre/pp/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/install.py b/fre/pp/install.py index 30b1cc9d..0df1bab7 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -29,7 +29,7 @@ def install_subtool(experiment, platform, target): print(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") else: print(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") - print(f"ERROR: Please manually remove or archive your workflow run directory '{install_dir}' and then run again") + print(f"ERROR: Please remove installed workflow with 'cylc clean {name}' or move the workflow run directory '{install_dir}'") exit(1) else: print(f"NOTE: About to install workflow into ~/cylc-run/{name}") From f81d60a871861eec7db5edd1a07b499857687360 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 09:44:39 -0500 Subject: [PATCH 077/111] make create_test_conda_env.yml run on pull_request --- .github/workflows/create_test_conda_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 25d382a5..7c353cfc 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -1,6 +1,6 @@ name: create_test_conda_env -on: [push] +on: [push, pull_request] jobs: build-linux: From e5cce133c29da7eccdd4b0034b89ee4f11ee108a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:02:59 -0500 Subject: [PATCH 078/111] skip fre make test thats curently failing, fix in development. --- fre/make/tests/compilation/test_fre_make_run_fremake.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index be91a547..b694f29b 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -14,6 +14,7 @@ # get HOME dir to check output HOME_DIR = os.environ["HOME"] +@pytest.mark.skip(reason='failing: fix in development, see PR 275') def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) From 6ad3f1af65438d3ff793e79534989f3af9664b4d Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:03:15 -0500 Subject: [PATCH 079/111] poke at for readability... --- fre/pp/checkoutScript.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 853d40a7..0a46f833 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python - -# Author: Avery Kiihne ''' Description: Checkout script which accounts for 4 different scenarios: 1. branch not given, folder does not exist, 2. branch given, folder does not exist, 3. branch not given, folder exists, 4. branch given and folder exists @@ -11,10 +8,12 @@ from subprocess import PIPE from subprocess import STDOUT import re + import click + from fre import fre -from click.testing import CliRunner +FRE_WORKFLOWS_URL='https://github.com/NOAA-GFDL/fre-workflows.git' ############################################# def checkoutTemplate(experiment, platform, target, branch=None): @@ -25,8 +24,8 @@ def checkoutTemplate(experiment, platform, target, branch=None): directory = os.path.expanduser("~/cylc-src") os.makedirs(directory, exist_ok=True) - # Chdir back to here before we exit - go_back_here = os.getcwd() + ## Chdir back to here before we exit + #go_back_here = os.getcwd() # Set the name of the directory name = f"{experiment}__{platform}__{target}" @@ -48,7 +47,8 @@ def checkoutTemplate(experiment, platform, target, branch=None): print('scenario 4: directory exists, and branch requested matches branch in use') else: #scenario 2 - clone_output = subprocess.run(['git', 'clone','--recursive', f'--branch={branch}', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) + clone_output = subprocess.run(['git', 'clone','--recursive', f'--branch={branch}', + FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) print('scenario 2: output of fre pp checkouts git clone command is as follows:',clone_output) else: @@ -64,9 +64,10 @@ def checkoutTemplate(experiment, platform, target, branch=None): print('scenario 3: directory exists, and its branch matches default tag') else: #scenario 1 - clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', 'https://github.com/NOAA-GFDL/fre-workflows.git', f'{name}'], capture_output=True, text=True) + clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', + FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) print('scenario 1: output of fre pp checkouts git clone command is as follows:',clone_output) - os.chdir(go_back_here) + #os.chdir(go_back_here) ############################################# From e158de2de99bc60f37d552834335a78a735e1e2e Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:10:28 -0500 Subject: [PATCH 080/111] fix workflow to only run on pull_request, not BOTH a push AND a pull_request (smh at you, github acitons...) --- .github/workflows/create_test_conda_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 7c353cfc..18ff20f0 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -1,6 +1,6 @@ name: create_test_conda_env -on: [push, pull_request] +on: [pull_request] jobs: build-linux: From da1c041c33046389c8747d2c383683f0d15eae07 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:11:42 -0500 Subject: [PATCH 081/111] import pytest to actually skip the test... --- fre/make/tests/compilation/test_fre_make_run_fremake.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index b694f29b..ade1b4db 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -1,9 +1,13 @@ ''' test "fre make run-fremake" calls ''' import os -from fre.make import runFremake from pathlib import Path +import pytest + +from fre.make import runFremake + + # command options YAMLFILE = "fre/make/tests/null_example/null_model.yaml" PLATFORM = [ "ci.gnu" ] From cf02067f63a44ab8d0006551df5c1db9eab6b2c8 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:12:13 -0500 Subject: [PATCH 082/111] add try/except for dir creation. --- fre/pp/checkoutScript.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index 0a46f833..cb40fec4 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -22,7 +22,11 @@ def checkoutTemplate(experiment, platform, target, branch=None): """ # Create the directory if it doesn't exist directory = os.path.expanduser("~/cylc-src") - os.makedirs(directory, exist_ok=True) + try: + os.makedirs(directory, exist_ok=True) + except Exception as exc: + raise OSError('(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc + ## Chdir back to here before we exit #go_back_here = os.getcwd() @@ -32,7 +36,11 @@ def checkoutTemplate(experiment, platform, target, branch=None): # branch and version parameters default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] + + + if branch != None: + print(branch) if os.path.isdir(name): #scenario 4 os.chdir(name) @@ -51,8 +59,7 @@ def checkoutTemplate(experiment, platform, target, branch=None): FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) print('scenario 2: output of fre pp checkouts git clone command is as follows:',clone_output) else: - - + if os.path.isdir(name): #scenario 3 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split()[0] From 4a314c6d49766ee0090ac2798a425e35d7dcd48f Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:15:08 -0500 Subject: [PATCH 083/111] tweak fre pp checkout test --- fre/tests/test_fre_pp_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index de47ca1f..e60d012d 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -1,6 +1,7 @@ ''' test "fre pp" calls ''' import os +import shutil from pathlib import Path from click.testing import CliRunner @@ -44,11 +45,13 @@ def test_cli_fre_pp_checkout_opt_dne(): def test_cli_fre_pp_checkout_case(): ''' fre pp checkout -e FOO -p BAR -t BAZ''' + directory = os.path.expanduser("~/cylc-src")+'/FOO__BAR__BAZ' + if Path(directory).exists(): + shutil.rmtree(directory) result = runner.invoke(fre.fre, args=["pp", "checkout", "-e", "FOO", "-p", "BAR", "-t", "BAZ"] ) - directory = os.path.expanduser("~/cylc-src")+'/FOO__BAR__BAZ' assert all( [ result.exit_code == 0, Path(directory).exists()] ) From b2ff4a52e898cc34d3e84471d5f7022298d61016 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:27:23 -0500 Subject: [PATCH 084/111] toc_tree depth now 2 --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 9c65e98a..07dcbeb9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,7 +11,7 @@ Welcome to ``fre-cli``'s documentation! .. the entry in the toc must be the .rst filename. what shows in the webpage is the first header or title .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Contents: what-is-fre From 16b5e65d7dfcd946d7bf501befd1e033d4915fa7 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 10:45:21 -0500 Subject: [PATCH 085/111] snake case changes and reference tweaks elsewhere as needed --- .../{checkoutScript.py => checkout_script.py} | 21 +++++++++---------- fre/pp/frepp.py | 4 ++-- fre/pp/wrapper.py | 4 ++-- 3 files changed, 14 insertions(+), 15 deletions(-) rename fre/pp/{checkoutScript.py => checkout_script.py} (88%) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkout_script.py similarity index 88% rename from fre/pp/checkoutScript.py rename to fre/pp/checkout_script.py index cb40fec4..53eec8cd 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkout_script.py @@ -14,9 +14,8 @@ from fre import fre FRE_WORKFLOWS_URL='https://github.com/NOAA-GFDL/fre-workflows.git' -############################################# -def checkoutTemplate(experiment, platform, target, branch=None): +def checkout_template(experiment, platform, target, branch=None): """ Checkout the workflow template files from the repo """ @@ -26,7 +25,7 @@ def checkoutTemplate(experiment, platform, target, branch=None): os.makedirs(directory, exist_ok=True) except Exception as exc: raise OSError('(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc - + ## Chdir back to here before we exit #go_back_here = os.getcwd() @@ -53,13 +52,13 @@ def checkoutTemplate(experiment, platform, target, branch=None): sys.exit(stop_report) return 1 print('scenario 4: directory exists, and branch requested matches branch in use') - + else: #scenario 2 clone_output = subprocess.run(['git', 'clone','--recursive', f'--branch={branch}', FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) print('scenario 2: output of fre pp checkouts git clone command is as follows:',clone_output) else: - + if os.path.isdir(name): #scenario 3 os.chdir(name) name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split()[0] @@ -68,8 +67,8 @@ def checkoutTemplate(experiment, platform, target, branch=None): stop_report = f"Tag of prexisting directory {diretory}/{name} does not match fre --version" sys.exit(stop_report) return 1 - print('scenario 3: directory exists, and its branch matches default tag') - + print('scenario 3: directory exists, and its branch matches default tag') + else: #scenario 1 clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) @@ -79,13 +78,13 @@ def checkoutTemplate(experiment, platform, target, branch=None): ############################################# @click.command() -def _checkoutTemplate(experiment, platform, target, branch=None): +def _checkout_template(experiment, platform, target, branch=None): ''' - Wrapper script for calling checkoutTemplate - allows the decorated version + Wrapper script for calling checkout_template - allows the decorated version of the function to be separate from the undecorated version ''' - return checkoutTemplate(experiment, platform, target, branch) + return checkout_template(experiment, platform, target, branch) if __name__ == '__main__': - checkoutTemplate() + checkout_template() diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index 12132f94..8494c720 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -1,7 +1,7 @@ ''' fre pp ''' import click -from .checkoutScript import _checkoutTemplate +from .checkoutScript import _checkout_template from .configure_script_yaml import _yamlInfo from .configure_script_xml import convert from .validate import _validate_subtool @@ -120,7 +120,7 @@ def configure_yaml(context,yamlfile,experiment,platform,target): def checkout(context, experiment, platform, target, branch=None): # pylint: disable=unused-argument """ - Execute fre pp checkout """ - context.forward(_checkoutTemplate) + context.forward(_checkout_template) @pp_cli.command() @click.option('-x', '--xml', diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index 895f7ee3..98f97a46 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -16,7 +16,7 @@ import click # Import from the local packages -from .checkoutScript import checkoutTemplate +from .checkoutScript import checkout_template from .configure_script_yaml import yamlInfo from .install import install_subtool from .run import pp_run_subtool @@ -32,7 +32,7 @@ def runFre2pp(experiment, platform, target, config_file, branch, time): config_file = os.path.abspath(config_file) - checkoutTemplate(experiment, platform, target, branch) + checkout_template(experiment, platform, target, branch) yamlInfo(config_file, experiment, platform, target) From 0827afeac6a4e0e7ddc804b6ec5c28a7f5b3e483 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:47:37 -0500 Subject: [PATCH 086/111] add gettid flag to null model --- fre/make/tests/null_example/compile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/null_example/compile.yaml b/fre/make/tests/null_example/compile.yaml index 85c7862d..68c151f8 100644 --- a/fre/make/tests/null_example/compile.yaml +++ b/fre/make/tests/null_example/compile.yaml @@ -5,7 +5,7 @@ compile: src: - component: "FMS" repo: "https://github.com/NOAA-GFDL/FMS.git" - cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML" + cppdefs: "-Duse_netCDF -Duse_libMPI -DMAXFIELDS_=200 -DMAXFIELDMETHODS_=200 -DINTERNAL_FILE_NML -DHAVE_GETTID" otherFlags: "-fallow-argument-mismatch" # only needed for gcc branch: *branch - component: "atmos_null" From 5528378d1c8871ee96b285799d80809e6e8d46b6 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 11:34:26 -0500 Subject: [PATCH 087/111] pylint things, clean up of checkout_script functioning, fix fre versioning and put it in init where it belongs --- fre/__init__.py | 12 ++++ fre/fre.py | 10 --- fre/pp/checkout_script.py | 107 ++++++++++++++++---------------- fre/pp/configure_script_xml.py | 1 - fre/pp/configure_script_yaml.py | 1 - fre/pp/frepp.py | 35 ++++++----- fre/pp/install.py | 1 - fre/pp/run.py | 1 - fre/pp/status.py | 1 - fre/pp/trigger.py | 1 - fre/pp/validate.py | 1 - fre/pp/wrapper.py | 2 +- 12 files changed, 85 insertions(+), 88 deletions(-) diff --git a/fre/__init__.py b/fre/__init__.py index ff9bed68..7481ac79 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -1,2 +1,14 @@ # aspirational.. like fms-yaml-tools and others specify version #__version__ = '2024.01' + + +# Horrible way to turn xxxx.y into xxxx.0y +import importlib.metadata +version_unexpanded = importlib.metadata.version('fre-cli') +version_unexpanded_split = version_unexpanded.split('.') +if len(version_unexpanded_split[1]) == 1: + version_minor = "0" + version_unexpanded_split[1] +else: + version_minor = version_unexpanded_split[1] +version = version_unexpanded_split[0] + '.' + version_minor +__version__=version diff --git a/fre/fre.py b/fre/fre.py index 573687c7..562cfffa 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -10,16 +10,6 @@ import click from .lazy_group import LazyGroup -# Horrible way to turn xxxx.y into xxxx.0y -import importlib.metadata -version_unexpanded = importlib.metadata.version('fre-cli') -version_unexpanded_split = version_unexpanded.split('.') -if len(version_unexpanded_split[1]) == 1: - version_minor = "0" + version_unexpanded_split[1] -else: - version_minor = version_unexpanded_split[1] -version = version_unexpanded_split[0] + '.' + version_minor - @click.group( cls = LazyGroup, lazy_subcommands = {"pp": ".pp.frepp.pp_cli", diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index 53eec8cd..a20b0cfc 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -5,80 +5,81 @@ import os import sys import subprocess -from subprocess import PIPE -from subprocess import STDOUT -import re import click from fre import fre -FRE_WORKFLOWS_URL='https://github.com/NOAA-GFDL/fre-workflows.git' +FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git' -def checkout_template(experiment, platform, target, branch=None): +def checkout_template(experiment = None, platform = None, target = None, branch = None): """ Checkout the workflow template files from the repo """ + ## Chdir back to here before we exit this routine + go_back_here = os.getcwd() + + # branch and version parameters + #default_tag = subprocess.run( ["fre","--version"], + # capture_output = True, text = True, check = True).stdout.split()[2] + default_tag = fre.__version__ + print(f'(checkout_script) default_tag is {default_tag}') + + + # check args + set the name of the directory + if None in [experiment, platform, target]: + raise ValueError( 'one of these are None: experiment / platform / target = \n' + f'{experiment} / {platform} / {target}' ) + name = f"{experiment}__{platform}__{target}" + # Create the directory if it doesn't exist directory = os.path.expanduser("~/cylc-src") try: - os.makedirs(directory, exist_ok=True) + os.makedirs(directory, exist_ok = True) except Exception as exc: raise OSError('(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc - - ## Chdir back to here before we exit - #go_back_here = os.getcwd() - - # Set the name of the directory - name = f"{experiment}__{platform}__{target}" - - # branch and version parameters - default_tag = subprocess.run(["fre","--version"],capture_output=True, text=True).stdout.split()[2] - - - - if branch != None: - - print(branch) - if os.path.isdir(name): #scenario 4 - os.chdir(name) - name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split('*') - name_path_branch=subprocess.run(["git","branch"],capture_output=True, text=True).stdout - name_path_branch = name_path_branch.split('*')[1].split()[0] - os.chdir(directory) - if default_tag not in name_path_tag and name_path_branch != branch: - stop_report = f"Tag and branch of prexisting directory {directory}/{name} does not match fre --version or branch requested" - sys.exit(stop_report) - return 1 - print('scenario 4: directory exists, and branch requested matches branch in use') - - else: #scenario 2 - clone_output = subprocess.run(['git', 'clone','--recursive', f'--branch={branch}', - FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) - print('scenario 2: output of fre pp checkouts git clone command is as follows:',clone_output) - else: - - if os.path.isdir(name): #scenario 3 - os.chdir(name) - name_path_tag=subprocess.run(["git","describe","--tags"],capture_output=True, text=True).stdout.split()[0] - os.chdir(directory) + print(f'(checkout_script) branch is {branch}') + checkout_exists = os.path.isdir(f'{directory}/{name}') + git_clone_branch_arg = branch if branch is not None else default_tag + if branch is not None: + print(f'(checkout_script) WARNING using default_tag as branch argument for git clone!') + + + if not checkout_exists: # scenarios 1 and 2, repo checkout doesn't exist, branch specified (or not) + clone_output = subprocess.run(['git', 'clone','--recursive', + f'--branch={git_clone_branch_arg}', + FRE_WORKFLOWS_URL, f'{directory}/{name}'], + capture_output = True, text = True, check = True) + print(f'(checkout_script) output git clone command: {clone_output}') + + else: # the repo checkout does exist, scenarios 3 and 4. + os.chdir(f'{directory}/{name}') + + name_path_tag_subproc_out = subprocess.run(["git","describe","--tags"],capture_output = True, text = True, check = True).stdout + if branch is not None: + name_path_tag = name_path_tag_subproc_out.split('*') + name_path_branch = subprocess.run(["git","branch"],capture_output = True, text = True,check = True).stdout.split()[0] + if all( [ default_tag not in name_path_tag, + name_path_branch != branch ] ): + sys.exit( + f"Tag and branch of prexisting directory {directory}/{name} does not match fre --version or branch requested") + else: + name_path_tag = name_path_tag_subproc_out.split()[0] if not default_tag in name_path_tag: - stop_report = f"Tag of prexisting directory {diretory}/{name} does not match fre --version" - sys.exit(stop_report) - return 1 - print('scenario 3: directory exists, and its branch matches default tag') + sys.exit( + f"Tag of prexisting directory {directory}/{name} does not match fre --version") - else: #scenario 1 - clone_output = subprocess.run(['git', 'clone', '--recursive', '-b',f'{default_tag}', - FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output=True, text=True) - print('scenario 1: output of fre pp checkouts git clone command is as follows:',clone_output) - #os.chdir(go_back_here) + # make sure we are back where we should be + if os.getcwd() != go_back_here: + os.chdir(go_back_here) + + return 0 ############################################# @click.command() -def _checkout_template(experiment, platform, target, branch=None): +def _checkout_template(experiment, platform, target, branch = None): ''' Wrapper script for calling checkout_template - allows the decorated version of the function to be separate from the undecorated version diff --git a/fre/pp/configure_script_xml.py b/fre/pp/configure_script_xml.py index 899b4830..38247ce1 100644 --- a/fre/pp/configure_script_xml.py +++ b/fre/pp/configure_script_xml.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 ''' Primary Usage: fre-bronx-to-canopy -x XML -e EXP -p PLATFORM -t TARGET diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index b782e3de..383d5acf 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Script creates rose-apps and rose-suite files for the workflow from the pp yaml. diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index 8494c720..d21c5d23 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -1,15 +1,16 @@ ''' fre pp ''' import click -from .checkoutScript import _checkout_template -from .configure_script_yaml import _yamlInfo -from .configure_script_xml import convert -from .validate import _validate_subtool -from .install import install_subtool -from .run import pp_run_subtool -from .trigger import _trigger -from .status import status_subtool -from .wrapper import runFre2pp + +from fre.pp import checkout_script +from fre.pp import configure_script_yaml +from fre.pp import configure_script_xml +#from fre.pp import validate +from fre.pp import install +from fre.pp import run +from fre.pp import trigger +from fre.pp import status +from fre.pp import wrapper @click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210))) def pp_cli(): @@ -31,7 +32,7 @@ def pp_cli(): def status(context, experiment, platform, target): # pylint: disable=unused-argument """ - Report status of PP configuration""" - context.forward(status_subtool) + context.forward(status.status_subtool) # fre pp run @pp_cli.command() @@ -48,7 +49,7 @@ def status(context, experiment, platform, target): def run(context, experiment, platform, target): # pylint: disable=unused-argument """ - Run PP configuration""" - context.forward(pp_run_subtool) + context.forward(run.pp_run_subtool) # fre pp validate @pp_cli.command() @@ -82,7 +83,7 @@ def validate(context, experiment, platform, target): def install(context, experiment, platform, target): # pylint: disable=unused-argument """ - Install PP configuration""" - context.forward(install_subtool) + context.forward(install.install_subtool) @pp_cli.command() @click.option("-y", "--yamlfile", type=str, @@ -101,7 +102,7 @@ def install(context, experiment, platform, target): def configure_yaml(context,yamlfile,experiment,platform,target): # pylint: disable=unused-argument """ - Execute fre pp configure """ - context.forward(_yamlInfo) + context.forward(configure_script_yaml._yamlInfo) @pp_cli.command() @click.option("-e", "--experiment", type=str, @@ -120,7 +121,7 @@ def configure_yaml(context,yamlfile,experiment,platform,target): def checkout(context, experiment, platform, target, branch=None): # pylint: disable=unused-argument """ - Execute fre pp checkout """ - context.forward(_checkout_template) + context.forward(checkout_script._checkout_template) @pp_cli.command() @click.option('-x', '--xml', @@ -178,7 +179,7 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual): # pylint: disable=unused-argument """ - Converts a Bronx XML to a Canopy rose-suite.conf """ - context.forward(convert) + context.forward(configure_script_xml.convert) #fre pp wrapper @pp_cli.command() @@ -204,7 +205,7 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo def wrapper(context, experiment, platform, target, config_file, time=None, branch=None): # pylint: disable=unused-argument """ - Execute fre pp steps in order """ - context.forward(runFre2pp) + context.forward(wrapper.runFre2pp) @pp_cli.command() @click.option("-e", "--experiment", type=str, @@ -223,7 +224,7 @@ def wrapper(context, experiment, platform, target, config_file, time=None, branc def trigger(context, experiment, platform, target, time): # pylint: disable=unused-argument """ - Start postprocessing for a particular time """ - context.forward(_trigger) + context.forward(trigger._trigger) if __name__ == "__main__": ''' entry point for click to fre pp commands ''' diff --git a/fre/pp/install.py b/fre/pp/install.py index 0df1bab7..cb2ed021 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' fre pp install ''' from pathlib import Path diff --git a/fre/pp/run.py b/fre/pp/run.py index 724acc7a..d50c9f01 100644 --- a/fre/pp/run.py +++ b/fre/pp/run.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' fre pp run ''' import subprocess diff --git a/fre/pp/status.py b/fre/pp/status.py index ebfa374d..0631be7a 100644 --- a/fre/pp/status.py +++ b/fre/pp/status.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' fre pp status ''' import subprocess diff --git a/fre/pp/trigger.py b/fre/pp/trigger.py index 1339e5cc..e6ebd01b 100644 --- a/fre/pp/trigger.py +++ b/fre/pp/trigger.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' fre pp trigger ''' import subprocess diff --git a/fre/pp/validate.py b/fre/pp/validate.py index a2f7e1b3..fd54c52c 100644 --- a/fre/pp/validate.py +++ b/fre/pp/validate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' fre pp validate ''' import os diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index 98f97a46..c9d6793b 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -16,7 +16,7 @@ import click # Import from the local packages -from .checkoutScript import checkout_template +from .checkout_script import checkout_template from .configure_script_yaml import yamlInfo from .install import install_subtool from .run import pp_run_subtool From 57780a45e055864df0b75b72d99eba199c610956 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 11:40:02 -0500 Subject: [PATCH 088/111] fix fre versioning reference, remove version input to click version option, since it seems to pick it up from the __init__.py correctly anyways --- fre/fre.py | 4 ++-- fre/pp/checkout_script.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/fre.py b/fre/fre.py index 562cfffa..d9adc8e5 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -29,8 +29,8 @@ @click.version_option( - package_name = "fre-cli", - version=version + package_name = "fre-cli" +# version=fre.__version__ ) def fre(): diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index a20b0cfc..13399b14 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -8,7 +8,7 @@ import click -from fre import fre +import fre FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git' From 4b43e0c68130b9375cbb54914c6db18dfe6bc7d4 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 11:44:10 -0500 Subject: [PATCH 089/111] pylint feedback --- fre/pp/checkout_script.py | 43 +++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index 13399b14..ccd99368 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -1,6 +1,9 @@ ''' -Description: Checkout script which accounts for 4 different scenarios: 1. branch not given, folder does not exist, -2. branch given, folder does not exist, 3. branch not given, folder exists, 4. branch given and folder exists +Description: Checkout script which accounts for 4 different scenarios: +1. branch not given, folder does not exist, +2. branch given, folder does not exist, +3. branch not given, folder exists, +4. branch given and folder exists ''' import os import sys @@ -20,8 +23,6 @@ def checkout_template(experiment = None, platform = None, target = None, branch go_back_here = os.getcwd() # branch and version parameters - #default_tag = subprocess.run( ["fre","--version"], - # capture_output = True, text = True, check = True).stdout.split()[2] default_tag = fre.__version__ print(f'(checkout_script) default_tag is {default_tag}') @@ -37,43 +38,49 @@ def checkout_template(experiment = None, platform = None, target = None, branch try: os.makedirs(directory, exist_ok = True) except Exception as exc: - raise OSError('(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc + raise OSError( + '(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc print(f'(checkout_script) branch is {branch}') checkout_exists = os.path.isdir(f'{directory}/{name}') git_clone_branch_arg = branch if branch is not None else default_tag if branch is not None: - print(f'(checkout_script) WARNING using default_tag as branch argument for git clone!') + print('(checkout_script) WARNING using default_tag as branch argument for git clone!') - if not checkout_exists: # scenarios 1 and 2, repo checkout doesn't exist, branch specified (or not) - clone_output = subprocess.run(['git', 'clone','--recursive', - f'--branch={git_clone_branch_arg}', - FRE_WORKFLOWS_URL, f'{directory}/{name}'], - capture_output = True, text = True, check = True) + if not checkout_exists: # scenarios 1+2, checkout doesn't exist, branch specified (or not) + clone_output = subprocess.run( ['git', 'clone','--recursive', + f'--branch={git_clone_branch_arg}', + FRE_WORKFLOWS_URL, f'{directory}/{name}'], + capture_output = True, text = True, check = True) print(f'(checkout_script) output git clone command: {clone_output}') else: # the repo checkout does exist, scenarios 3 and 4. os.chdir(f'{directory}/{name}') - - name_path_tag_subproc_out = subprocess.run(["git","describe","--tags"],capture_output = True, text = True, check = True).stdout + + name_path_tag_subproc_out = subprocess.run(["git","describe","--tags"], + capture_output = True, + text = True, check = True).stdout if branch is not None: name_path_tag = name_path_tag_subproc_out.split('*') - name_path_branch = subprocess.run(["git","branch"],capture_output = True, text = True,check = True).stdout.split()[0] + name_path_branch = subprocess.run(["git","branch"], + capture_output = True, + text = True, check = True).stdout.split()[0] if all( [ default_tag not in name_path_tag, name_path_branch != branch ] ): sys.exit( - f"Tag and branch of prexisting directory {directory}/{name} does not match fre --version or branch requested") + f"Tag and branch of prexisting directory {directory}/{name} does not match " + "fre --version or branch requested" ) else: name_path_tag = name_path_tag_subproc_out.split()[0] if not default_tag in name_path_tag: - sys.exit( - f"Tag of prexisting directory {directory}/{name} does not match fre --version") + sys.exit( + f"Tag of prexisting directory {directory}/{name} does not match fre --version") # make sure we are back where we should be if os.getcwd() != go_back_here: os.chdir(go_back_here) - + return 0 ############################################# From cc8f830a44ce5ac3266e6631487ac21aa7beebbb Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 11:54:03 -0500 Subject: [PATCH 090/111] switch target/time flags in wrapper calls to be consistent with other flags for target --- fre/__init__.py | 8 ++------ fre/fre.py | 1 - fre/pp/frepp.py | 8 ++++---- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/fre/__init__.py b/fre/__init__.py index 7481ac79..4524edb3 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -1,8 +1,4 @@ -# aspirational.. like fms-yaml-tools and others specify version -#__version__ = '2024.01' - - -# Horrible way to turn xxxx.y into xxxx.0y +# turn xxxx.y into xxxx.0y import importlib.metadata version_unexpanded = importlib.metadata.version('fre-cli') version_unexpanded_split = version_unexpanded.split('.') @@ -11,4 +7,4 @@ else: version_minor = version_unexpanded_split[1] version = version_unexpanded_split[0] + '.' + version_minor -__version__=version +__version__ = version diff --git a/fre/fre.py b/fre/fre.py index d9adc8e5..388a4588 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -30,7 +30,6 @@ @click.version_option( package_name = "fre-cli" -# version=fre.__version__ ) def fre(): diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index d21c5d23..f9e21ad3 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -189,13 +189,13 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo @click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-T", "--target", type=str, +@click.option("-t", "--target", type=str, help="Target name", required=True) @click.option("-c", "--config-file", type=str, help="Path to a configuration file in either XML or YAML", required=True) -@click.option("-t", "--time", +@click.option("-T", "--time", required=False, help="Time whose history files are ready") @click.option("-b", "--branch", @@ -214,10 +214,10 @@ def wrapper(context, experiment, platform, target, config_file, time=None, branc @click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-T", "--target", type=str, +@click.option("-t", "--target", type=str, help="Target name", required=True) -@click.option("-t", "--time", +@click.option("-T", "--time", required=True, help="Time whose history files are ready") @click.pass_context From caffa7f4f845d3007082dca1b2bf42afefdf6edd Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Tue, 26 Nov 2024 11:58:36 -0500 Subject: [PATCH 091/111] add pull_request trigger to create_test_conda_env workflow --- .github/workflows/create_test_conda_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 25d382a5..7c353cfc 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -1,6 +1,6 @@ name: create_test_conda_env -on: [push] +on: [push, pull_request] jobs: build-linux: From 7fe174ec57eccca8fe557dc07cf289eaf5f17c2e Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Tue, 26 Nov 2024 12:03:43 -0500 Subject: [PATCH 092/111] put the -t/-T convention back to what it was for bronx --- fre/pp/frepp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index f9e21ad3..d21c5d23 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -189,13 +189,13 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo @click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", "--target", type=str, +@click.option("-T", "--target", type=str, help="Target name", required=True) @click.option("-c", "--config-file", type=str, help="Path to a configuration file in either XML or YAML", required=True) -@click.option("-T", "--time", +@click.option("-t", "--time", required=False, help="Time whose history files are ready") @click.option("-b", "--branch", @@ -214,10 +214,10 @@ def wrapper(context, experiment, platform, target, config_file, time=None, branc @click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", "--target", type=str, +@click.option("-T", "--target", type=str, help="Target name", required=True) -@click.option("-T", "--time", +@click.option("-t", "--time", required=True, help="Time whose history files are ready") @click.pass_context From 7b06b5a760cea13eb8edb171e07f00ee41a88eb8 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Tue, 26 Nov 2024 15:01:20 -0500 Subject: [PATCH 093/111] update null model platforms for latest c5 modules --- fre/make/tests/null_example/platforms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/null_example/platforms.yaml b/fre/make/tests/null_example/platforms.yaml index fdfa9d4f..a1595bee 100644 --- a/fre/make/tests/null_example/platforms.yaml +++ b/fre/make/tests/null_example/platforms.yaml @@ -10,7 +10,7 @@ platforms: - name: ncrc5.intel23 compiler: intel modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"] - modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3] + modules: [!join [*INTEL, "/2023.2.0"],"fre/bronx-21",cray-hdf5/1.12.2.11, cray-netcdf/4.9.0.11] fc: ftn cc: cc mkTemplate: !join ["/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/", *INTEL, ".mk"] From c52a769aefa36ba1d7a1df8386e9fd3b8b91ef38 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Tue, 26 Nov 2024 15:50:55 -0500 Subject: [PATCH 094/111] changes to always create the container build script --- fre/make/gfdlfremake/buildDocker.py | 19 +++---------------- fre/make/runFremake.py | 13 +++++++------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/fre/make/gfdlfremake/buildDocker.py b/fre/make/gfdlfremake/buildDocker.py index cb48b48c..491d5c2c 100644 --- a/fre/make/gfdlfremake/buildDocker.py +++ b/fre/make/gfdlfremake/buildDocker.py @@ -184,24 +184,10 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk): self.d.write('ENTRYPOINT ["/bin/bash"]') self.d.close() - def build(self,containerBuild,containerRun): - """ - Brief: Builds the container image for the model - Param: - - self : The dockerfile object - - containerBuild : The tool used to build the container; - docker or podman used - - containerRun : The container platform used with `exec` to - run the container; apptainer or singularity used - """ - os.system(containerBuild+" build -f Dockerfile -t "+self.e+":"+self.target.gettargetName()) - os.system("rm -f "+self.e+".tar "+self.e+".sif") - os.system(containerBuild+" save -o "+self.e+"-"+self.target.gettargetName()+".tar localhost/"+self.e+":"+self.target.gettargetName()) - os.system(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar") - def createBuildScript(self,containerBuild,containerRun): """ - Brief: Writes out the build commands for the created dockerfile in a script, uses the same commands as the routine above + Brief: Writes out the build commands for the created dockerfile in a script, + which builds the dockerfile and then converts the format to a singularity image file. Param: - self : The dockerfile object - containerBuild : The tool used to build the container; @@ -216,6 +202,7 @@ def createBuildScript(self,containerBuild,containerRun): self.userScript.append(containerRun+" build --disable-cache "+self.e+"-"+self.target.gettargetName()+".sif docker-archive://"+self.e+"-"+self.target.gettargetName()+".tar\n") self.userScriptFile = open("createContainer.sh","w") self.userScriptFile.writelines(self.userScript) + self.userScriptFile.close() os.chmod("createContainer.sh", 0o744) self.userScriptPath = os.getcwd()+"/createContainer.sh" diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 0e4cfecd..17e1c28b 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -10,6 +10,7 @@ from multiprocessing.dummy import Pool from pathlib import Path import click +import subprocess import fre.yamltools.combine_yamls as cy from .gfdlfremake import ( targetfre, varsfre, yamlfre, checkout, @@ -190,13 +191,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec dockerBuild.writeRunscript(RUNenv,containerRun,tmpDir+"/execrunscript.sh") - ## Run the dockerfile and create the container if execute option is given - ## otherwise create a build script and print out its path + # Create build script for container + dockerBuild.createBuildScript(containerBuild, containerRun) + print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") + + # Execute if flag is given if execute: - dockerBuild.build(containerBuild,containerRun) - else: - dockerBuild.createBuildScript(containerBuild, containerRun) - print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") + subprocess.run(args=[dockerBuild.userScriptPath], check=True) #freCheckout.cleanup() #buildDockerfile(fremakeYaml,image) From 879a772bc0f7fef4ae9a69c11d59fdaf34f9afc8 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Tue, 26 Nov 2024 16:17:37 -0500 Subject: [PATCH 095/111] always create a container buildscript for createDockerfile subcommand --- fre/make/createDocker.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index 59b73ee9..7dfefaa6 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -72,10 +72,13 @@ def dockerfile_create(yamlfile,platform,target,execute): click.echo("\ntmpDir created in " + currDir + "/tmp") click.echo("Dockerfile created in " + currDir +"\n") - if run: - dockerBuild.build(containerBuild, containerRun) - else: - sys.exit() + # create build script for container + dockerBuild.createBuildScript(containerBuild, containerRun) + print("Container build script created at "+dockerBuild.userScriptPath+"\n\n") + + # run the script if option is given + if run: + subprocess.run(args=[dockerBuild.userScriptPath], check=True) @click.command() def _dockerfile_create(yamlfile,platform,target,execute): From e18742ff32e1adbff6fd16d07be7086c6b018550 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 10:48:21 -0500 Subject: [PATCH 096/111] rename fre/pp/*.py files to not clash with click cli command names. some pylint feedback here and there. --- fre/pp/checkout_script.py | 2 +- fre/pp/configure_script_yaml.py | 10 ++-- fre/pp/frepp.py | 52 +++++++++--------- fre/pp/{install.py => install_script.py} | 0 fre/pp/{run.py => run_script.py} | 6 +- fre/pp/{status.py => status_script.py} | 5 +- fre/pp/tests/test_configure_script_yaml.py | 2 +- fre/pp/tests/test_rose_quoting.py | 3 + fre/pp/{trigger.py => trigger_script.py} | 3 + fre/pp/{validate.py => validate_script.py} | 3 + fre/pp/wrapper.py | 49 ----------------- fre/pp/wrapper_script.py | 64 ++++++++++++++++++++++ 12 files changed, 116 insertions(+), 83 deletions(-) rename fre/pp/{install.py => install_script.py} (100%) rename fre/pp/{run.py => run_script.py} (81%) rename fre/pp/{status.py => status_script.py} (82%) rename fre/pp/{trigger.py => trigger_script.py} (92%) rename fre/pp/{validate.py => validate_script.py} (94%) delete mode 100644 fre/pp/wrapper.py create mode 100644 fre/pp/wrapper_script.py diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index ccd99368..339d8b21 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -86,7 +86,7 @@ def checkout_template(experiment = None, platform = None, target = None, branch ############################################# @click.command() -def _checkout_template(experiment, platform, target, branch = None): +def _checkout_template(experiment, platform, target, branch ): ''' Wrapper script for calling checkout_template - allows the decorated version of the function to be separate from the undecorated version diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 383d5acf..25507c14 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -146,7 +146,7 @@ def set_rose_apps(yamlfile,rose_regrid,rose_remap): value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') #################### -def yamlInfo(yamlfile,experiment,platform,target): +def yaml_info(yamlfile,experiment,platform,target): """ Using a valid pp.yaml, the rose-app and rose-suite configuration files are created in the cylc-src @@ -199,13 +199,13 @@ def yamlInfo(yamlfile,experiment,platform,target): print(" " + outfile) @click.command() -def _yamlInfo(yamlfile,experiment,platform,target): +def _yaml_info(yamlfile,experiment,platform,target): ''' - Wrapper script for calling yamlInfo - allows the decorated version + Wrapper script for calling yaml_info - allows the decorated version of the function to be separate from the undecorated version ''' - return yamlInfo(yamlfile,experiment,platform,target) + return yaml_info(yamlfile,experiment,platform,target) # Use parseyaml function to parse created edits.yaml if __name__ == '__main__': - yamlInfo() + yaml_info() diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index d21c5d23..cdd905d4 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -2,15 +2,15 @@ import click -from fre.pp import checkout_script -from fre.pp import configure_script_yaml -from fre.pp import configure_script_xml -#from fre.pp import validate -from fre.pp import install -from fre.pp import run -from fre.pp import trigger -from fre.pp import status -from fre.pp import wrapper +from fre.pp import checkout_script +from fre.pp import configure_script_yaml +from fre.pp import configure_script_xml +from fre.pp import validate_script +from fre.pp import install_script +from fre.pp import run_script +from fre.pp import trigger_script +from fre.pp import status_script +from fre.pp import wrapper_script @click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210))) def pp_cli(): @@ -32,7 +32,7 @@ def pp_cli(): def status(context, experiment, platform, target): # pylint: disable=unused-argument """ - Report status of PP configuration""" - context.forward(status.status_subtool) + context.forward(status_script.status_subtool) # fre pp run @pp_cli.command() @@ -49,7 +49,7 @@ def status(context, experiment, platform, target): def run(context, experiment, platform, target): # pylint: disable=unused-argument """ - Run PP configuration""" - context.forward(run.pp_run_subtool) + context.forward(run_script.pp_run_subtool) # fre pp validate @pp_cli.command() @@ -66,7 +66,7 @@ def run(context, experiment, platform, target): def validate(context, experiment, platform, target): # pylint: disable=unused-argument """ - Validate PP configuration""" - context.forward(_validate_subtool) + context.forward(validate_script._validate_subtool) # fre pp install @pp_cli.command() @@ -83,7 +83,7 @@ def validate(context, experiment, platform, target): def install(context, experiment, platform, target): # pylint: disable=unused-argument """ - Install PP configuration""" - context.forward(install.install_subtool) + context.forward(install_script.install_subtool) @pp_cli.command() @click.option("-y", "--yamlfile", type=str, @@ -102,7 +102,7 @@ def install(context, experiment, platform, target): def configure_yaml(context,yamlfile,experiment,platform,target): # pylint: disable=unused-argument """ - Execute fre pp configure """ - context.forward(configure_script_yaml._yamlInfo) + context.forward(configure_script_yaml._yaml_info) @pp_cli.command() @click.option("-e", "--experiment", type=str, @@ -114,8 +114,8 @@ def configure_yaml(context,yamlfile,experiment,platform,target): @click.option("-t", "--target", type=str, help="Target name", required=True) -@click.option("-b", "--branch", - required=False, +@click.option("-b", "--branch", type =str, + required=False, default = None, help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.pass_context def checkout(context, experiment, platform, target, branch=None): @@ -153,11 +153,11 @@ def checkout(context, experiment, platform, target, branch=None): is_flag=True, default=False, help="Optional. Process refineDiag scripts") -@click.option('--pp_start', +@click.option('--pp_start', type=str, default='0000', help="Optional. Starting year of postprocessing. " \ "If not specified, a default value of '0000' " \ "will be set and must be changed in rose-suite.conf") -@click.option('--pp_stop', +@click.option('--pp_stop', type=str, default='0000', help="Optional. Ending year of postprocessing. " \ "If not specified, a default value of '0000' " \ "will be set and must be changed in rose-suite.conf") @@ -195,17 +195,19 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo @click.option("-c", "--config-file", type=str, help="Path to a configuration file in either XML or YAML", required=True) -@click.option("-t", "--time", - required=False, - help="Time whose history files are ready") @click.option("-b", "--branch", - required=False, + required=False, default=None, help="fre-workflows branch/tag to clone; default is $(fre --version)") +@click.option("-t", "--time", + required=False, default=None, + help="Time whose history files are ready") @click.pass_context -def wrapper(context, experiment, platform, target, config_file, time=None, branch=None): +def wrapper(context, experiment, platform, target, config_file, branch, time): # pylint: disable=unused-argument """ - Execute fre pp steps in order """ - context.forward(wrapper.runFre2pp) + print(f'(frepp.wrapper) about to foward context to wrapper.run_all_fre_pp_steps via click...') + context.forward(wrapper_script._run_all_fre_pp_steps) + print(f'(frepp.wrapper) done fowarding context to wrapper.run_all_fre_pp_steps via click.') @pp_cli.command() @click.option("-e", "--experiment", type=str, @@ -224,7 +226,7 @@ def wrapper(context, experiment, platform, target, config_file, time=None, branc def trigger(context, experiment, platform, target, time): # pylint: disable=unused-argument """ - Start postprocessing for a particular time """ - context.forward(trigger._trigger) + context.forward(trigger_script._trigger) if __name__ == "__main__": ''' entry point for click to fre pp commands ''' diff --git a/fre/pp/install.py b/fre/pp/install_script.py similarity index 100% rename from fre/pp/install.py rename to fre/pp/install_script.py diff --git a/fre/pp/run.py b/fre/pp/run_script.py similarity index 81% rename from fre/pp/run.py rename to fre/pp/run_script.py index d50c9f01..b6ca607a 100644 --- a/fre/pp/run.py +++ b/fre/pp/run_script.py @@ -16,4 +16,8 @@ def pp_run_subtool(experiment, platform, target): @click.command() def _pp_run_subtool(experiment, platform, target): ''' entry point to run for click ''' - return _pp_run_subtool(experiment, platform, target) + return pp_run_subtool(experiment, platform, target) + + +if __name__ == "__main__": + pp_run_subtool() diff --git a/fre/pp/status.py b/fre/pp/status_script.py similarity index 82% rename from fre/pp/status.py rename to fre/pp/status_script.py index 0631be7a..e7c04f4f 100644 --- a/fre/pp/status.py +++ b/fre/pp/status_script.py @@ -17,4 +17,7 @@ def status_subtool(experiment, platform, target): @click.command() def _status_subtool(experiment, platform, target): ''' entry point to status for click ''' - return _status_subtool(experiment, platform, target) + return status_subtool(experiment, platform, target) + +if __name__ == "__main__": + status_subtool() diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index eaf1fc2e..e6391513 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -37,7 +37,7 @@ def test_configure_script(): model_yaml = str(Path(f"{test_dir}/{test_yaml}")) # Invoke configure_yaml_script.py - csy.yamlInfo(model_yaml,EXPERIMENT,PLATFORM,TARGET) + csy.yaml_info(model_yaml,EXPERIMENT,PLATFORM,TARGET) # Check for configuration creation and final combined yaml assert all([Path(f"{out_dir}/{EXPERIMENT}.yaml").exists(), diff --git a/fre/pp/tests/test_rose_quoting.py b/fre/pp/tests/test_rose_quoting.py index bd5353b1..13b3141a 100644 --- a/fre/pp/tests/test_rose_quoting.py +++ b/fre/pp/tests/test_rose_quoting.py @@ -1,7 +1,10 @@ +''' quick tests to make sure rose handles certain types of values with quotes correctly ''' from fre.pp.configure_script_yaml import quote_rose_values def test_boolean(): + ''' check that boolean values with quotes are handled correctly by rose''' assert quote_rose_values(True) == 'True' def test_string(): + ''' check that string values with quotes are handled correctly by rose''' assert quote_rose_values('foo') == "'foo'" diff --git a/fre/pp/trigger.py b/fre/pp/trigger_script.py similarity index 92% rename from fre/pp/trigger.py rename to fre/pp/trigger_script.py index e6ebd01b..14ce4291 100644 --- a/fre/pp/trigger.py +++ b/fre/pp/trigger_script.py @@ -17,3 +17,6 @@ def trigger(experiment, platform, target, time): def _trigger(experiment, platform, target, time): ''' entry point to trigger for click ''' return trigger(experiment, platform, target, time) + +if __name__ == "__main__": + trigger() diff --git a/fre/pp/validate.py b/fre/pp/validate_script.py similarity index 94% rename from fre/pp/validate.py rename to fre/pp/validate_script.py index fd54c52c..d48f5f47 100644 --- a/fre/pp/validate.py +++ b/fre/pp/validate_script.py @@ -28,3 +28,6 @@ def validate_subtool(experiment, platform, target): def _validate_subtool(experiment, platform, target): ''' entry point to validate for click ''' return validate_subtool(experiment, platform, target) + +if __name__ == "__main__": + validate_subtool() diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py deleted file mode 100644 index c9d6793b..00000000 --- a/fre/pp/wrapper.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -frepp.py, a replacement for the frepp bash script located at: -https://gitlab.gfdl.noaa.gov/fre2/system-settings/-/blob/main/bin/frepp -Author: Carolyn.Whitlock -""" - -#todo: -# add relative path import to rest of pp tools -# add command-line args using same format as fre.py -# include arg for pp start / stop -# test yaml path -# error handling - -import os -import time -import click - -# Import from the local packages -from .checkout_script import checkout_template -from .configure_script_yaml import yamlInfo -from .install import install_subtool -from .run import pp_run_subtool -from .trigger import trigger -from .status import status_subtool - -@click.command() -def runFre2pp(experiment, platform, target, config_file, branch, time): - ''' - Wrapper script for calling a FRE2 pp experiment with the canopy-style - infrastructure and fre-cli - ''' - - config_file = os.path.abspath(config_file) - - checkout_template(experiment, platform, target, branch) - - yamlInfo(config_file, experiment, platform, target) - - install_subtool(experiment, platform, target) - - pp_run_subtool(experiment, platform, target) - - if time: - trigger(experiment, platform, target, time) - - status_subtool(experiment, platform, target) - -if __name__ == '__main__': - runFre2pp() diff --git a/fre/pp/wrapper_script.py b/fre/pp/wrapper_script.py new file mode 100644 index 00000000..15a48dea --- /dev/null +++ b/fre/pp/wrapper_script.py @@ -0,0 +1,64 @@ +""" +frepp.py, a replacement for the frepp bash script located at: +https://gitlab.gfdl.noaa.gov/fre2/system-settings/-/blob/main/bin/frepp +Author: Carolyn.Whitlock +""" + +# add relative path import to rest of pp tools +# add command-line args using same format as fre.py +# include arg for pp start / stop +# test yaml path +# error handling + +import os +#import time +import click + +# Import from the local packages +from fre.pp.checkout_script import checkout_template +from fre.pp.configure_script_yaml import yaml_info +from fre.pp.install_script import install_subtool +from fre.pp.run_script import pp_run_subtool +from fre.pp.trigger_script import trigger +from fre.pp.status_script import status_subtool + +def run_all_fre_pp_steps(experiment, platform, target, config_file, branch=None, time=None): + ''' + Wrapper script for calling a FRE2 pp experiment with the canopy-style + infrastructure and fre-cli + ''' + print('(run_all_fre_pp_steps) config_file path resolving...') + config_file = os.path.abspath(config_file) + print(f' config_file={config_file}') + + print('(run_all_fre_pp_steps) calling checkout_template') + checkout_template(experiment, platform, target, branch) + + print('(run_all_fre_pp_steps) calling yaml_info') + yaml_info(config_file, experiment, platform, target) + + print('(run_all_fre_pp_steps) calling install_subtool') + install_subtool(experiment, platform, target) + + print('(run_all_fre_pp_steps) calling pp_run_subtool') + pp_run_subtool(experiment, platform, target) + + if time is not None: + print('(run_all_fre_pp_steps) calling trigger') + trigger(experiment, platform, target, time) + + print('(run_all_fre_pp_steps) calling status_subtool') + status_subtool(experiment, platform, target) + + print('(run_all_fre_pp_steps) done.') + + +@click.command() +def _run_all_fre_pp_steps(experiment, platform, target, config_file, branch, time): + ''' + click entry point for run_all_fre_pp_steps. + ''' + return run_all_fre_pp_steps(experiment, platform, target, config_file, branch, time) + +if __name__ == '__main__': + run_all_fre_pp_steps() From 5503fee5e0efee77268561dc853ddf62c84f38fc Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 10:54:06 -0500 Subject: [PATCH 097/111] adjust workflow run condition. --- .github/workflows/create_test_conda_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 7c353cfc..18ff20f0 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -1,6 +1,6 @@ name: create_test_conda_env -on: [push, pull_request] +on: [pull_request] jobs: build-linux: From ab50f685e6aaad205fd7a2888a877534aeab4e0a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 11:09:50 -0500 Subject: [PATCH 098/111] unused arguments, a doc string here and there --- fre/__init__.py | 5 +++++ fre/app/freapp.py | 1 + fre/make/fremake.py | 5 +++++ fre/yamltools/combine_yamls.py | 7 +++++-- fre/yamltools/freyamltools.py | 1 + 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/fre/__init__.py b/fre/__init__.py index 4524edb3..bfc4bfe1 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -1,3 +1,8 @@ +''' +initialization file for the fre-cli module. +sets __version__ to 'YYYY.XX' +''' + # turn xxxx.y into xxxx.0y import importlib.metadata version_unexpanded = importlib.metadata.version('fre-cli') diff --git a/fre/app/freapp.py b/fre/app/freapp.py index 545356e4..29c34385 100644 --- a/fre/app/freapp.py +++ b/fre/app/freapp.py @@ -58,6 +58,7 @@ def app_cli(): def regrid(context, input_dir, output_dir, begin, tmp_dir, remap_dir, source, grid_spec, def_xy_interp ): + # pylint: disable=unused-argument ''' regrid target netcdf file ''' context.forward(_regrid_xy) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index f39a6be0..a1fb30b4 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -72,6 +72,7 @@ def make_cli(): help = verbose_opt_help) @click.pass_context def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): + # pylint: disable=unused-argument """ - Perform all fremake functions to run checkout and compile model""" context.forward(runFremake._fremake_run) @@ -113,6 +114,7 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel help = verbose_opt_help) @click.pass_context def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): + # pylint: disable=unused-argument """ - Write the checkout script """ context.forward(createCheckout._checkout_create) @@ -135,6 +137,7 @@ def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,e required = True) @click.pass_context def create_makefile(context,yamlfile,platform,target): + # pylint: disable=unused-argument """ - Write the makefile """ context.forward(createMakefile._makefile_create) @@ -177,6 +180,7 @@ def create_makefile(context,yamlfile,platform,target): help = verbose_opt_help) @click.pass_context def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose): + # pylint: disable=unused-argument """ - Write the compile script """ context.forward(createCompile._compile_create) @@ -201,6 +205,7 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos help = "Build Dockerfile that has been generated by create-docker.") @click.pass_context def create_dockerfile(context,yamlfile,platform,target,execute): + # pylint: disable=unused-argument """ - Write the dockerfile """ context.forward(createDocker._dockerfile_create) diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index 006bc0d6..080a4e86 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -94,7 +94,7 @@ def experiment_check(mainyaml_dir,comb,experiment): ey=Path(os.path.join(mainyaml_dir,e)) ey_path.append(ey) else: - raise ValueError(f"Incorrect experiment yaml path given ({e}); does not exist.") + raise ValueError(f"Experiment yaml path given ({e}) does not exist.") else: raise ValueError("No experiment yaml path given!") @@ -115,6 +115,7 @@ def experiment_check(mainyaml_dir,comb,experiment): ## COMPILE CLASS ## class init_compile_yaml(): + """ class holding routines for initalizing compilation yamls """ def __init__(self,yamlfile,platform,target): """ Process to combine yamls applicable to compilation @@ -132,7 +133,8 @@ def __init__(self,yamlfile,platform,target): self.mainyaml_dir = os.path.dirname(self.yml) # Name of the combined yaml - self.combined= f"combined-{self.namenopath}.yaml" if len(self.mainyaml_dir) == 0 else f"{self.mainyaml_dir}/combined-{self.namenopath}.yaml" + base_name=f"combined-{self.namenopath}.yaml" + self.combined = base_name if len(self.mainyaml_dir) == 0 else f"{self.mainyaml_dir}/{base_name}" print("Combining yaml files: ") @@ -206,6 +208,7 @@ def clean_yaml(self): ## PP CLASS ## class init_pp_yaml(): + """ class holding routines for initalizing post-processing yamls """ def __init__(self,yamlfile,experiment,platform,target): """ Process to combine the applicable yamls for post-processing diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 55817472..97a06721 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -33,6 +33,7 @@ def yamltools_cli(): required=True) @click.pass_context def combine_yamls(context,yamlfile,experiment,platform,target,use): + # pylint: disable=unused-argument """ - Combine the model yaml with the compile, platform, experiment, and analysis yamls From 97719cdd0694b98b61c1a187d3a8e5c0d8aa43ca Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 11:14:12 -0500 Subject: [PATCH 099/111] timeout time now at top of script in constant var notation to avoid pylint shaming --- fre/pp/status_script.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fre/pp/status_script.py b/fre/pp/status_script.py index e7c04f4f..b4f579c7 100644 --- a/fre/pp/status_script.py +++ b/fre/pp/status_script.py @@ -3,6 +3,8 @@ import subprocess import click +TIMEOUT_SECS=120#30 + def status_subtool(experiment, platform, target): """ Report workflow state for the Cylc workflow @@ -11,7 +13,7 @@ def status_subtool(experiment, platform, target): name = experiment + '__' + platform + '__' + target cmd = f"cylc workflow-state {name}" - subprocess.run(cmd, shell=True, check=True, timeout=30) + subprocess.run(cmd, shell=True, check=True, timeout=TIMEOUT_SECS) @click.command() From 919ebe5615d681a1444d38e1513bc88840334830 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 11:22:13 -0500 Subject: [PATCH 100/111] fix indentation for combine_yamls classes and functions --- fre/yamltools/combine_yamls.py | 354 ++++++++++++++++----------------- 1 file changed, 177 insertions(+), 177 deletions(-) diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index 080a4e86..6deb9ae2 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -115,192 +115,192 @@ def experiment_check(mainyaml_dir,comb,experiment): ## COMPILE CLASS ## class init_compile_yaml(): - """ class holding routines for initalizing compilation yamls """ - def __init__(self,yamlfile,platform,target): - """ - Process to combine yamls applicable to compilation - """ - self.yml = yamlfile - self.name = yamlfile.split(".")[0] - self.namenopath = self.name.split("/")[-1].split(".")[0] - self.platform = platform - self.target = target + """ class holding routines for initalizing compilation yamls """ + def __init__(self,yamlfile,platform,target): + """ + Process to combine yamls applicable to compilation + """ + self.yml = yamlfile + self.name = yamlfile.split(".")[0] + self.namenopath = self.name.split("/")[-1].split(".")[0] + self.platform = platform + self.target = target + + # Register tag handler + yaml.add_constructor('!join', join_constructor) + + # Path to the main model yaml + self.mainyaml_dir = os.path.dirname(self.yml) + + # Name of the combined yaml + base_name=f"combined-{self.namenopath}.yaml" + self.combined = base_name if len(self.mainyaml_dir) == 0 else f"{self.mainyaml_dir}/{base_name}" + + print("Combining yaml files: ") + + def combine_model(self): + """ + Create the combined.yaml and merge it with the model yaml + """ + # copy model yaml info into combined yaml + with open(self.combined,'w+',encoding='UTF-8') as f1: + f1.write(f'name: &name "{self.name}"\n') + f1.write(f'platform: &platform "{self.platform}"\n') + f1.write(f'target: &target "{self.target}"\n\n') + try: + with open(self.yml,'r',encoding='UTF-8') as f2: + f1.write("### MODEL YAML SETTINGS ###\n") + shutil.copyfileobj(f2,f1) + except Exception as exc: + raise FileNotFoundError(f'{self.yml} not found') from exc + print(f" model yaml: {self.yml}") + + def combine_compile(self): + """ + Combine compile yaml with the defined combined.yaml + """ + # Get compile info + (py_path,cy_path) = get_compile_paths(self.mainyaml_dir,self.combined) + + # copy compile yaml info into combined yaml + if cy_path is not None: + with open(self.combined,'a',encoding='UTF-8') as f1: + with open(cy_path,'r',encoding='UTF-8') as f2: + f1.write("\n### COMPILE INFO ###\n") + shutil.copyfileobj(f2,f1) + print(f" compile yaml: {cy_path}") - # Register tag handler - yaml.add_constructor('!join', join_constructor) + def combine_platforms(self): + """ + Combine platforms yaml with the defined combined.yaml + """ + # Get compile info + (py_path,cy_path) = get_compile_paths(self.mainyaml_dir,self.combined) - # Path to the main model yaml - self.mainyaml_dir = os.path.dirname(self.yml) + # combine platform yaml + if py_path is not None: + with open(self.combined,'a',encoding='UTF-8') as f1: + with open(py_path,'r',encoding='UTF-8') as f2: + f1.write("\n### PLATFORM INFO ###\n") + shutil.copyfileobj(f2,f1) + print(f" platforms yaml: {py_path}") - # Name of the combined yaml - base_name=f"combined-{self.namenopath}.yaml" - self.combined = base_name if len(self.mainyaml_dir) == 0 else f"{self.mainyaml_dir}/{base_name}" + def clean_yaml(self): + """ + Clean the yaml; remove unnecessary sections in + final combined yaml. + """ + # Load the fully combined yaml + full_yaml = yaml_load(self.combined) - print("Combining yaml files: ") + # Clean the yaml + # If keys exists, delete: + keys_clean=["fre_properties", "shared", "experiments"] + for kc in keys_clean: + if kc in full_yaml.keys(): + del full_yaml[kc] - def combine_model(self): - """ - Create the combined.yaml and merge it with the model yaml - """ - # copy model yaml info into combined yaml - with open(self.combined,'w+',encoding='UTF-8') as f1: - f1.write(f'name: &name "{self.name}"\n') - f1.write(f'platform: &platform "{self.platform}"\n') - f1.write(f'target: &target "{self.target}"\n\n') - try: - with open(self.yml,'r',encoding='UTF-8') as f2: - f1.write("### MODEL YAML SETTINGS ###\n") - shutil.copyfileobj(f2,f1) - except Exception as exc: - raise FileNotFoundError(f'{self.yml} not found') from exc - print(f" model yaml: {self.yml}") - - def combine_compile(self): - """ - Combine compile yaml with the defined combined.yaml - """ - # Get compile info - (py_path,cy_path) = get_compile_paths(self.mainyaml_dir,self.combined) - - # copy compile yaml info into combined yaml - if cy_path is not None: - with open(self.combined,'a',encoding='UTF-8') as f1: - with open(cy_path,'r',encoding='UTF-8') as f2: - f1.write("\n### COMPILE INFO ###\n") - shutil.copyfileobj(f2,f1) - print(f" compile yaml: {cy_path}") - - def combine_platforms(self): - """ - Combine platforms yaml with the defined combined.yaml - """ - # Get compile info - (py_path,cy_path) = get_compile_paths(self.mainyaml_dir,self.combined) - - # combine platform yaml - if py_path is not None: - with open(self.combined,'a',encoding='UTF-8') as f1: - with open(py_path,'r',encoding='UTF-8') as f2: - f1.write("\n### PLATFORM INFO ###\n") - shutil.copyfileobj(f2,f1) - print(f" platforms yaml: {py_path}") - - def clean_yaml(self): - """ - Clean the yaml; remove unnecessary sections in - final combined yaml. - """ - # Load the fully combined yaml - full_yaml = yaml_load(self.combined) - - # Clean the yaml - # If keys exists, delete: - keys_clean=["fre_properties", "shared", "experiments"] - for kc in keys_clean: - if kc in full_yaml.keys(): - del full_yaml[kc] - - with open(self.combined,'w',encoding='UTF-8') as f: - yaml.safe_dump(full_yaml,f,default_flow_style=False,sort_keys=False) - - print(f"Combined yaml located here: {os.path.dirname(self.combined)}/{self.combined}") - return self.combined + with open(self.combined,'w',encoding='UTF-8') as f: + yaml.safe_dump(full_yaml,f,default_flow_style=False,sort_keys=False) + + print(f"Combined yaml located here: {os.path.dirname(self.combined)}/{self.combined}") + return self.combined ## PP CLASS ## class init_pp_yaml(): - """ class holding routines for initalizing post-processing yamls """ - def __init__(self,yamlfile,experiment,platform,target): - """ - Process to combine the applicable yamls for post-processing - """ - self.yml = yamlfile - self.name = experiment - self.platform = platform - self.target = target - - # Regsiter tag handler - yaml.add_constructor('!join', join_constructor) - - # Path to the main model yaml - self.mainyaml_dir = os.path.dirname(self.yml) - - # Name of the combined yaml - self.combined=f"combined-{self.name}.yaml" - - print("Combining yaml files: ") - - def combine_model(self): - """ - Create the combined.yaml and merge it with the model yaml - """ - # copy model yaml info into combined yaml - with open(self.combined,'w+',encoding='UTF-8') as f1: - f1.write(f'name: &name "{self.name}"\n') - f1.write(f'platform: &platform "{self.platform}"\n') - f1.write(f'target: &target "{self.target}"\n\n') - try: - with open(self.yml,'r',encoding='UTF-8') as f2: - f1.write("### MODEL YAML SETTINGS ###\n") - shutil.copyfileobj(f2,f1) - except Exception as exc: - raise FileNotFoundError(f'{self.yml} not found') from exc - print(f" model yaml: {self.yml}") - - def combine_experiment(self): - """ - Combine experiment yamls with the defined combined.yaml - """ - # Experiment Check - (ey_path,ay_path) = experiment_check(self.mainyaml_dir,self.combined,self.name) - - ## COMBINE EXPERIMENT YAML INFO - if ey_path is not None: - for i in ey_path: - #expyaml_path = os.path.join(mainyaml_dir, i) - with open(self.combined,'a',encoding='UTF-8') as f1: - with open(i,'r',encoding='UTF-8') as f2: - #copy expyaml into combined - shutil.copyfileobj(f2,f1) - print(f" experiment yaml: {i}") - - def combine_analysis(self): - """ - Combine analysis yamls with the defined combined.yaml - """ - # Experiment Check - (ey_path,ay_path) = experiment_check(self.mainyaml_dir,self.combined,self.name) - - ## COMBINE EXPERIMENT YAML INFO - if ay_path is not None: - for i in ay_path: - #analysisyaml_path = os.path.join(mainyaml_dir, i) - with open(self.combined,'a',encoding='UTF-8') as f1: - with open(i,'r',encoding='UTF-8') as f2: - #f1.write(f"\n### {i.upper()} settings ###\n") - #copy expyaml into combined + """ class holding routines for initalizing post-processing yamls """ + def __init__(self,yamlfile,experiment,platform,target): + """ + Process to combine the applicable yamls for post-processing + """ + self.yml = yamlfile + self.name = experiment + self.platform = platform + self.target = target + + # Regsiter tag handler + yaml.add_constructor('!join', join_constructor) + + # Path to the main model yaml + self.mainyaml_dir = os.path.dirname(self.yml) + + # Name of the combined yaml + self.combined=f"combined-{self.name}.yaml" + + print("Combining yaml files: ") + + def combine_model(self): + """ + Create the combined.yaml and merge it with the model yaml + """ + # copy model yaml info into combined yaml + with open(self.combined,'w+',encoding='UTF-8') as f1: + f1.write(f'name: &name "{self.name}"\n') + f1.write(f'platform: &platform "{self.platform}"\n') + f1.write(f'target: &target "{self.target}"\n\n') + try: + with open(self.yml,'r',encoding='UTF-8') as f2: + f1.write("### MODEL YAML SETTINGS ###\n") shutil.copyfileobj(f2,f1) - print(f" analysis yaml: {i}") - - def clean_yaml(self): - """ - Clean the yaml; remove unnecessary sections in - final combined yaml. - """ - # Load the fully combined yaml - full_yaml = yaml_load(self.combined) - - # Clean the yaml - # If keys exists, delete: - keys_clean=["fre_properties", "shared", "experiments"] - for kc in keys_clean: - if kc in full_yaml.keys(): - del full_yaml[kc] - - with open(self.combined,'w') as f: - yaml.safe_dump(full_yaml,f,default_flow_style=False,sort_keys=False) - - print(f"Combined yaml located here: {os.path.dirname(self.combined)}/{self.combined}") - return self.combined + except Exception as exc: + raise FileNotFoundError(f'{self.yml} not found') from exc + print(f" model yaml: {self.yml}") + + def combine_experiment(self): + """ + Combine experiment yamls with the defined combined.yaml + """ + # Experiment Check + (ey_path,ay_path) = experiment_check(self.mainyaml_dir,self.combined,self.name) + + ## COMBINE EXPERIMENT YAML INFO + if ey_path is not None: + for i in ey_path: + #expyaml_path = os.path.join(mainyaml_dir, i) + with open(self.combined,'a',encoding='UTF-8') as f1: + with open(i,'r',encoding='UTF-8') as f2: + #copy expyaml into combined + shutil.copyfileobj(f2,f1) + print(f" experiment yaml: {i}") + + def combine_analysis(self): + """ + Combine analysis yamls with the defined combined.yaml + """ + # Experiment Check + (ey_path,ay_path) = experiment_check(self.mainyaml_dir,self.combined,self.name) + + ## COMBINE EXPERIMENT YAML INFO + if ay_path is not None: + for i in ay_path: + #analysisyaml_path = os.path.join(mainyaml_dir, i) + with open(self.combined,'a',encoding='UTF-8') as f1: + with open(i,'r',encoding='UTF-8') as f2: + #f1.write(f"\n### {i.upper()} settings ###\n") + #copy expyaml into combined + shutil.copyfileobj(f2,f1) + print(f" analysis yaml: {i}") + + def clean_yaml(self): + """ + Clean the yaml; remove unnecessary sections in + final combined yaml. + """ + # Load the fully combined yaml + full_yaml = yaml_load(self.combined) + + # Clean the yaml + # If keys exists, delete: + keys_clean=["fre_properties", "shared", "experiments"] + for kc in keys_clean: + if kc in full_yaml.keys(): + del full_yaml[kc] + + with open(self.combined,'w') as f: + yaml.safe_dump(full_yaml,f,default_flow_style=False,sort_keys=False) + + print(f"Combined yaml located here: {os.path.dirname(self.combined)}/{self.combined}") + return self.combined ## Functions to combine the yaml files ## def get_combined_compileyaml(comb): From a7f3ee947bc0860b9693b04fcfd857105ba7d2b6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 27 Nov 2024 11:42:13 -0500 Subject: [PATCH 101/111] #216 Wait for process1 to finish before checking return code --- fre/make/gfdlfremake/buildBaremetal.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fre/make/gfdlfremake/buildBaremetal.py b/fre/make/gfdlfremake/buildBaremetal.py index a6a9c208..71c81bab 100644 --- a/fre/make/gfdlfremake/buildBaremetal.py +++ b/fre/make/gfdlfremake/buildBaremetal.py @@ -140,10 +140,14 @@ def run(self): # Direct output to log file as well p2 = subprocess.Popen(["tee",self.bld+"/log.compile"], stdin=p1.stdout) - # Allow p1 to receive SIGPIPE is p2 exits + # Allow process1 to receive SIGPIPE is process2 exits p1.stdout.close() p2.communicate() + # wait for process1 to finish before checking return code + p1.wait() if p1.returncode != 0: print(f"\nThere was an error running {self.bld}/compile.sh") print(f"Check the log file: {self.bld}/log.compile") + else: + print(f"\nSuccessful run of {self.bld}/compile.sh") From 8fa356beb20f8cf6e75d64b6dd09182248fa6de5 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Wed, 27 Nov 2024 12:11:00 -0500 Subject: [PATCH 102/111] add output for checkout and makefile creation for runfremake --- fre/make/runFremake.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 17e1c28b..9e1c1730 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -88,6 +88,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) freCheckout.finish(pc) os.chmod(srcDir+"/checkout.sh", 0o744) + print("\nCheckout script created at "+ srcDir + "/checkout.sh \n") ## TODO: Options for running on login cluster? freCheckout.run() @@ -127,6 +128,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec # Loop through components, send component name/requires/overrides for Makefile for c in fremakeYaml['src']: freMakefile.addComponent(c['component'],c['requires'],c['makeOverrides']) + print("\nMakefile created at " + bldDir + "/Makefile" + "\n") freMakefile.writeMakefile() ## Create a list of compile scripts to run in parallel From 2c887f39d83bb9402595d460a026a8e16c3dce0d Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 12:33:34 -0500 Subject: [PATCH 103/111] attempting to address some bad indentation and variable names. --- fre/make/createCheckout.py | 116 ----------------- fre/make/create_checkout_script.py | 119 ++++++++++++++++++ ...ateCompile.py => create_compile_script.py} | 0 ...reateDocker.py => create_docker_script.py} | 0 ...eMakefile.py => create_makefile_script.py} | 0 fre/make/fremake.py | 86 ++++++------- .../{runFremake.py => run_fremake_script.py} | 0 7 files changed, 162 insertions(+), 159 deletions(-) delete mode 100644 fre/make/createCheckout.py create mode 100644 fre/make/create_checkout_script.py rename fre/make/{createCompile.py => create_compile_script.py} (100%) rename fre/make/{createDocker.py => create_docker_script.py} (100%) rename fre/make/{createMakefile.py => create_makefile_script.py} (100%) rename fre/make/{runFremake.py => run_fremake_script.py} (100%) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py deleted file mode 100644 index fc4d2df4..00000000 --- a/fre/make/createCheckout.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/python3 - -import os -import subprocess -import logging -import sys -import click -import fre.yamltools.combine_yamls as cy -from .gfdlfremake import varsfre, yamlfre, checkout, targetfre - -def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): - # Define variables - yml = yamlfile - name = yamlfile.split(".")[0] - run = execute - jobs = str(jobs) - pcheck = no_parallel_checkout - - if pcheck: - pc = "" - else: - pc = " &" - - if verbose: - logging.basicConfig(level=logging.INFO) - else: - logging.basicConfig(level=logging.ERROR) - - srcDir="src" - checkoutScriptName = "checkout.sh" - baremetalRun = False # This is needed if there are no bare metal runs - - ## Split and store the platforms and targets in a list - plist = platform - tlist = target - - # Combine model, compile, and platform yamls - # Default behavior - combine yamls / rewrite combined yaml - comb = cy.init_compile_yaml(yml,platform,target) - full_combined = cy.get_combined_compileyaml(comb) - - ## Get the variables in the model yaml - freVars = varsfre.frevars(full_combined) - - ## Open the yaml file, validate the yaml, and parse as fremakeYaml - modelYaml = yamlfre.freyaml(full_combined,freVars) - fremakeYaml = modelYaml.getCompileYaml() - - ## Error checking the targets - for targetName in tlist: - target = targetfre.fretarget(targetName) - - ## Loop through the platforms specified on the command line - ## If the platform is a baremetal platform, write the checkout script and run it once - ## This should be done separately and serially because bare metal platforms should all be using - ## the same source code. - for platformName in plist: - if modelYaml.platforms.hasPlatform(platformName): - pass - else: - raise ValueError (platformName + " does not exist in platforms.yaml") - ( compiler, modules, modulesInit, fc, cc, modelRoot, - iscontainer, mkTemplate, containerBuild, ContainerRun, - RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName) - - ## Create the source directory for the platform - if iscontainer is False: - srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" - # if the source directory does not exist, it is created - if not os.path.exists(srcDir): - os.system("mkdir -p " + srcDir) - # if the checkout script does not exist, it is created - if not os.path.exists(srcDir+"/checkout.sh"): - freCheckout = checkout.checkout("checkout.sh",srcDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) - freCheckout.finish(pc) - # Make checkout script executable - os.chmod(srcDir+"/checkout.sh", 0o744) - print("\nCheckout script created in "+ srcDir + "/checkout.sh \n") - - # Run the checkout script - if run is True: - freCheckout.run() - else: - sys.exit() - else: - print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") - if run == True: - try: - subprocess.run(args=[srcDir+"/checkout.sh"], check=True) - except: - print("\nThere was an error with the checkout script "+srcDir+"/checkout.sh.", - "\nTry removing test folder: " + modelRoot +"\n") - raise - else: - sys.exit() - - else: - image="ecpe4s/noaa-intel-prototype:2023.09.25" - bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec" - tmpDir = "tmp/"+platformName - freCheckout = checkout.checkoutForContainer("checkout.sh", srcDir, tmpDir) - freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc) - freCheckout.finish(pc) - print("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n") - -@click.command() -def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): - ''' - Decorator for calling checkout_create - allows the decorated version - of the function to be separate from the undecorated version - ''' - return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) - -if __name__ == "__main__": - checkout_create() diff --git a/fre/make/create_checkout_script.py b/fre/make/create_checkout_script.py new file mode 100644 index 00000000..6d5bb2aa --- /dev/null +++ b/fre/make/create_checkout_script.py @@ -0,0 +1,119 @@ +''' +checks out a makefile for a given model from the yamls +i think! +''' + +import os +import subprocess +import logging +import sys +import click +import fre.yamltools.combine_yamls as cy +from .gfdlfremake import varsfre, yamlfre, checkout, targetfre + +def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): + # Define variables + yml = yamlfile + name = yamlfile.split(".")[0] + run = execute + jobs = str(jobs) + pcheck = no_parallel_checkout + + if pcheck: + pc = "" + else: + pc = " &" + + if verbose: + logging.basicConfig(level=logging.INFO) + else: + logging.basicConfig(level=logging.ERROR) + + src_dir="src" + checkout_script_name = "checkout.sh" + baremetal_run = False # This is needed if there are no bare metal runs + + ## Split and store the platforms and targets in a list + plist = platform + tlist = target + + # Combine model, compile, and platform yamls + # Default behavior - combine yamls / rewrite combined yaml + comb = cy.init_compile_yaml(yml,platform,target) + full_combined = cy.get_combined_compileyaml(comb) + + ## Get the variables in the model yaml + fre_vars = varsfre.frevars(full_combined) + + ## Open the yaml file, validate the yaml, and parse as fremake_yaml + model_yaml = yamlfre.freyaml(full_combined,fre_vars) + fremake_yaml = model_yaml.getCompileYaml() + + ## Error checking the targets + for target_name in tlist: + target = targetfre.fretarget(target_name) + + ## Loop through the platforms specified on the command line + ## If the platform is a baremetal platform, write the checkout script and run it once + ## This should be done separately and serially because bare metal platforms should all be using + ## the same source code. + for platform_name in plist: + if model_yaml.platforms.hasPlatform(platform_name): + pass + else: + raise ValueError (platform_name + " does not exist in platforms.yaml") + ( compiler, modules, modules_init, fc, cc, model_root, + iscontainer, mk_template, container_build, container_run, + RUNenv ) = model_yaml.platforms.getPlatformFromName(platform_name) + + # ceate the source directory for the platform + if not iscontainer: + src_dir = model_root + "/" + fremake_yaml["experiment"] + "/src" + # if the source directory does not exist, it is created + if not os.path.exists(src_dir): + os.system("mkdir -p " + src_dir) + # if the checkout script does not exist, it is created + if not os.path.exists(src_dir+"/checkout.sh"): + fre_checkout = checkout.checkout("checkout.sh",src_dir) + fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) + fre_checkout.finish(pc) + # Make checkout script executable + os.chmod(src_dir+"/checkout.sh", 0o744) + print("\nCheckout script created in "+ src_dir + "/checkout.sh \n") + + # Run the checkout script + if run: + fre_checkout.run() + else: + sys.exit() + else: + print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n") + if run: + try: + subprocess.run(args=[src_dir+"/checkout.sh"], check=True) + except: + print("\nThere was an error with the checkout script "+src_dir+"/checkout.sh.", + "\nTry removing test folder: " + model_root +"\n") + raise + else: + sys.exit() + + else: + image="ecpe4s/noaa-intel-prototype:2023.09.25" + bld_dir = model_root + "/" + fremake_yaml["experiment"] + "/exec" + tmp_dir = "tmp/"+platform_name + fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir) + fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc) + fre_checkout.finish(pc) + print("\nCheckout script created at " + tmp_dir + "/checkout.sh" + "\n") + +@click.command() +def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): + ''' + Decorator for calling checkout_create - allows the decorated version + of the function to be separate from the undecorated version + ''' + return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) + +if __name__ == "__main__": + checkout_create() diff --git a/fre/make/createCompile.py b/fre/make/create_compile_script.py similarity index 100% rename from fre/make/createCompile.py rename to fre/make/create_compile_script.py diff --git a/fre/make/createDocker.py b/fre/make/create_docker_script.py similarity index 100% rename from fre/make/createDocker.py rename to fre/make/create_docker_script.py diff --git a/fre/make/createMakefile.py b/fre/make/create_makefile_script.py similarity index 100% rename from fre/make/createMakefile.py rename to fre/make/create_makefile_script.py diff --git a/fre/make/fremake.py b/fre/make/fremake.py index a1fb30b4..5f7ab303 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -1,31 +1,31 @@ import click -from fre.make import createCheckout -from fre.make import createMakefile -from fre.make import createCompile -from fre.make import createDocker -from fre.make import runFremake +from fre.make import create_checkout_script +from fre.make import create_makefile_script +from fre.make import create_compile_script +from fre.make import create_docker_script +from fre.make import run_fremake_script -yamlfile_opt_help = """Experiment yaml compile FILE +YAMLFILE_OPT_HELP = """Experiment yaml compile FILE """ -experiment_opt_help = """Name of experiment""" -platform_opt_help = """Hardware and software FRE platform space separated list of STRING(s). +EXPERIMENT_OPT_HELP = """Name of experiment""" +PLATFORM_OPT_HELP = """Hardware and software FRE platform space separated list of STRING(s). This sets platform-specific data and instructions """ -target_opt_help = """a space separated list of STRING(s) that defines compilation settings and +TARGET_OPT_HELP = """a space separated list of STRING(s) that defines compilation settings and linkage directives for experiments. Predefined targets refer to groups of directives that exist in the mkmf template file (referenced in buildDocker.py). Possible predefined targets include 'prod', 'openmp', 'repro', 'debug, 'hdf5'; however 'prod', 'repro', and 'debug' are mutually exclusive (cannot not use more than one of these in the target list). Any number of targets can be used. """ -parallel_opt_help = """Number of concurrent model compiles (default 1) +PARALLEL_OPT_HELP = """Number of concurrent model compiles (default 1) """ -jobs_opt_help = """Number of jobs to run simultaneously. Used for make -jJOBS and git clone +JOBS_OPT_HELP = """Number of jobs to run simultaneously. Used for make -jJOBS and git clone recursive --jobs=JOBS """ -no_parallel_checkout_opt_help = """Use this option if you do not want a parallel checkout. +NO_PARALLEL_CHECKOUT_OPT_HELP = """Use this option if you do not want a parallel checkout. The default is to have parallel checkouts. """ -verbose_opt_help = """Get verbose messages (repeat the option to increase verbosity level) +VERBOSE_OPT_HELP = """Get verbose messages (repeat the option to increase verbosity level) """ @@ -38,72 +38,72 @@ def make_cli(): @click.option("-y", "--yamlfile", type = str, - help = yamlfile_opt_help, + help = YAMLFILE_OPT_HELP, required = True) # use click.option() over click.argument(), we want help statements @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = platform_opt_help, required = True) + help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = target_opt_help, + help = TARGET_OPT_HELP, required = True) @click.option("-n", "--parallel", type = int, metavar = '', default = 1, - help = parallel_opt_help) + help = PARALLEL_OPT_HELP) @click.option("-j", "--jobs", type = int, metavar = '', default = 4, - help = jobs_opt_help) + help = JOBS_OPT_HELP) @click.option("-npc", "--no-parallel-checkout", is_flag = True, - help = no_parallel_checkout_opt_help) + help = NO_PARALLEL_CHECKOUIT_OPT_HELP) @click.option("-v", "--verbose", is_flag = True, - help = verbose_opt_help) + help = VERBOSE_OPT_HELP) @click.pass_context def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): # pylint: disable=unused-argument """ - Perform all fremake functions to run checkout and compile model""" - context.forward(runFremake._fremake_run) + context.forward(run_fremake_script._fremake_run) #### @make_cli.command() @click.option("-y", "--yamlfile", type = str, - help = yamlfile_opt_help, + help = YAMLFILE_OPT_HELP, required = True) # use click.option() over click.argument(), we want help statements @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = platform_opt_help, + help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = target_opt_help, + help = TARGET_OPT_HELP, required = True) @click.option("-j", "--jobs", type = int, metavar = '', default = 4, - help = jobs_opt_help) + help = JOBS_OPT_HELP) @click.option("-npc", "--no-parallel-checkout", is_flag = True, - help = no_parallel_checkout_opt_help) + help = no_parallel_checkout_OPT_HELP) @click.option("--execute", is_flag = True, default = False, @@ -111,35 +111,35 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel @click.option("-v", "--verbose", is_flag = True, - help = verbose_opt_help) + help = VERBOSE_OPT_HELP) @click.pass_context def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): # pylint: disable=unused-argument """ - Write the checkout script """ - context.forward(createCheckout._checkout_create) + context.forward(create_checkout_script._checkout_create) ##### @make_cli.command @click.option("-y", "--yamlfile", type = str, - help = yamlfile_opt_help, + help = YAMLFILE_OPT_HELP, required = True) # use click.option() over click.argument(), we want help statements @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = platform_opt_help, required = True) + help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = target_opt_help, + help = TARGET_OPT_HELP, required = True) @click.pass_context def create_makefile(context,yamlfile,platform,target): # pylint: disable=unused-argument """ - Write the makefile """ - context.forward(createMakefile._makefile_create) + context.forward(create_makefile_script._makefile_create) ##### @@ -147,29 +147,29 @@ def create_makefile(context,yamlfile,platform,target): @click.option("-y", "--yamlfile", type = str, - help = yamlfile_opt_help, + help = YAMLFILE_OPT_HELP, required = True) # use click.option() over click.argument(), we want help statements @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = platform_opt_help, required = True) + help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = target_opt_help, + help = TARGET_OPT_HELP, required = True) @click.option("-j", "--jobs", type = int, metavar = '', default = 4, - help = jobs_opt_help) + help = JOBS_OPT_HELP) @click.option("-n", "--parallel", type = int, metavar = '', default = 1, - help = parallel_opt_help) + help = PARALLEL_OPT_HELP) @click.option("--execute", is_flag = True, default = False, @@ -177,28 +177,28 @@ def create_makefile(context,yamlfile,platform,target): @click.option("-v", "--verbose", is_flag = True, - help = verbose_opt_help) + help = VERBOSE_OPT_HELP) @click.pass_context def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose): # pylint: disable=unused-argument """ - Write the compile script """ - context.forward(createCompile._compile_create) + context.forward(create_compile_script._compile_create) @make_cli.command @click.option("-y", "--yamlfile", type = str, - help = yamlfile_opt_help, + help = YAMLFILE_OPT_HELP, required = True) # use click.option() over click.argument(), we want help statements @click.option("-p", "--platform", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = platform_opt_help, required = True) + help = PLATFORM_OPT_HELP, required = True) @click.option("-t", "--target", multiple = True, # replaces nargs = -1, since click.option() type = str, - help = target_opt_help, + help = TARGET_OPT_HELP, required = True) @click.option("--execute", is_flag = True, @@ -207,7 +207,7 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos def create_dockerfile(context,yamlfile,platform,target,execute): # pylint: disable=unused-argument """ - Write the dockerfile """ - context.forward(createDocker._dockerfile_create) + context.forward(create_docker_script._dockerfile_create) if __name__ == "__main__": make_cli() diff --git a/fre/make/runFremake.py b/fre/make/run_fremake_script.py similarity index 100% rename from fre/make/runFremake.py rename to fre/make/run_fremake_script.py From b1b0ec2eaec1154a4d9c321124ea9bccbd17cee5 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 12:40:46 -0500 Subject: [PATCH 104/111] fix imports --- fre/make/tests/compilation/test_fre_make_run_fremake.py | 4 ++-- fre/make/tests/test_create_makefile.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index ade1b4db..e6fecccb 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -5,7 +5,7 @@ import pytest -from fre.make import runFremake +from fre.make import run_fremake_script # command options @@ -21,6 +21,6 @@ @pytest.mark.skip(reason='failing: fix in development, see PR 275') def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' - runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) + run_fremake_script.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() diff --git a/fre/make/tests/test_create_makefile.py b/fre/make/tests/test_create_makefile.py index 1fe2001c..dd180262 100644 --- a/fre/make/tests/test_create_makefile.py +++ b/fre/make/tests/test_create_makefile.py @@ -4,7 +4,7 @@ import os import shutil from pathlib import Path -from fre.make import createMakefile +from fre.make import create_makefile_script # SET-UP TEST_DIR = Path("fre/make/tests") @@ -58,7 +58,7 @@ def test_bm_makefile_creation(): targ = TARGET[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - createMakefile.makefile_create(yamlfile_path,BM_PLATFORM,TARGET) + create_makefile_script.makefile_create(yamlfile_path,BM_PLATFORM,TARGET) assert Path(f"{OUT}/fremake_canopy/test/{EXPERIMENT}/{bm_plat}-{targ}/exec/Makefile").exists() os.environ["HOME"] = def_home @@ -70,6 +70,6 @@ def test_container_makefile_creation(): """ container_plat = CONTAINER_PLATFORM[0] yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}" - createMakefile.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET) + create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET) assert Path(f"tmp/{container_plat}/Makefile").exists() From c9ea70d0500a85b686f6556048760bcdb1840f93 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 12:45:33 -0500 Subject: [PATCH 105/111] typo fix --- fre/make/fremake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 5f7ab303..ba34ff64 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -65,7 +65,7 @@ def make_cli(): @click.option("-npc", "--no-parallel-checkout", is_flag = True, - help = NO_PARALLEL_CHECKOUIT_OPT_HELP) + help = NO_PARALLEL_CHECKOUT_OPT_HELP) @click.option("-v", "--verbose", is_flag = True, @@ -103,7 +103,7 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel @click.option("-npc", "--no-parallel-checkout", is_flag = True, - help = no_parallel_checkout_OPT_HELP) + help = NO_PARALLEL_CHECKOUT_OPT_HELP) @click.option("--execute", is_flag = True, default = False, From 894d9885d882f90753528477996dc7689fb05339 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 12:54:34 -0500 Subject: [PATCH 106/111] add positional argument for new execute flag --- fre/make/tests/compilation/test_fre_make_run_fremake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index e6fecccb..0b9f94b3 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -21,6 +21,6 @@ @pytest.mark.skip(reason='failing: fix in development, see PR 275') def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' - run_fremake_script.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) + run_fremake_script.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False, False) assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() From 999e45931ada9ea09b3a65753052a1f853c3789c Mon Sep 17 00:00:00 2001 From: Ryan Mulhall Date: Wed, 27 Nov 2024 13:10:14 -0500 Subject: [PATCH 107/111] add import to createDockerfile.py --- fre/make/createDocker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index 7dfefaa6..5731991e 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -2,6 +2,7 @@ import os import sys +import subprocess from pathlib import Path import click #from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker From 6a146a3287ab912036dae2d4ce2c89fa4af94ed1 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 13:30:28 -0500 Subject: [PATCH 108/111] versioning is not fun --- fre/__init__.py | 4 ++-- fre/fre.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/fre/__init__.py b/fre/__init__.py index bfc4bfe1..215c8142 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -11,5 +11,5 @@ version_minor = "0" + version_unexpanded_split[1] else: version_minor = version_unexpanded_split[1] -version = version_unexpanded_split[0] + '.' + version_minor -__version__ = version +__version__ = version_unexpanded_split[0] + '.' + version_minor + diff --git a/fre/fre.py b/fre/fre.py index 388a4588..8e5695c4 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -7,6 +7,21 @@ be called via this script. I.e. 'fre' is the entry point """ +#versioning... always fun... +# turn xxxx.y into xxxx.0y +import importlib.metadata +version_unexpanded = importlib.metadata.version('fre-cli') +version_unexpanded_split = version_unexpanded.split('.') +if len(version_unexpanded_split[1]) == 1: + version_minor = "0" + version_unexpanded_split[1] +else: + version_minor = version_unexpanded_split[1] +version = version_unexpanded_split[0] + '.' + version_minor + + + + + import click from .lazy_group import LazyGroup @@ -29,12 +44,12 @@ @click.version_option( - package_name = "fre-cli" + package_name = "fre-cli", + version=version ) def fre(): ''' entry point function to subgroup functions ''' - if __name__ == '__main__': fre() From 712e4eba1ae373b6a9ebc1d9788392197393b835 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 13:38:14 -0500 Subject: [PATCH 109/111] GOT the version info all in one place! --- fre/__init__.py | 15 --------------- fre/pp/checkout_script.py | 4 ++-- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/fre/__init__.py b/fre/__init__.py index 215c8142..e69de29b 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -1,15 +0,0 @@ -''' -initialization file for the fre-cli module. -sets __version__ to 'YYYY.XX' -''' - -# turn xxxx.y into xxxx.0y -import importlib.metadata -version_unexpanded = importlib.metadata.version('fre-cli') -version_unexpanded_split = version_unexpanded.split('.') -if len(version_unexpanded_split[1]) == 1: - version_minor = "0" + version_unexpanded_split[1] -else: - version_minor = version_unexpanded_split[1] -__version__ = version_unexpanded_split[0] + '.' + version_minor - diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index 339d8b21..e3e9e29b 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -11,7 +11,7 @@ import click -import fre +from fre import fre FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git' @@ -23,7 +23,7 @@ def checkout_template(experiment = None, platform = None, target = None, branch go_back_here = os.getcwd() # branch and version parameters - default_tag = fre.__version__ + default_tag = fre.version print(f'(checkout_script) default_tag is {default_tag}') From f78650b762552f4a962256c431433bb69bf0bcae Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Wed, 27 Nov 2024 13:44:13 -0500 Subject: [PATCH 110/111] fre version testing!! --- fre/tests/test_fre_cli.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fre/tests/test_fre_cli.py b/fre/tests/test_fre_cli.py index 5c5bdb12..e7bad1d0 100644 --- a/fre/tests/test_fre_cli.py +++ b/fre/tests/test_fre_cli.py @@ -20,3 +20,15 @@ def test_cli_fre_option_dne(): ''' fre optionDNE ''' result = runner.invoke(fre.fre, args='optionDNE') assert result.exit_code == 2 + + +def test_fre_version(): + ''' module import flavor of below cli test ''' + assert '2024.01' == fre.version + +def test_cli_fre_version(): + ''' fre --version ''' + result = runner.invoke(fre.fre, args='--version') + expected_out = 'fre, version 2024.01' + assert all( [ result.exit_code == 0, + expected_out in result.stdout.split('\n') ] ) From 4409b588245cb9dbee3ea667dc2bf523865361c8 Mon Sep 17 00:00:00 2001 From: Chris Blanton Date: Wed, 27 Nov 2024 15:24:11 -0500 Subject: [PATCH 111/111] Fixed logic for fre pp checkout's branch and tag checking --- fre/pp/checkout_script.py | 47 ++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py index e3e9e29b..02dca1f7 100644 --- a/fre/pp/checkout_script.py +++ b/fre/pp/checkout_script.py @@ -24,8 +24,11 @@ def checkout_template(experiment = None, platform = None, target = None, branch # branch and version parameters default_tag = fre.version - print(f'(checkout_script) default_tag is {default_tag}') - + git_clone_branch_arg = branch if branch is not None else default_tag + if branch is None: + print(f"(checkout_script) default tag is '{default_tag}'") + else: + print(f"(checkout_script) requested branch/tag is '{branch}'") # check args + set the name of the directory if None in [experiment, platform, target]: @@ -41,41 +44,35 @@ def checkout_template(experiment = None, platform = None, target = None, branch raise OSError( '(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc - print(f'(checkout_script) branch is {branch}') checkout_exists = os.path.isdir(f'{directory}/{name}') - git_clone_branch_arg = branch if branch is not None else default_tag - if branch is not None: - print('(checkout_script) WARNING using default_tag as branch argument for git clone!') - if not checkout_exists: # scenarios 1+2, checkout doesn't exist, branch specified (or not) + print(f'(checkout_script) checkout does not yet exist; will create now') clone_output = subprocess.run( ['git', 'clone','--recursive', f'--branch={git_clone_branch_arg}', FRE_WORKFLOWS_URL, f'{directory}/{name}'], capture_output = True, text = True, check = True) - print(f'(checkout_script) output git clone command: {clone_output}') + print(f'(checkout_script) {clone_output}') else: # the repo checkout does exist, scenarios 3 and 4. os.chdir(f'{directory}/{name}') - name_path_tag_subproc_out = subprocess.run(["git","describe","--tags"], - capture_output = True, - text = True, check = True).stdout - if branch is not None: - name_path_tag = name_path_tag_subproc_out.split('*') - name_path_branch = subprocess.run(["git","branch"], - capture_output = True, - text = True, check = True).stdout.split()[0] - if all( [ default_tag not in name_path_tag, - name_path_branch != branch ] ): - sys.exit( - f"Tag and branch of prexisting directory {directory}/{name} does not match " - "fre --version or branch requested" ) + # capture the branch and tag + # if either match git_clone_branch_arg, then success. otherwise, fail. + + current_tag = subprocess.run(["git","describe","--tags"], + capture_output = True, + text = True, check = True).stdout.strip() + current_branch = subprocess.run(["git", "branch", "--show-current"], + capture_output = True, + text = True, check = True).stdout.strip() + + if current_tag == git_clone_branch_arg or current_branch == git_clone_branch_arg: + print(f"(checkout_script) checkout exists ('{directory}/{name}'), and matches '{git_clone_branch_arg}'") else: - name_path_tag = name_path_tag_subproc_out.split()[0] - if not default_tag in name_path_tag: - sys.exit( - f"Tag of prexisting directory {directory}/{name} does not match fre --version") + print(f"(checkout_script) ERROR: checkout exists ('{directory}/{name}') and does not match '{git_clone_branch_arg}'") + print(f"(checkout_script) ERROR: current branch is '{current_branch}', current tag-describe is '{current_tag}'") + exit(1) # make sure we are back where we should be if os.getcwd() != go_back_here: