From da934a960080fda166fbf2bbcdf5c271ecc73f69 Mon Sep 17 00:00:00 2001 From: AymenFJA Date: Wed, 25 Oct 2023 15:04:20 -0400 Subject: [PATCH] separate radical test from all executors tests --- Makefile | 20 +++++++-------- parsl/tests/configs/local_radical.py | 29 +++++++++++----------- parsl/tests/configs/local_radical_mpi.py | 24 ++++++++++++++++++ parsl/tests/test_radical/test_bulk.py | 20 --------------- parsl/tests/test_radical/test_mpi_funcs.py | 8 +++--- 5 files changed, 51 insertions(+), 50 deletions(-) create mode 100644 parsl/tests/configs/local_radical_mpi.py delete mode 100644 parsl/tests/test_radical/test_bulk.py diff --git a/Makefile b/Makefile index 65827cf7f5..a89b28f374 100644 --- a/Makefile +++ b/Makefile @@ -49,39 +49,37 @@ mypy: ## run mypy checks .PHONY: local_thread_test local_thread_test: ## run all tests with local_thread config - pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/local_threads.py --random-order --durations 10 + pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/local_threads.py --random-order --durations 10 .PHONY: htex_local_test htex_local_test: ## run all tests with htex_local config pip3 install . - pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/htex_local.py --random-order --durations 10 + pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local.py --random-order --durations 10 .PHONY: htex_local_alternate_test htex_local_alternate_test: ## run all tests with htex_local config pip3 install ".[monitoring]" - pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/htex_local_alternate.py --random-order --durations 10 + pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local_alternate.py --random-order --durations 10 $(CCTOOLS_INSTALL): #CCtools contains both taskvine and workqueue so install only once parsl/executors/taskvine/install-taskvine.sh .PHONY: vineex_local_test vineex_local_test: $(CCTOOLS_INSTALL) ## run all tests with taskvine_ex config - PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages pytest parsl/tests/ -k "not cleannet and not issue363 and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/taskvine_ex.py --random-order --durations 10 + PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/taskvine_ex.py --random-order --durations 10 .PHONY: wqex_local_test wqex_local_test: $(CCTOOLS_INSTALL) ## run all tests with workqueue_ex config - PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages pytest parsl/tests/ -k "not cleannet and not issue363 and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/workqueue_ex.py --random-order --durations 10 + PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/workqueue_ex.py --random-order --durations 10 -.PHONY: radical_ex_test -radical_ex_test: - RPEX_BULK=True pytest parsl/tests/test_radical/test_bulk.py --config parsl/tests/configs/local_radical.py +.PHONY: radical_local_test +radical_local_test: pytest parsl/tests/test_callables.py -k "not cleannet and not dynamically_loaded_module and not test_check_importlib_file_function" --config parsl/tests/configs/local_radical.py - RPEX_MPI=True pytest parsl/tests/test_radical/test_mpi_funcs.py --config parsl/tests/configs/local_radical.py .PHONY: config_local_test config_local_test: pip3 install ".[monitoring,visualization,proxystore]" - pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config local --random-order --durations 10 + pytest parsl/tests/ -k "not cleannet" --config local --random-order --durations 10 .PHONY: site_test site_test: @@ -93,7 +91,7 @@ perf_test: parsl-perf --time 5 --config parsl/tests/configs/local_threads.py .PHONY: test ## run all tests with all config types -test: clean_coverage lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test config_local_test perf_test ## run all tests +test: clean_coverage lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test radical_ex_test config_local_test perf_test ## run all tests .PHONY: tag tag: ## create a tag in git. to run, do a 'make VERSION="version string" tag diff --git a/parsl/tests/configs/local_radical.py b/parsl/tests/configs/local_radical.py index 565e745d3a..481d0418e6 100644 --- a/parsl/tests/configs/local_radical.py +++ b/parsl/tests/configs/local_radical.py @@ -4,21 +4,20 @@ from parsl.executors import RadicalPilotExecutor from parsl.executors.radical.rpex_resources import ResourceConfig as rpex_cfg -bulk_mode = False - -# start the MPI workers -if os.environ.get("RPEX_MPI"): - rpex_cfg.worker_type = "MPI" - -# submit in bulks instead of stream -if os.environ.get("RPEX_BULK"): - bulk_mode = True - # This is temporary; once everything is merged, we will use Parsl instead of # this fork. -rpex_cfg.pilot_env_setup.append("pip install git+https://github.com/AymenFJA/parsl.git") +parsl_src = "pip install git+https://github.com/AymenFJA/parsl.git" +rpex_cfg.pilot_env_setup.append(parsl_src) + + +def fresh_config(): -config = Config( - executors=[RadicalPilotExecutor(rpex_cfg=rpex_cfg.get_cfg_file(), - bulk_mode=bulk_mode, resource='local.localhost', - access_schema='local', walltime=30, cores=4)]) + return Config( + executors=[ + RadicalPilotExecutor( + label='RPEXBulk', + rpex_cfg=rpex_cfg.get_cfg_file(), + bulk_mode=True, + resource='local.localhost', + access_schema='local', + walltime=30, cores=4)]) diff --git a/parsl/tests/configs/local_radical_mpi.py b/parsl/tests/configs/local_radical_mpi.py new file mode 100644 index 0000000000..e0d5c6d688 --- /dev/null +++ b/parsl/tests/configs/local_radical_mpi.py @@ -0,0 +1,24 @@ +import os + +from parsl.config import Config +from parsl.executors import RadicalPilotExecutor +from parsl.executors.radical.rpex_resources import ResourceConfig as rpex_cfg + +# This is temporary; once everything is merged, we will use Parsl instead of +# this fork. +parsl_src = "pip install git+https://github.com/AymenFJA/parsl.git" +rpex_cfg.pilot_env_setup.append(parsl_src) + + +def fresh_config(): + rpex_cfg.worker_type = "MPI" + + return Config( + executors=[ + RadicalPilotExecutor( + label='RPEXMPI', + rpex_cfg=rpex_cfg.get_cfg_file(), + bulk_mode=True, + resource='local.localhost', + access_schema='local', + walltime=30, cores=4)]) diff --git a/parsl/tests/test_radical/test_bulk.py b/parsl/tests/test_radical/test_bulk.py deleted file mode 100644 index 74e2af8570..0000000000 --- a/parsl/tests/test_radical/test_bulk.py +++ /dev/null @@ -1,20 +0,0 @@ -import parsl - - -@parsl.bash_app -def test_bulk_bashApps(): - return 'echo "Hello World!"' - - -apps = [] - - -def test_radical_bulk(n=100): - for _ in range(n): - t = test_bulk_bashApps() - apps.append(t) - [app.result() for app in apps] - - -if __name__ == "__main__": - test_radical_bulk() diff --git a/parsl/tests/test_radical/test_mpi_funcs.py b/parsl/tests/test_radical/test_mpi_funcs.py index 830af6de3d..537d9b2a1f 100644 --- a/parsl/tests/test_radical/test_mpi_funcs.py +++ b/parsl/tests/test_radical/test_mpi_funcs.py @@ -1,4 +1,7 @@ import parsl +import pytest + +from parsl.tests.configs.local_radical_mpi import fresh_config as local_config @parsl.python_app @@ -12,12 +15,9 @@ def test_mpi_func(comm, msg, sleep, ranks): apps = [] +@pytest.mark.local def test_radical_mpi(n=10): for i in range(n): t = test_mpi_func(None, msg='mpi.func.%06d' % i, sleep=1, ranks=2) apps.append(t) [app.result() for app in apps] - - -if __name__ == "__main__": - test_radical_mpi()