separate radical test from all executors tests

Makefile

@@ -49,39 +49,37 @@ mypy: ## run mypy checks
 
 .PHONY: local_thread_test
 local_thread_test: ## run all tests with local_thread config
-	pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/local_threads.py --random-order --durations 10
+	pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/local_threads.py --random-order --durations 10
 
 .PHONY: htex_local_test
 htex_local_test: ## run all tests with htex_local config
 	pip3 install .
-	pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/htex_local.py --random-order --durations 10
+	pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local.py --random-order --durations 10
 
 .PHONY: htex_local_alternate_test
 htex_local_alternate_test: ## run all tests with htex_local config
 	pip3 install ".[monitoring]"
-	pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/htex_local_alternate.py --random-order --durations 10
+	pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local_alternate.py --random-order --durations 10
 
 $(CCTOOLS_INSTALL):	#CCtools contains both taskvine and workqueue so install only once
 	parsl/executors/taskvine/install-taskvine.sh
 
 .PHONY: vineex_local_test
 vineex_local_test: $(CCTOOLS_INSTALL)  ## run all tests with taskvine_ex config
-	PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363 and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/taskvine_ex.py --random-order --durations 10
+	PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/taskvine_ex.py --random-order --durations 10
 
 .PHONY: wqex_local_test
 wqex_local_test: $(CCTOOLS_INSTALL)  ## run all tests with workqueue_ex config
-	PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363 and not test_radical_bulk and not test_radical_mpi" --config parsl/tests/configs/workqueue_ex.py --random-order --durations 10
+	PYTHONPATH=/tmp/cctools/lib/python3.8/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/workqueue_ex.py --random-order --durations 10
 
-.PHONY: radical_ex_test
-radical_ex_test:
-	RPEX_BULK=True pytest parsl/tests/test_radical/test_bulk.py --config parsl/tests/configs/local_radical.py
+.PHONY: radical_local_test
+radical_local_test:
 	pytest parsl/tests/test_callables.py -k "not cleannet and not dynamically_loaded_module and not test_check_importlib_file_function" --config parsl/tests/configs/local_radical.py
-	RPEX_MPI=True pytest parsl/tests/test_radical/test_mpi_funcs.py --config parsl/tests/configs/local_radical.py
 
 .PHONY: config_local_test
 config_local_test:
 	pip3 install ".[monitoring,visualization,proxystore]"
-	pytest parsl/tests/ -k "not cleannet and not test_radical_bulk and not test_radical_mpi" --config local --random-order --durations 10
+	pytest parsl/tests/ -k "not cleannet" --config local --random-order --durations 10
 
 .PHONY: site_test
 site_test:
@@ -93,7 +91,7 @@ perf_test:
 	parsl-perf --time 5 --config parsl/tests/configs/local_threads.py
 
 .PHONY: test ## run all tests with all config types
-test: clean_coverage lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test config_local_test perf_test ## run all tests
+test: clean_coverage lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test radical_ex_test config_local_test perf_test ## run all tests
 
 .PHONY: tag
 tag: ## create a tag in git. to run, do a 'make VERSION="version string" tag

parsl/tests/configs/local_radical.py

@@ -4,21 +4,20 @@
 from parsl.executors import RadicalPilotExecutor
 from parsl.executors.radical.rpex_resources import ResourceConfig as rpex_cfg
 
-bulk_mode = False
-
-# start the MPI workers
-if os.environ.get("RPEX_MPI"):
-    rpex_cfg.worker_type = "MPI"
-
-# submit in bulks instead of stream
-if os.environ.get("RPEX_BULK"):
-    bulk_mode = True
-
 # This is temporary; once everything is merged, we will use Parsl instead of
 # this fork.
-rpex_cfg.pilot_env_setup.append("pip install git+https://github.com/AymenFJA/parsl.git")
+parsl_src = "pip install git+https://github.com/AymenFJA/parsl.git"
+rpex_cfg.pilot_env_setup.append(parsl_src)
+
+
+def fresh_config():
 
-config = Config(
-    executors=[RadicalPilotExecutor(rpex_cfg=rpex_cfg.get_cfg_file(),
-                                    bulk_mode=bulk_mode, resource='local.localhost',
-                                    access_schema='local', walltime=30, cores=4)])
+    return Config(
+            executors=[
+                RadicalPilotExecutor(
+                    label='RPEXBulk',
+                    rpex_cfg=rpex_cfg.get_cfg_file(),
+                    bulk_mode=True,
+                    resource='local.localhost',
+                    access_schema='local',
+                    walltime=30, cores=4)])

parsl/tests/configs/local_radical_mpi.py

@@ -0,0 +1,24 @@
+import os
+
+from parsl.config import Config
+from parsl.executors import RadicalPilotExecutor
+from parsl.executors.radical.rpex_resources import ResourceConfig as rpex_cfg
+
+# This is temporary; once everything is merged, we will use Parsl instead of
+# this fork.
+parsl_src = "pip install git+https://github.com/AymenFJA/parsl.git"
+rpex_cfg.pilot_env_setup.append(parsl_src)
+
+
+def fresh_config():
+    rpex_cfg.worker_type = "MPI"
+
+    return Config(
+            executors=[
+                RadicalPilotExecutor(
+                    label='RPEXMPI',
+                    rpex_cfg=rpex_cfg.get_cfg_file(),
+                    bulk_mode=True,
+                    resource='local.localhost',
+                    access_schema='local',
+                    walltime=30, cores=4)])

parsl/tests/test_radical/test_mpi_funcs.py

@@ -1,4 +1,7 @@
 import parsl
+import pytest
+
+from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
 
 
 @parsl.python_app
@@ -12,12 +15,9 @@ def test_mpi_func(comm, msg, sleep, ranks):
 apps = []
 
 
+@pytest.mark.local
 def test_radical_mpi(n=10):
     for i in range(n):
         t = test_mpi_func(None, msg='mpi.func.%06d' % i, sleep=1, ranks=2)
         apps.append(t)
     [app.result() for app in apps]
-
-
-if __name__ == "__main__":
-    test_radical_mpi()