From 8bb9f4d5590dd4595829cd6eea2996a47a4cc5dd Mon Sep 17 00:00:00 2001 From: Yadu Babuji Date: Thu, 9 May 2024 15:00:59 -0500 Subject: [PATCH 1/3] Adding isort config, entries to Makefile to run checks along with CI, and test requirements --- .isort.cfg | 2 ++ Makefile | 6 +++++- test-requirements.txt | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000000..f238bf7ea1 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile = black diff --git a/Makefile b/Makefile index f3b40cdbf3..6ce923a9ac 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,10 @@ deps: $(DEPS) ## install the dependencies lint: ## run linter script parsl/tests/lint-inits.sh +.PHONY: isort +isort: ## run isort on all files + isort --check . + .PHONY: flake8 flake8: ## run flake flake8 parsl/ @@ -93,7 +97,7 @@ perf_test: parsl-perf --time 5 --config parsl/tests/configs/local_threads.py .PHONY: test ## run all tests with all config types -test: clean_coverage lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test radical_local_test config_local_test perf_test ## run all tests +test: clean_coverage isort lint flake8 mypy local_thread_test htex_local_test htex_local_alternate_test wqex_local_test vineex_local_test radical_local_test config_local_test perf_test ## run all tests .PHONY: tag tag: ## create a tag in git. to run, do a 'make VERSION="version string" tag diff --git a/test-requirements.txt b/test-requirements.txt index 050c8e829b..c735de8d5c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,3 +21,5 @@ sqlalchemy2-stubs Sphinx==4.5.0 twine wheel +isort + From b90d5f3eda949896b6c7031ed8c6d7903adc4a39 Mon Sep 17 00:00:00 2001 From: Yadu Babuji Date: Mon, 3 Jun 2024 12:05:39 -0500 Subject: [PATCH 2/3] Only isort changes --- docs/conf.py | 3 ++ docs/teaching_scripts/test_apps.py | 2 +- docs/userguide/examples/config.py | 2 +- docs/userguide/examples/library.py | 1 + docs/userguide/examples/run_increment.py | 3 +- parsl/__init__.py | 19 ++++--- parsl/addresses.py | 10 ++-- parsl/app/app.py | 9 ++-- parsl/app/bash.py | 8 +-- parsl/app/errors.py | 8 +-- parsl/app/futures.py | 6 +-- parsl/app/python.py | 3 +- parsl/benchmark/perf.py | 3 +- parsl/channels/__init__.py | 4 +- parsl/channels/base.py | 1 - parsl/channels/errors.py | 3 +- parsl/channels/oauth_ssh/oauth_ssh.py | 7 +-- parsl/channels/ssh/ssh.py | 10 +++- parsl/channels/ssh_il/ssh_il.py | 1 + parsl/concurrent/__init__.py | 4 +- parsl/config.py | 10 ++-- parsl/configs/ASPIRE1.py | 6 +-- parsl/configs/Azure.py | 13 +++-- parsl/configs/ad_hoc.py | 7 +-- parsl/configs/bridges.py | 6 +-- parsl/configs/cc_in2p3.py | 4 +- parsl/configs/ec2.py | 2 +- parsl/configs/expanse.py | 3 +- parsl/configs/frontera.py | 5 +- parsl/configs/htex_local.py | 3 +- parsl/configs/illinoiscluster.py | 2 +- parsl/configs/kubernetes.py | 3 +- parsl/configs/midway.py | 6 +-- parsl/configs/osg.py | 2 +- parsl/configs/polaris.py | 2 +- parsl/configs/stampede2.py | 9 ++-- parsl/configs/summit.py | 4 +- parsl/configs/toss3_llnl.py | 3 +- parsl/configs/vineex_local.py | 6 +-- parsl/configs/wqex_local.py | 4 +- parsl/data_provider/data_manager.py | 6 +-- parsl/data_provider/file_noop.py | 3 +- parsl/data_provider/files.py | 6 +-- parsl/data_provider/ftp.py | 4 +- parsl/data_provider/globus.py | 13 ++--- parsl/data_provider/http.py | 4 +- parsl/data_provider/rsync.py | 2 +- parsl/data_provider/staging.py | 4 +- parsl/data_provider/zip.py | 9 ++-- parsl/dataflow/dflow.py | 37 +++++++------ parsl/dataflow/errors.py | 3 +- parsl/dataflow/futures.py | 3 +- parsl/dataflow/memoization.py | 10 ++-- parsl/dataflow/rundirs.py | 2 +- parsl/dataflow/taskrecord.py | 9 ++-- parsl/executors/__init__.py | 6 +-- parsl/executors/base.py | 1 + parsl/executors/flux/execute_parsl_task.py | 4 +- parsl/executors/flux/executor.py | 23 ++++---- parsl/executors/flux/flux_instance_manager.py | 6 +-- parsl/executors/high_throughput/executor.py | 47 ++++++++-------- .../executors/high_throughput/interchange.py | 26 +++++---- .../high_throughput/manager_record.py | 1 + .../high_throughput/monitoring_info.py | 3 +- .../executors/high_throughput/mpi_executor.py | 7 ++- .../high_throughput/mpi_prefix_composer.py | 2 +- .../mpi_resource_management.py | 3 +- parsl/executors/high_throughput/probe.py | 10 ++-- .../high_throughput/process_worker_pool.py | 38 ++++++------- parsl/executors/high_throughput/zmq_pipes.py | 8 ++- parsl/executors/radical/executor.py | 30 +++++------ parsl/executors/radical/rpex_master.py | 3 +- parsl/executors/radical/rpex_resources.py | 3 +- parsl/executors/radical/rpex_worker.py | 3 +- parsl/executors/status_handling.py | 9 ++-- parsl/executors/taskvine/__init__.py | 2 +- parsl/executors/taskvine/errors.py | 2 +- .../executors/taskvine/exec_parsl_function.py | 4 +- parsl/executors/taskvine/executor.py | 47 ++++++++-------- parsl/executors/taskvine/factory.py | 2 +- parsl/executors/taskvine/manager.py | 24 ++++----- parsl/executors/threads.py | 9 ++-- parsl/executors/workqueue/errors.py | 2 +- .../workqueue/exec_parsl_function.py | 9 ++-- parsl/executors/workqueue/executor.py | 53 +++++++++---------- parsl/executors/workqueue/parsl_coprocess.py | 2 +- parsl/jobs/error_handlers.py | 2 +- parsl/jobs/job_status_poller.py | 7 +-- parsl/jobs/states.py | 2 +- parsl/jobs/strategy.py | 4 +- parsl/launchers/__init__.py | 15 ++++-- parsl/launchers/errors.py | 2 +- parsl/log_utils.py | 3 +- parsl/monitoring/db_manager.py | 26 +++++---- parsl/monitoring/monitoring.py | 26 ++++----- parsl/monitoring/queries/pandas.py | 3 +- parsl/monitoring/radios.py | 6 +-- parsl/monitoring/remote.py | 21 +++++--- parsl/monitoring/router.py | 19 +++---- parsl/monitoring/types.py | 2 + parsl/monitoring/visualization/app.py | 6 ++- parsl/monitoring/visualization/models.py | 1 - .../plots/default/workflow_plots.py | 12 +++-- .../plots/default/workflow_resource_plots.py | 1 + parsl/monitoring/visualization/utils.py | 1 - parsl/monitoring/visualization/views.py | 25 +++++---- parsl/multiprocessing.py | 1 - parsl/process_loggers.py | 3 +- parsl/providers/__init__.py | 26 ++++----- parsl/providers/ad_hoc/ad_hoc.py | 2 +- parsl/providers/aws/aws.py | 5 +- parsl/providers/azure/azure.py | 9 ++-- parsl/providers/base.py | 2 +- parsl/providers/cluster_provider.py | 2 +- parsl/providers/cobalt/cobalt.py | 6 +-- parsl/providers/condor/condor.py | 6 ++- parsl/providers/errors.py | 4 +- parsl/providers/googlecloud/googlecloud.py | 3 +- parsl/providers/grid_engine/grid_engine.py | 4 +- parsl/providers/kubernetes/kube.py | 8 +-- parsl/providers/local/local.py | 6 ++- parsl/providers/lsf/lsf.py | 4 +- parsl/providers/pbspro/pbspro.py | 2 +- parsl/providers/slurm/slurm.py | 10 ++-- parsl/providers/torque/torque.py | 2 +- parsl/serialize/__init__.py | 11 ++-- parsl/serialize/base.py | 3 +- parsl/serialize/concretes.py | 9 ++-- parsl/serialize/proxystore.py | 5 +- parsl/tests/__init__.py | 2 +- parsl/tests/configs/ad_hoc_cluster_htex.py | 8 +-- parsl/tests/configs/azure_single_node.py | 9 ++-- parsl/tests/configs/bridges.py | 5 +- parsl/tests/configs/cc_in2p3.py | 4 +- parsl/tests/configs/comet.py | 3 +- parsl/tests/configs/ec2_single_node.py | 3 +- parsl/tests/configs/ec2_spot.py | 3 +- parsl/tests/configs/frontera.py | 5 +- parsl/tests/configs/htex_ad_hoc_cluster.py | 6 +-- parsl/tests/configs/htex_local.py | 5 +- parsl/tests/configs/htex_local_alternate.py | 19 +++---- .../configs/htex_local_intask_staging.py | 12 ++--- .../tests/configs/htex_local_rsync_staging.py | 10 ++-- parsl/tests/configs/local_adhoc.py | 2 +- parsl/tests/configs/local_radical.py | 4 +- parsl/tests/configs/local_radical_mpi.py | 4 +- parsl/tests/configs/midway.py | 4 +- parsl/tests/configs/nscc_singapore.py | 6 +-- parsl/tests/configs/osg_htex.py | 2 +- parsl/tests/configs/petrelkube.py | 5 +- parsl/tests/configs/summit.py | 1 + parsl/tests/configs/swan_htex.py | 4 +- parsl/tests/configs/taskvine_ex.py | 8 ++- parsl/tests/configs/theta.py | 4 +- parsl/tests/configs/workqueue_ex.py | 7 ++- parsl/tests/conftest.py | 8 +-- .../test_channels/test_ssh_errors.py | 2 +- .../test_stress/test_python_simple.py | 6 ++- .../test_stress/test_python_threads.py | 8 ++- parsl/tests/manual_tests/htex_local.py | 8 +-- parsl/tests/manual_tests/test_ad_hoc_htex.py | 3 +- parsl/tests/manual_tests/test_basic.py | 1 + .../test_fan_in_out_htex_remote.py | 8 +-- parsl/tests/manual_tests/test_log_filter.py | 4 +- .../tests/manual_tests/test_memory_limits.py | 12 ++--- .../tests/manual_tests/test_regression_220.py | 3 +- parsl/tests/manual_tests/test_udp_simple.py | 7 +-- parsl/tests/manual_tests/test_worker_count.py | 5 +- parsl/tests/scaling_tests/htex_local.py | 4 +- parsl/tests/scaling_tests/test_scale.py | 9 ++-- parsl/tests/scaling_tests/vineex_condor.py | 3 +- parsl/tests/scaling_tests/vineex_local.py | 3 +- parsl/tests/site_tests/test_provider.py | 4 +- parsl/tests/site_tests/test_site.py | 2 + parsl/tests/sites/test_affinity.py | 12 +++-- parsl/tests/sites/test_dynamic_executor.py | 6 +-- parsl/tests/sites/test_ec2.py | 5 +- parsl/tests/sites/test_local_adhoc.py | 3 +- parsl/tests/sites/test_worker_info.py | 7 +-- parsl/tests/test_aalst_patterns.py | 1 - parsl/tests/test_bash_apps/test_apptimeout.py | 4 +- .../tests/test_bash_apps/test_error_codes.py | 5 +- .../test_memoize_ignore_args.py | 1 + .../test_memoize_ignore_args_regr.py | 4 +- parsl/tests/test_bash_apps/test_pipeline.py | 2 +- parsl/tests/test_bash_apps/test_std_uri.py | 7 +-- parsl/tests/test_callables.py | 4 +- .../test_python_checkpoint_1.py | 1 + .../test_python_checkpoint_2.py | 3 +- .../test_checkpointing/test_regression_239.py | 2 +- .../test_checkpointing/test_task_exit.py | 2 +- parsl/tests/test_docs/test_from_slides.py | 4 +- parsl/tests/test_docs/test_kwargs.py | 2 +- parsl/tests/test_docs/test_tutorial_1.py | 3 +- parsl/tests/test_docs/test_workflow1.py | 4 +- parsl/tests/test_docs/test_workflow2.py | 1 - .../test_error_handling/test_rand_fail.py | 4 +- .../test_error_handling/test_resource_spec.py | 6 ++- .../tests/test_error_handling/test_retries.py | 3 +- .../test_error_handling/test_retry_handler.py | 1 + .../test_retry_handler_failure.py | 3 +- .../test_serialization_fail.py | 2 +- .../test_wrap_with_logs.py | 1 + parsl/tests/test_flux.py | 2 +- .../test_htex/test_command_client_timeout.py | 10 ++-- .../tests/test_htex/test_connected_blocks.py | 5 +- .../test_htex/test_cpu_affinity_explicit.py | 6 ++- .../test_htex/test_disconnected_blocks.py | 10 ++-- parsl/tests/test_htex/test_drain.py | 10 ++-- parsl/tests/test_htex/test_htex.py | 3 +- .../tests/test_htex/test_managers_command.py | 5 +- .../test_multiple_disconnected_blocks.py | 10 ++-- parsl/tests/test_monitoring/test_app_names.py | 5 +- parsl/tests/test_monitoring/test_basic.py | 8 +-- parsl/tests/test_monitoring/test_db_locks.py | 9 ++-- parsl/tests/test_monitoring/test_fuzz_zmq.py | 9 ++-- .../test_htex_init_blocks_vs_monitoring.py | 10 ++-- .../test_incomplete_futures.py | 8 +-- .../test_memoization_representation.py | 5 +- parsl/tests/test_monitoring/test_stdouterr.py | 8 +-- .../test_monitoring/test_viz_colouring.py | 1 + .../test_mpi_apps/test_bad_mpi_config.py | 2 +- .../test_mpi_apps/test_mpi_mode_disabled.py | 2 + .../test_mpi_apps/test_mpi_mode_enabled.py | 12 +++-- parsl/tests/test_mpi_apps/test_mpi_prefix.py | 8 +-- .../tests/test_mpi_apps/test_mpi_scheduler.py | 9 +++- parsl/tests/test_mpi_apps/test_mpiex.py | 7 +-- .../tests/test_mpi_apps/test_resource_spec.py | 19 ++++--- .../test_cobalt_deprecation_warning.py | 2 + .../test_providers/test_local_provider.py | 3 +- .../test_providers/test_pbspro_template.py | 2 +- .../test_providers/test_slurm_template.py | 2 +- .../test_submiterror_deprecation.py | 3 +- .../test_python_apps/test_context_manager.py | 7 +-- .../test_dep_standard_futures.py | 3 +- parsl/tests/test_python_apps/test_futures.py | 3 +- parsl/tests/test_python_apps/test_join.py | 1 - parsl/tests/test_python_apps/test_lifted.py | 6 +-- .../test_memoize_bad_id_for_memo.py | 1 + .../test_pluggable_future_resolution.py | 2 +- parsl/tests/test_radical/test_mpi_funcs.py | 2 +- parsl/tests/test_regression/test_1480.py | 3 +- parsl/tests/test_regression/test_1653.py | 3 +- parsl/tests/test_regression/test_2652.py | 1 + parsl/tests/test_regression/test_69a.py | 1 - parsl/tests/test_regression/test_854.py | 6 ++- .../test_regression/test_97_parallelism_0.py | 3 +- parsl/tests/test_regression/test_98.py | 1 - .../test_scaling/test_block_error_handler.py | 13 +++-- parsl/tests/test_scaling/test_scale_down.py | 5 +- .../test_scale_down_htex_auto_scale.py | 9 ++-- .../test_scale_down_htex_unregistered.py | 7 ++- .../test_scaling/test_shutdown_scalein.py | 3 +- .../test_2555_caching_deserializer.py | 2 +- parsl/tests/test_serialization/test_basic.py | 3 +- .../test_htex_code_cache.py | 7 ++- .../test_pack_resource_spec.py | 3 +- .../test_proxystore_configured.py | 15 ++++-- .../test_proxystore_impl.py | 8 +-- .../test_shutdown/test_kill_monitoring.py | 5 +- parsl/tests/test_staging/staging_provider.py | 4 +- parsl/tests/test_staging/test_1316.py | 5 +- parsl/tests/test_staging/test_docs_1.py | 2 +- parsl/tests/test_staging/test_docs_2.py | 3 +- .../test_staging/test_elaborate_noop_file.py | 4 +- .../tests/test_staging/test_staging_https.py | 4 +- .../tests/test_staging/test_staging_stdout.py | 7 +-- parsl/tests/test_staging/test_zip_in.py | 14 +++-- parsl/tests/test_staging/test_zip_out.py | 16 +++--- parsl/tests/test_staging/test_zip_to_zip.py | 14 +++-- parsl/tests/test_summary.py | 3 +- parsl/tests/test_thread_parallelism.py | 1 - parsl/tests/test_threads/test_configs.py | 2 +- parsl/tests/test_threads/test_lazy_errors.py | 3 +- parsl/usage_tracking/api.py | 4 +- parsl/usage_tracking/usage.py | 13 ++--- parsl/utils.py | 15 +++++- setup.py | 2 +- 278 files changed, 960 insertions(+), 850 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a475012603..27ee2f3f35 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,9 +19,12 @@ # import os import sys + import requests + sys.path.insert(0, os.path.abspath('../')) import parsl + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. diff --git a/docs/teaching_scripts/test_apps.py b/docs/teaching_scripts/test_apps.py index db9c858f78..f05a5f7f3f 100644 --- a/docs/teaching_scripts/test_apps.py +++ b/docs/teaching_scripts/test_apps.py @@ -3,8 +3,8 @@ import numpy as np -from parsl import python_app, HighThroughputExecutor, Config import parsl +from parsl import Config, HighThroughputExecutor, python_app parsl.load(Config(executors=[HighThroughputExecutor(label='htex_spawn', max_workers_per_node=1, address='127.0.0.1')])) diff --git a/docs/userguide/examples/config.py b/docs/userguide/examples/config.py index fa5fe6ff4a..166faaf4ac 100644 --- a/docs/userguide/examples/config.py +++ b/docs/userguide/examples/config.py @@ -1,5 +1,5 @@ -from parsl.config import Config from parsl.channels import LocalChannel +from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.providers import LocalProvider diff --git a/docs/userguide/examples/library.py b/docs/userguide/examples/library.py index e26210eff5..2992a2dfc2 100644 --- a/docs/userguide/examples/library.py +++ b/docs/userguide/examples/library.py @@ -1,5 +1,6 @@ from parsl import python_app + @python_app def increment(x): return x + 1 diff --git a/docs/userguide/examples/run_increment.py b/docs/userguide/examples/run_increment.py index 71bc633573..b265640edc 100644 --- a/docs/userguide/examples/run_increment.py +++ b/docs/userguide/examples/run_increment.py @@ -1,7 +1,8 @@ -import parsl from config import htex_config from library import increment +import parsl + parsl.load(htex_config) for i in range(5): diff --git a/parsl/__init__.py b/parsl/__init__.py index 13151698a1..5baafb9e6e 100644 --- a/parsl/__init__.py +++ b/parsl/__init__.py @@ -15,24 +15,23 @@ """ import logging +import multiprocessing as _multiprocessing import os import platform -from parsl.version import VERSION from parsl.app.app import bash_app, join_app, python_app from parsl.config import Config -from parsl.executors import ThreadPoolExecutor -from parsl.executors import HighThroughputExecutor -from parsl.executors import WorkQueueExecutor -from parsl.log_utils import set_stream_logger -from parsl.log_utils import set_file_logger -from parsl.monitoring import MonitoringHub - from parsl.data_provider.files import File - from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader +from parsl.executors import ( + HighThroughputExecutor, + ThreadPoolExecutor, + WorkQueueExecutor, +) +from parsl.log_utils import set_file_logger, set_stream_logger +from parsl.monitoring import MonitoringHub +from parsl.version import VERSION -import multiprocessing as _multiprocessing if platform.system() == 'Darwin': _multiprocessing.set_start_method('fork', force=True) diff --git a/parsl/addresses.py b/parsl/addresses.py index b06ea696bf..f45e6dfbd1 100644 --- a/parsl/addresses.py +++ b/parsl/addresses.py @@ -8,17 +8,19 @@ import logging import platform -import requests import socket + +import requests + try: import fcntl except ImportError: fcntl = None # type: ignore[assignment] import struct -import typeguard -import psutil +from typing import Callable, List, Set -from typing import Set, List, Callable +import psutil +import typeguard logger = logging.getLogger(__name__) diff --git a/parsl/app/app.py b/parsl/app/app.py index 63dcf5bbe8..6097415c9e 100644 --- a/parsl/app/app.py +++ b/parsl/app/app.py @@ -3,19 +3,16 @@ The App class encapsulates a generic leaf task that can be executed asynchronously. """ import logging -import typeguard from abc import ABCMeta, abstractmethod from inspect import signature -from typing import List, Optional, Sequence, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Union + +import typeguard from typing_extensions import Literal from parsl.dataflow.dflow import DataFlowKernel - -from typing import Any, Callable, Dict - from parsl.dataflow.futures import AppFuture - logger = logging.getLogger(__name__) diff --git a/parsl/app/bash.py b/parsl/app/bash.py index 07488769c2..4ab0add68b 100644 --- a/parsl/app/bash.py +++ b/parsl/app/bash.py @@ -1,10 +1,9 @@ -from functools import update_wrapper -from functools import partial -from inspect import signature, Parameter import logging +from functools import partial, update_wrapper +from inspect import Parameter, signature -from parsl.app.errors import wrap_error from parsl.app.app import AppBase +from parsl.app.errors import wrap_error from parsl.data_provider.files import File from parsl.dataflow.dflow import DataFlowKernelLoader @@ -17,6 +16,7 @@ def remote_side_bash_executor(func, *args, **kwargs): """ import os import subprocess + import parsl.app.errors as pe from parsl.utils import get_std_fname_mode diff --git a/parsl/app/errors.py b/parsl/app/errors.py index 7b2912bba5..24b941d285 100644 --- a/parsl/app/errors.py +++ b/parsl/app/errors.py @@ -1,10 +1,11 @@ """Exceptions raised by Apps.""" +import logging from functools import wraps -from typing import Callable, List, Optional, TypeVar, Union -from typing_extensions import ParamSpec from types import TracebackType -import logging +from typing import Callable, List, Optional, TypeVar, Union + from tblib import Traceback +from typing_extensions import ParamSpec from parsl.data_provider.files import File from parsl.errors import ParslError @@ -134,6 +135,7 @@ def wrap_error(func: Callable[P, R]) -> Callable[P, Union[R, RemoteExceptionWrap @wraps(func) def wrapper(*args: P.args, **kwargs: P.kwargs) -> Union[R, RemoteExceptionWrapper]: import sys + from parsl.app.errors import RemoteExceptionWrapper try: return func(*args, **kwargs) diff --git a/parsl/app/futures.py b/parsl/app/futures.py index 8441e7a890..f3bc067c84 100644 --- a/parsl/app/futures.py +++ b/parsl/app/futures.py @@ -1,12 +1,12 @@ """This module implements DataFutures. """ import logging -import typeguard from concurrent.futures import Future +from typing import Optional -from parsl.data_provider.files import File +import typeguard -from typing import Optional +from parsl.data_provider.files import File logger = logging.getLogger(__name__) diff --git a/parsl/app/python.py b/parsl/app/python.py index 27a2c464ba..58839cffd5 100644 --- a/parsl/app/python.py +++ b/parsl/app/python.py @@ -2,6 +2,7 @@ import threading import tblib.pickling_support + tblib.pickling_support.install() from functools import wraps @@ -11,7 +12,6 @@ from parsl.dataflow.dflow import DataFlowKernelLoader from parsl.utils import AutoCancelTimer - logger = logging.getLogger(__name__) @@ -19,6 +19,7 @@ def timeout(f, seconds: float): @wraps(f) def wrapper(*args, **kwargs): import ctypes + import parsl.app.errors def inject_exception(thread): diff --git a/parsl/benchmark/perf.py b/parsl/benchmark/perf.py index 16002eae3b..f96789648d 100644 --- a/parsl/benchmark/perf.py +++ b/parsl/benchmark/perf.py @@ -1,7 +1,8 @@ import argparse +import concurrent.futures import importlib import time -import concurrent.futures + import parsl min_iterations = 2 diff --git a/parsl/channels/__init__.py b/parsl/channels/__init__.py index 924670256a..5a45d15278 100644 --- a/parsl/channels/__init__.py +++ b/parsl/channels/__init__.py @@ -1,7 +1,7 @@ from parsl.channels.base import Channel -from parsl.channels.ssh.ssh import SSHChannel from parsl.channels.local.local import LocalChannel -from parsl.channels.ssh_il.ssh_il import SSHInteractiveLoginChannel from parsl.channels.oauth_ssh.oauth_ssh import OAuthSSHChannel +from parsl.channels.ssh.ssh import SSHChannel +from parsl.channels.ssh_il.ssh_il import SSHInteractiveLoginChannel __all__ = ['Channel', 'SSHChannel', 'LocalChannel', 'SSHInteractiveLoginChannel', 'OAuthSSHChannel'] diff --git a/parsl/channels/base.py b/parsl/channels/base.py index cf6a1e5df8..18a573d0e6 100644 --- a/parsl/channels/base.py +++ b/parsl/channels/base.py @@ -1,5 +1,4 @@ from abc import ABCMeta, abstractmethod, abstractproperty - from typing import Dict, Tuple diff --git a/parsl/channels/errors.py b/parsl/channels/errors.py index 662ef83c82..0358dc3ece 100644 --- a/parsl/channels/errors.py +++ b/parsl/channels/errors.py @@ -1,8 +1,9 @@ ''' Exceptions raise by Apps. ''' -from parsl.errors import ParslError from typing import Optional +from parsl.errors import ParslError + class ChannelError(ParslError): """ Base class for all exceptions diff --git a/parsl/channels/oauth_ssh/oauth_ssh.py b/parsl/channels/oauth_ssh/oauth_ssh.py index 4419076ca7..c9efa27767 100644 --- a/parsl/channels/oauth_ssh/oauth_ssh.py +++ b/parsl/channels/oauth_ssh/oauth_ssh.py @@ -1,13 +1,14 @@ import logging -import paramiko import socket -from parsl.errors import OptionalModuleMissing +import paramiko + from parsl.channels.ssh.ssh import SSHChannel +from parsl.errors import OptionalModuleMissing try: - from oauth_ssh.ssh_service import SSHService from oauth_ssh.oauth_ssh_token import find_access_token + from oauth_ssh.ssh_service import SSHService _oauth_ssh_enabled = True except (ImportError, NameError): _oauth_ssh_enabled = False diff --git a/parsl/channels/ssh/ssh.py b/parsl/channels/ssh/ssh.py index 530189d07c..6b38ed68e6 100644 --- a/parsl/channels/ssh/ssh.py +++ b/parsl/channels/ssh/ssh.py @@ -3,8 +3,16 @@ import os import paramiko + from parsl.channels.base import Channel -from parsl.channels.errors import BadHostKeyException, AuthException, SSHException, BadScriptPath, BadPermsScriptPath, FileCopyException +from parsl.channels.errors import ( + AuthException, + BadHostKeyException, + BadPermsScriptPath, + BadScriptPath, + FileCopyException, + SSHException, +) from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/channels/ssh_il/ssh_il.py b/parsl/channels/ssh_il/ssh_il.py index 0905334329..02e7a58cd4 100644 --- a/parsl/channels/ssh_il/ssh_il.py +++ b/parsl/channels/ssh_il/ssh_il.py @@ -2,6 +2,7 @@ import logging import paramiko + from parsl.channels.ssh.ssh import SSHChannel logger = logging.getLogger(__name__) diff --git a/parsl/concurrent/__init__.py b/parsl/concurrent/__init__.py index 50aba49c42..996c7639f4 100644 --- a/parsl/concurrent/__init__.py +++ b/parsl/concurrent/__init__.py @@ -1,8 +1,8 @@ """Interfaces modeled after Python's `concurrent library `_""" -from typing import Callable, Dict, Optional, Iterator, Iterable +import time from concurrent.futures import Executor +from typing import Callable, Dict, Iterable, Iterator, Optional from warnings import warn -import time from parsl import Config, DataFlowKernel from parsl.app.python import PythonApp diff --git a/parsl/config.py b/parsl/config.py index 14a6e7ad71..8858ddb042 100644 --- a/parsl/config.py +++ b/parsl/config.py @@ -1,17 +1,17 @@ import logging -import typeguard - from typing import Callable, Iterable, Optional, Sequence, Union + +import typeguard from typing_extensions import Literal -from parsl.utils import RepresentationMixin from parsl.dataflow.dependency_resolvers import DependencyResolver +from parsl.dataflow.taskrecord import TaskRecord +from parsl.errors import ConfigurationError from parsl.executors.base import ParslExecutor from parsl.executors.threads import ThreadPoolExecutor -from parsl.errors import ConfigurationError -from parsl.dataflow.taskrecord import TaskRecord from parsl.monitoring import MonitoringHub from parsl.usage_tracking.api import UsageInformation +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/configs/ASPIRE1.py b/parsl/configs/ASPIRE1.py index 380f8bf686..1b502fadaf 100644 --- a/parsl/configs/ASPIRE1.py +++ b/parsl/configs/ASPIRE1.py @@ -1,9 +1,9 @@ -from parsl.providers import PBSProProvider -from parsl.launchers import MpiRunLauncher +from parsl.addresses import address_by_interface from parsl.config import Config from parsl.executors import HighThroughputExecutor -from parsl.addresses import address_by_interface +from parsl.launchers import MpiRunLauncher from parsl.monitoring.monitoring import MonitoringHub +from parsl.providers import PBSProProvider config = Config( executors=[ diff --git a/parsl/configs/Azure.py b/parsl/configs/Azure.py index 41332ed762..9d05be7940 100644 --- a/parsl/configs/Azure.py +++ b/parsl/configs/Azure.py @@ -1,14 +1,13 @@ """Config for Azure""" -from parsl.config import Config +import getpass -from parsl.providers import AzureProvider -from parsl.executors import HighThroughputExecutor -from parsl.data_provider.http import HTTPInTaskStaging +from parsl.addresses import address_by_query +from parsl.config import Config from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.rsync import RSyncStaging -from parsl.addresses import address_by_query - -import getpass +from parsl.executors import HighThroughputExecutor +from parsl.providers import AzureProvider vm_reference = { # All fields below are required diff --git a/parsl/configs/ad_hoc.py b/parsl/configs/ad_hoc.py index f0049b442e..daee13ea00 100644 --- a/parsl/configs/ad_hoc.py +++ b/parsl/configs/ad_hoc.py @@ -1,8 +1,9 @@ -from parsl.providers import AdHocProvider +from typing import Any, Dict + from parsl.channels import SSHChannel -from parsl.executors import HighThroughputExecutor from parsl.config import Config -from typing import Any, Dict +from parsl.executors import HighThroughputExecutor +from parsl.providers import AdHocProvider user_opts: Dict[str, Dict[str, Any]] user_opts = {'adhoc': diff --git a/parsl/configs/bridges.py b/parsl/configs/bridges.py index d6c4d63c1a..928cd70549 100644 --- a/parsl/configs/bridges.py +++ b/parsl/configs/bridges.py @@ -1,8 +1,8 @@ +from parsl.addresses import address_by_interface from parsl.config import Config -from parsl.providers import SlurmProvider -from parsl.launchers import SrunLauncher from parsl.executors import HighThroughputExecutor -from parsl.addresses import address_by_interface +from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider """ This config assumes that it is used to launch parsl tasks from the login nodes of Bridges at PSC. Each job submitted to the scheduler will request 2 nodes for 10 minutes. diff --git a/parsl/configs/cc_in2p3.py b/parsl/configs/cc_in2p3.py index 69e74402d6..4016977aed 100644 --- a/parsl/configs/cc_in2p3.py +++ b/parsl/configs/cc_in2p3.py @@ -1,7 +1,7 @@ -from parsl.config import Config from parsl.channels import LocalChannel -from parsl.providers import GridEngineProvider +from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import GridEngineProvider config = Config( executors=[ diff --git a/parsl/configs/ec2.py b/parsl/configs/ec2.py index 9028baff8d..efe2afcfe8 100644 --- a/parsl/configs/ec2.py +++ b/parsl/configs/ec2.py @@ -1,6 +1,6 @@ from parsl.config import Config -from parsl.providers import AWSProvider from parsl.executors import HighThroughputExecutor +from parsl.providers import AWSProvider config = Config( executors=[ diff --git a/parsl/configs/expanse.py b/parsl/configs/expanse.py index 2335831c07..e8f5db9cb7 100644 --- a/parsl/configs/expanse.py +++ b/parsl/configs/expanse.py @@ -1,8 +1,7 @@ from parsl.config import Config +from parsl.executors import HighThroughputExecutor from parsl.launchers import SrunLauncher from parsl.providers import SlurmProvider -from parsl.executors import HighThroughputExecutor - config = Config( executors=[ diff --git a/parsl/configs/frontera.py b/parsl/configs/frontera.py index 3b73587b18..1aa4639bea 100644 --- a/parsl/configs/frontera.py +++ b/parsl/configs/frontera.py @@ -1,9 +1,8 @@ -from parsl.config import Config from parsl.channels import LocalChannel -from parsl.providers import SlurmProvider +from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.launchers import SrunLauncher - +from parsl.providers import SlurmProvider """ This config assumes that it is used to launch parsl tasks from the login nodes of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes. diff --git a/parsl/configs/htex_local.py b/parsl/configs/htex_local.py index 1070d16309..da34f59f81 100644 --- a/parsl/configs/htex_local.py +++ b/parsl/configs/htex_local.py @@ -1,8 +1,7 @@ -from parsl.providers import LocalProvider from parsl.channels import LocalChannel - from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import LocalProvider config = Config( executors=[ diff --git a/parsl/configs/illinoiscluster.py b/parsl/configs/illinoiscluster.py index 3e417d62a7..3f3585d3b6 100644 --- a/parsl/configs/illinoiscluster.py +++ b/parsl/configs/illinoiscluster.py @@ -1,7 +1,7 @@ from parsl.config import Config -from parsl.providers import SlurmProvider from parsl.executors import HighThroughputExecutor from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider """ This config assumes that it is used to launch parsl tasks from the login nodes of the Campus Cluster at UIUC. Each job submitted to the scheduler will request 2 nodes for 10 minutes. diff --git a/parsl/configs/kubernetes.py b/parsl/configs/kubernetes.py index 7f085745f1..829f3b81c3 100644 --- a/parsl/configs/kubernetes.py +++ b/parsl/configs/kubernetes.py @@ -1,8 +1,7 @@ +from parsl.addresses import address_by_route from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.providers import KubernetesProvider -from parsl.addresses import address_by_route - config = Config( executors=[ diff --git a/parsl/configs/midway.py b/parsl/configs/midway.py index 7e88b55fb5..251eb419b1 100644 --- a/parsl/configs/midway.py +++ b/parsl/configs/midway.py @@ -1,8 +1,8 @@ +from parsl.addresses import address_by_interface from parsl.config import Config -from parsl.providers import SlurmProvider -from parsl.launchers import SrunLauncher from parsl.executors import HighThroughputExecutor -from parsl.addresses import address_by_interface +from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider config = Config( executors=[ diff --git a/parsl/configs/osg.py b/parsl/configs/osg.py index 0f01009901..016d40630d 100644 --- a/parsl/configs/osg.py +++ b/parsl/configs/osg.py @@ -1,6 +1,6 @@ from parsl.config import Config -from parsl.providers import CondorProvider from parsl.executors import HighThroughputExecutor +from parsl.providers import CondorProvider config = Config( executors=[ diff --git a/parsl/configs/polaris.py b/parsl/configs/polaris.py index 94f96963ce..3c6b96959d 100644 --- a/parsl/configs/polaris.py +++ b/parsl/configs/polaris.py @@ -1,8 +1,8 @@ from parsl.addresses import address_by_interface +from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.launchers import MpiExecLauncher from parsl.providers import PBSProProvider -from parsl.config import Config # There are three user parameters to change for the PBSProProvider: # YOUR_ACCOUNT: Account to charge usage diff --git a/parsl/configs/stampede2.py b/parsl/configs/stampede2.py index f39fe0791d..0ffb0e3314 100644 --- a/parsl/configs/stampede2.py +++ b/parsl/configs/stampede2.py @@ -1,10 +1,9 @@ +from parsl.addresses import address_by_interface from parsl.config import Config -from parsl.providers import SlurmProvider -from parsl.launchers import SrunLauncher -from parsl.executors import HighThroughputExecutor from parsl.data_provider.globus import GlobusStaging -from parsl.addresses import address_by_interface - +from parsl.executors import HighThroughputExecutor +from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider config = Config( executors=[ diff --git a/parsl/configs/summit.py b/parsl/configs/summit.py index 575a16b1a7..2695f2da7f 100644 --- a/parsl/configs/summit.py +++ b/parsl/configs/summit.py @@ -1,11 +1,9 @@ +from parsl.addresses import address_by_interface from parsl.config import Config from parsl.executors import HighThroughputExecutor - from parsl.launchers import JsrunLauncher from parsl.providers import LSFProvider -from parsl.addresses import address_by_interface - config = Config( executors=[ HighThroughputExecutor( diff --git a/parsl/configs/toss3_llnl.py b/parsl/configs/toss3_llnl.py index 1ced615f4d..a7820b3ca4 100644 --- a/parsl/configs/toss3_llnl.py +++ b/parsl/configs/toss3_llnl.py @@ -1,8 +1,7 @@ from parsl.config import Config from parsl.executors import FluxExecutor -from parsl.providers import SlurmProvider from parsl.launchers import SrunLauncher - +from parsl.providers import SlurmProvider config = Config( executors=[ diff --git a/parsl/configs/vineex_local.py b/parsl/configs/vineex_local.py index 488b74a973..c88d92213c 100644 --- a/parsl/configs/vineex_local.py +++ b/parsl/configs/vineex_local.py @@ -1,8 +1,8 @@ -from parsl.config import Config -from parsl.executors.taskvine import TaskVineExecutor -from parsl.executors.taskvine import TaskVineManagerConfig import uuid +from parsl.config import Config +from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig + config = Config( executors=[ TaskVineExecutor( diff --git a/parsl/configs/wqex_local.py b/parsl/configs/wqex_local.py index 7cf1d5bde0..8a4d570883 100644 --- a/parsl/configs/wqex_local.py +++ b/parsl/configs/wqex_local.py @@ -1,8 +1,8 @@ +import uuid + from parsl.config import Config from parsl.executors import WorkQueueExecutor -import uuid - config = Config( executors=[ WorkQueueExecutor( diff --git a/parsl/data_provider/data_manager.py b/parsl/data_provider/data_manager.py index d7da4bbfe6..db5d242d36 100644 --- a/parsl/data_provider/data_manager.py +++ b/parsl/data_provider/data_manager.py @@ -1,14 +1,14 @@ import logging from concurrent.futures import Future -from typing import Any, Callable, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable, List, Optional from parsl.app.futures import DataFuture -from parsl.data_provider.files import File from parsl.data_provider.file_noop import NoOpFileStaging +from parsl.data_provider.files import File from parsl.data_provider.ftp import FTPSeparateTaskStaging from parsl.data_provider.http import HTTPSeparateTaskStaging -from parsl.data_provider.zip import ZipFileStaging from parsl.data_provider.staging import Staging +from parsl.data_provider.zip import ZipFileStaging if TYPE_CHECKING: from parsl.dataflow.dflow import DataFlowKernel diff --git a/parsl/data_provider/file_noop.py b/parsl/data_provider/file_noop.py index 6f627c2257..69c386697d 100644 --- a/parsl/data_provider/file_noop.py +++ b/parsl/data_provider/file_noop.py @@ -1,8 +1,7 @@ import logging -from parsl.utils import RepresentationMixin from parsl.data_provider.staging import Staging - +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/data_provider/files.py b/parsl/data_provider/files.py index ba4cfa7711..4263753dce 100644 --- a/parsl/data_provider/files.py +++ b/parsl/data_provider/files.py @@ -5,13 +5,13 @@ on where (client-side, remote-side, intermediary-side) the File.filepath is being called from. """ -import os - -import typeguard import logging +import os from typing import Optional, Union from urllib.parse import urlparse +import typeguard + logger = logging.getLogger(__name__) diff --git a/parsl/data_provider/ftp.py b/parsl/data_provider/ftp.py index d9f1931fd6..88cefe612c 100644 --- a/parsl/data_provider/ftp.py +++ b/parsl/data_provider/ftp.py @@ -3,10 +3,8 @@ import os import parsl - -from parsl.utils import RepresentationMixin from parsl.data_provider.staging import Staging - +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/data_provider/globus.py b/parsl/data_provider/globus.py index 50bddc5ecf..e63b0aa82f 100644 --- a/parsl/data_provider/globus.py +++ b/parsl/data_provider/globus.py @@ -1,15 +1,16 @@ -import logging import json -import globus_sdk +import logging import os -import parsl -import typeguard - from functools import partial from typing import Optional + +import globus_sdk +import typeguard + +import parsl from parsl.app.app import python_app -from parsl.utils import RepresentationMixin from parsl.data_provider.staging import Staging +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/data_provider/http.py b/parsl/data_provider/http.py index ee15c44965..4553839ba8 100644 --- a/parsl/data_provider/http.py +++ b/parsl/data_provider/http.py @@ -1,11 +1,11 @@ import logging import os + import requests import parsl - -from parsl.utils import RepresentationMixin from parsl.data_provider.staging import Staging +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/data_provider/rsync.py b/parsl/data_provider/rsync.py index f8cf5c615f..d86b37bdf3 100644 --- a/parsl/data_provider/rsync.py +++ b/parsl/data_provider/rsync.py @@ -1,8 +1,8 @@ import logging import os -from parsl.utils import RepresentationMixin from parsl.data_provider.staging import Staging +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/data_provider/staging.py b/parsl/data_provider/staging.py index 4ce9929613..37e5c7df80 100644 --- a/parsl/data_provider/staging.py +++ b/parsl/data_provider/staging.py @@ -1,9 +1,9 @@ from concurrent.futures import Future -from typing import Optional, Callable +from typing import TYPE_CHECKING, Callable, Optional + from parsl.app.futures import DataFuture from parsl.data_provider.files import File -from typing import TYPE_CHECKING if TYPE_CHECKING: from parsl.data_provider.data_manager import DataManager diff --git a/parsl/data_provider/zip.py b/parsl/data_provider/zip.py index 58a067ea76..167ddbbdf7 100644 --- a/parsl/data_provider/zip.py +++ b/parsl/data_provider/zip.py @@ -1,16 +1,15 @@ -import filelock import logging import os -import parsl import zipfile - from typing import Tuple -from parsl.data_provider.staging import Staging +import filelock + +import parsl from parsl.data_provider.files import File +from parsl.data_provider.staging import Staging from parsl.errors import ParslError - logger = logging.getLogger(__name__) diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py index 90004a4737..dffa7e52fd 100644 --- a/parsl/dataflow/dflow.py +++ b/parsl/dataflow/dflow.py @@ -1,23 +1,25 @@ from __future__ import annotations + import atexit +import datetime +import inspect import logging import os import pathlib import pickle import random -import time -import typeguard -import inspect -import threading import sys -import datetime +import threading +import time +from concurrent.futures import Future +from functools import partial from getpass import getuser -from typeguard import typechecked +from socket import gethostname from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union from uuid import uuid4 -from socket import gethostname -from concurrent.futures import Future -from functools import partial + +import typeguard +from typeguard import typechecked import parsl from parsl.app.errors import RemoteExceptionWrapper @@ -31,21 +33,24 @@ from parsl.dataflow.futures import AppFuture from parsl.dataflow.memoization import Memoizer from parsl.dataflow.rundirs import make_rundir -from parsl.dataflow.states import States, FINAL_STATES, FINAL_FAILURE_STATES +from parsl.dataflow.states import FINAL_FAILURE_STATES, FINAL_STATES, States from parsl.dataflow.taskrecord import TaskRecord -from parsl.errors import ConfigurationError, InternalConsistencyError, NoDataFlowKernelError -from parsl.jobs.job_status_poller import JobStatusPoller -from parsl.usage_tracking.usage import UsageTracker +from parsl.errors import ( + ConfigurationError, + InternalConsistencyError, + NoDataFlowKernelError, +) from parsl.executors.base import ParslExecutor from parsl.executors.status_handling import BlockProviderExecutor from parsl.executors.threads import ThreadPoolExecutor +from parsl.jobs.job_status_poller import JobStatusPoller from parsl.monitoring import MonitoringHub +from parsl.monitoring.message_type import MessageType from parsl.monitoring.remote import monitor_wrapper from parsl.process_loggers import wrap_with_logs from parsl.providers.base import ExecutionProvider -from parsl.utils import get_version, get_std_fname_mode, get_all_checkpoints, Timer - -from parsl.monitoring.message_type import MessageType +from parsl.usage_tracking.usage import UsageTracker +from parsl.utils import Timer, get_all_checkpoints, get_std_fname_mode, get_version logger = logging.getLogger(__name__) diff --git a/parsl/dataflow/errors.py b/parsl/dataflow/errors.py index 5d6c0c8710..926039c4e7 100644 --- a/parsl/dataflow/errors.py +++ b/parsl/dataflow/errors.py @@ -1,6 +1,7 @@ -from parsl.errors import ParslError from typing import Optional, Sequence, Tuple +from parsl.errors import ParslError + class DataFlowException(ParslError): """Base class for all exceptions. diff --git a/parsl/dataflow/futures.py b/parsl/dataflow/futures.py index 8568b677dc..616a3d3bad 100644 --- a/parsl/dataflow/futures.py +++ b/parsl/dataflow/futures.py @@ -1,12 +1,11 @@ from __future__ import annotations -from concurrent.futures import Future import logging import threading +from concurrent.futures import Future from typing import Any, Optional, Sequence, Union import parsl.app.app as app - from parsl.app.futures import DataFuture from parsl.dataflow.taskrecord import TaskRecord diff --git a/parsl/dataflow/memoization.py b/parsl/dataflow/memoization.py index e4d657ccaa..551bd0b9d4 100644 --- a/parsl/dataflow/memoization.py +++ b/parsl/dataflow/memoization.py @@ -1,18 +1,18 @@ from __future__ import annotations + import hashlib -from functools import lru_cache, singledispatch import logging import pickle -from parsl.dataflow.taskrecord import TaskRecord +from functools import lru_cache, singledispatch +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from typing import Dict, Any, List, Optional, TYPE_CHECKING +from parsl.dataflow.taskrecord import TaskRecord if TYPE_CHECKING: from parsl import DataFlowKernel # import loop at runtime - needed for typechecking - TODO turn into "if typing:" -from concurrent.futures import Future - import types +from concurrent.futures import Future logger = logging.getLogger(__name__) diff --git a/parsl/dataflow/rundirs.py b/parsl/dataflow/rundirs.py index f32979473f..f5ecda6984 100644 --- a/parsl/dataflow/rundirs.py +++ b/parsl/dataflow/rundirs.py @@ -1,6 +1,6 @@ +import logging import os from glob import glob -import logging logger = logging.getLogger(__name__) diff --git a/parsl/dataflow/taskrecord.py b/parsl/dataflow/taskrecord.py index 07da5386f8..f48aecceb9 100644 --- a/parsl/dataflow/taskrecord.py +++ b/parsl/dataflow/taskrecord.py @@ -1,19 +1,18 @@ from __future__ import annotations -import threading import datetime -from typing_extensions import TypedDict +import threading from concurrent.futures import Future - # only for type checking: -from typing import Any, Callable, Dict, Optional, List, Sequence, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union + +from typing_extensions import TypedDict if TYPE_CHECKING: from parsl.dataflow.futures import AppFuture import parsl.dataflow.dflow as dflow - from parsl.dataflow.states import States diff --git a/parsl/executors/__init__.py b/parsl/executors/__init__.py index 707c336caf..bc29204502 100644 --- a/parsl/executors/__init__.py +++ b/parsl/executors/__init__.py @@ -1,8 +1,8 @@ -from parsl.executors.threads import ThreadPoolExecutor -from parsl.executors.workqueue.executor import WorkQueueExecutor +from parsl.executors.flux.executor import FluxExecutor from parsl.executors.high_throughput.executor import HighThroughputExecutor from parsl.executors.high_throughput.mpi_executor import MPIExecutor -from parsl.executors.flux.executor import FluxExecutor +from parsl.executors.threads import ThreadPoolExecutor +from parsl.executors.workqueue.executor import WorkQueueExecutor __all__ = ['ThreadPoolExecutor', 'HighThroughputExecutor', diff --git a/parsl/executors/base.py b/parsl/executors/base.py index 966705c7c5..b00aa55680 100644 --- a/parsl/executors/base.py +++ b/parsl/executors/base.py @@ -2,6 +2,7 @@ from abc import ABCMeta, abstractmethod from concurrent.futures import Future from typing import Any, Callable, Dict, Optional + from typing_extensions import Literal, Self from parsl.monitoring.radios import MonitoringRadio diff --git a/parsl/executors/flux/execute_parsl_task.py b/parsl/executors/flux/execute_parsl_task.py index a2ce0ed011..ddf3c67e14 100644 --- a/parsl/executors/flux/execute_parsl_task.py +++ b/parsl/executors/flux/execute_parsl_task.py @@ -1,12 +1,12 @@ """Script for executing tasks inside of Flux jobs.""" import argparse -import os import logging +import os +from parsl.executors.flux import TaskResult from parsl.executors.high_throughput.process_worker_pool import execute_task from parsl.serialize import serialize -from parsl.executors.flux import TaskResult def main(): diff --git a/parsl/executors/flux/executor.py b/parsl/executors/flux/executor.py index dfea11acb3..c4926abb68 100644 --- a/parsl/executors/flux/executor.py +++ b/parsl/executors/flux/executor.py @@ -1,33 +1,32 @@ """Defines the FluxExecutor class.""" +import collections import concurrent.futures as cf import functools +import itertools import os +import queue +import shutil import sys -import uuid import threading -import itertools -import shutil -import queue -from socket import gethostname -import collections -from collections.abc import Mapping, Callable -from typing import Optional, Any, Dict +import uuid import weakref +from collections.abc import Callable, Mapping +from socket import gethostname +from typing import Any, Dict, Optional import zmq -from parsl.utils import RepresentationMixin +from parsl.app.errors import AppException from parsl.executors.base import ParslExecutor +from parsl.executors.errors import ScalingFailed from parsl.executors.flux.execute_parsl_task import __file__ as _WORKER_PATH from parsl.executors.flux.flux_instance_manager import __file__ as _MANAGER_PATH -from parsl.executors.errors import ScalingFailed from parsl.providers import LocalProvider from parsl.providers.base import ExecutionProvider from parsl.serialize import deserialize, pack_res_spec_apply_message from parsl.serialize.errors import SerializationError -from parsl.app.errors import AppException - +from parsl.utils import RepresentationMixin _WORKER_PATH = os.path.realpath(_WORKER_PATH) _MANAGER_PATH = os.path.realpath(_MANAGER_PATH) diff --git a/parsl/executors/flux/flux_instance_manager.py b/parsl/executors/flux/flux_instance_manager.py index cf0f7d376a..3d760bb5c8 100644 --- a/parsl/executors/flux/flux_instance_manager.py +++ b/parsl/executors/flux/flux_instance_manager.py @@ -1,10 +1,10 @@ """Script meant to be the initial program of a Flux instance.""" import argparse +import logging import os from os.path import dirname -import logging -from socket import gethostname, gethostbyname +from socket import gethostbyname, gethostname import zmq @@ -16,8 +16,8 @@ def main(): encapsulating Flux instance. """ # flux imports only available when launched under Flux instance - import flux.job import flux + import flux.job logging.basicConfig( level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(message)s" diff --git a/parsl/executors/high_throughput/executor.py b/parsl/executors/high_throughput/executor.py index 39ebccecde..b5480e7937 100644 --- a/parsl/executors/high_throughput/executor.py +++ b/parsl/executors/high_throughput/executor.py @@ -1,44 +1,39 @@ +import logging +import math +import pickle +import threading import typing +import warnings from collections import defaultdict from concurrent.futures import Future -import typeguard -import logging -import threading -import pickle from dataclasses import dataclass from multiprocessing import Process -from typing import Dict, Sequence -from typing import List, Optional, Tuple, Union, Callable -import math -import warnings +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union + +import typeguard import parsl.launchers -from parsl.usage_tracking.api import UsageInformation -from parsl.serialize import pack_res_spec_apply_message, deserialize -from parsl.serialize.errors import SerializationError, DeserializationError +from parsl import curvezmq +from parsl.addresses import get_all_addresses from parsl.app.errors import RemoteExceptionWrapper -from parsl.jobs.states import JobStatus, JobState, TERMINAL_STATES -from parsl.executors.high_throughput import zmq_pipes -from parsl.executors.high_throughput import interchange +from parsl.data_provider.staging import Staging +from parsl.executors.errors import BadMessage, ScalingFailed +from parsl.executors.high_throughput import interchange, zmq_pipes from parsl.executors.high_throughput.errors import CommandClientTimeoutError -from parsl.executors.errors import ( - BadMessage, ScalingFailed, -) from parsl.executors.high_throughput.mpi_prefix_composer import ( VALID_LAUNCHERS, - validate_resource_spec + validate_resource_spec, ) - -from parsl import curvezmq from parsl.executors.status_handling import BlockProviderExecutor -from parsl.providers.base import ExecutionProvider -from parsl.data_provider.staging import Staging -from parsl.addresses import get_all_addresses -from parsl.process_loggers import wrap_with_logs - +from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus from parsl.multiprocessing import ForkProcess -from parsl.utils import RepresentationMixin +from parsl.process_loggers import wrap_with_logs from parsl.providers import LocalProvider +from parsl.providers.base import ExecutionProvider +from parsl.serialize import deserialize, pack_res_spec_apply_message +from parsl.serialize.errors import DeserializationError, SerializationError +from parsl.usage_tracking.api import UsageInformation +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/executors/high_throughput/interchange.py b/parsl/executors/high_throughput/interchange.py index 7034d703c7..4b3bab3563 100644 --- a/parsl/executors/high_throughput/interchange.py +++ b/parsl/executors/high_throughput/interchange.py @@ -1,30 +1,28 @@ #!/usr/bin/env python -import zmq +import datetime +import json +import logging import os -import sys +import pickle import platform +import queue import random -import time -import datetime -import pickle import signal -import logging -import queue +import sys import threading -import json +import time +from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, cast -from typing import cast, Any, Dict, NoReturn, Sequence, Set, Optional, Tuple, List +import zmq from parsl import curvezmq -from parsl.utils import setproctitle -from parsl.version import VERSION as PARSL_VERSION -from parsl.serialize import serialize as serialize_object - from parsl.app.errors import RemoteExceptionWrapper from parsl.executors.high_throughput.manager_record import ManagerRecord from parsl.monitoring.message_type import MessageType from parsl.process_loggers import wrap_with_logs - +from parsl.serialize import serialize as serialize_object +from parsl.utils import setproctitle +from parsl.version import VERSION as PARSL_VERSION PKL_HEARTBEAT_CODE = pickle.dumps((2 ** 32) - 1) PKL_DRAINED_CODE = pickle.dumps((2 ** 32) - 2) diff --git a/parsl/executors/high_throughput/manager_record.py b/parsl/executors/high_throughput/manager_record.py index 0ae4fcedc1..7e58b53954 100644 --- a/parsl/executors/high_throughput/manager_record.py +++ b/parsl/executors/high_throughput/manager_record.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Any, List, Optional + from typing_extensions import TypedDict diff --git a/parsl/executors/high_throughput/monitoring_info.py b/parsl/executors/high_throughput/monitoring_info.py index 0903b9be04..f39f4693e4 100644 --- a/parsl/executors/high_throughput/monitoring_info.py +++ b/parsl/executors/high_throughput/monitoring_info.py @@ -3,6 +3,7 @@ # then be acquired by any other code running in # a worker context - specifically the monitoring # wrapper code. -from typing import Optional from queue import Queue +from typing import Optional + result_queue: Optional[Queue] = None diff --git a/parsl/executors/high_throughput/mpi_executor.py b/parsl/executors/high_throughput/mpi_executor.py index 829319cbef..69071557c8 100644 --- a/parsl/executors/high_throughput/mpi_executor.py +++ b/parsl/executors/high_throughput/mpi_executor.py @@ -1,10 +1,13 @@ """A simplified interface for HTEx when running in MPI mode""" -from typing import Optional, Tuple, List, Union, Callable, Dict +from typing import Callable, Dict, List, Optional, Tuple, Union import typeguard from parsl.data_provider.staging import Staging -from parsl.executors.high_throughput.executor import HighThroughputExecutor, GENERAL_HTEX_PARAM_DOCS +from parsl.executors.high_throughput.executor import ( + GENERAL_HTEX_PARAM_DOCS, + HighThroughputExecutor, +) from parsl.executors.status_handling import BlockProviderExecutor from parsl.jobs.states import JobStatus from parsl.providers import LocalProvider diff --git a/parsl/executors/high_throughput/mpi_prefix_composer.py b/parsl/executors/high_throughput/mpi_prefix_composer.py index 07abf8f9d5..78c5d8b867 100644 --- a/parsl/executors/high_throughput/mpi_prefix_composer.py +++ b/parsl/executors/high_throughput/mpi_prefix_composer.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, List, Tuple, Set +from typing import Dict, List, Set, Tuple logger = logging.getLogger(__name__) diff --git a/parsl/executors/high_throughput/mpi_resource_management.py b/parsl/executors/high_throughput/mpi_resource_management.py index adb5ee252e..3dd3294648 100644 --- a/parsl/executors/high_throughput/mpi_resource_management.py +++ b/parsl/executors/high_throughput/mpi_resource_management.py @@ -8,8 +8,7 @@ from typing import Dict, List from parsl.multiprocessing import SpawnContext -from parsl.serialize import (pack_res_spec_apply_message, - unpack_res_spec_apply_message) +from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message logger = logging.getLogger(__name__) diff --git a/parsl/executors/high_throughput/probe.py b/parsl/executors/high_throughput/probe.py index e1131b0b90..6b8defe601 100644 --- a/parsl/executors/high_throughput/probe.py +++ b/parsl/executors/high_throughput/probe.py @@ -1,11 +1,13 @@ -import zmq import argparse -import uuid -import time import logging -from parsl.addresses import get_all_addresses +import time +import uuid + +import zmq from zmq.utils.monitor import recv_monitor_message +from parsl.addresses import get_all_addresses + logger = logging.getLogger(__name__) diff --git a/parsl/executors/high_throughput/process_worker_pool.py b/parsl/executors/high_throughput/process_worker_pool.py index 21d0dd5c2b..5a3b383dad 100755 --- a/parsl/executors/high_throughput/process_worker_pool.py +++ b/parsl/executors/high_throughput/process_worker_pool.py @@ -1,39 +1,41 @@ #!/usr/bin/env python3 import argparse +import json import logging +import math +import multiprocessing import os -import sys +import pickle import platform +import queue +import sys import threading -import pickle import time -import queue import uuid -from typing import Sequence, Optional, Dict, List - -import zmq -import math -import json -import psutil -import multiprocessing from multiprocessing.managers import DictProxy from multiprocessing.sharedctypes import Synchronized +from typing import Dict, List, Optional, Sequence + +import psutil +import zmq from parsl import curvezmq -from parsl.process_loggers import wrap_with_logs -from parsl.version import VERSION as PARSL_VERSION from parsl.app.errors import RemoteExceptionWrapper from parsl.executors.high_throughput.errors import WorkerLost -from parsl.executors.high_throughput.probe import probe_addresses -from parsl.multiprocessing import SpawnContext -from parsl.serialize import unpack_res_spec_apply_message, serialize +from parsl.executors.high_throughput.mpi_prefix_composer import ( + VALID_LAUNCHERS, + compose_all, +) from parsl.executors.high_throughput.mpi_resource_management import ( + MPITaskScheduler, TaskScheduler, - MPITaskScheduler ) - -from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS +from parsl.executors.high_throughput.probe import probe_addresses +from parsl.multiprocessing import SpawnContext +from parsl.process_loggers import wrap_with_logs +from parsl.serialize import serialize, unpack_res_spec_apply_message +from parsl.version import VERSION as PARSL_VERSION HEARTBEAT_CODE = (2 ** 32) - 1 DRAINED_CODE = (2 ** 32) - 2 diff --git a/parsl/executors/high_throughput/zmq_pipes.py b/parsl/executors/high_throughput/zmq_pipes.py index ca0a6ff251..eb0e0f515b 100644 --- a/parsl/executors/high_throughput/zmq_pipes.py +++ b/parsl/executors/high_throughput/zmq_pipes.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 -import zmq import logging import threading import time from typing import Optional +import zmq + from parsl import curvezmq from parsl.errors import InternalConsistencyError -from parsl.executors.high_throughput.errors import CommandClientBadError, CommandClientTimeoutError +from parsl.executors.high_throughput.errors import ( + CommandClientBadError, + CommandClientTimeoutError, +) logger = logging.getLogger(__name__) diff --git a/parsl/executors/radical/executor.py b/parsl/executors/radical/executor.py index 2200fefe26..c7ea1a8dd6 100644 --- a/parsl/executors/radical/executor.py +++ b/parsl/executors/radical/executor.py @@ -1,30 +1,30 @@ """RadicalPilotExecutor builds on the RADICAL-Pilot/Parsl """ +import inspect +import logging import os -import sys -import time -import parsl import queue -import logging -import inspect -import requests -import typeguard +import sys import threading as mt - +import time +from concurrent.futures import Future from functools import partial -from typing import Optional, Dict from pathlib import Path, PosixPath -from concurrent.futures import Future +from typing import Dict, Optional + +import requests +import typeguard +import parsl +from parsl.app.errors import BashExitFailure, RemoteExceptionWrapper from parsl.app.python import timeout -from .rpex_resources import ResourceConfig from parsl.data_provider.files import File -from parsl.utils import RepresentationMixin -from parsl.app.errors import BashExitFailure from parsl.executors.base import ParslExecutor -from parsl.app.errors import RemoteExceptionWrapper from parsl.serialize import deserialize, pack_res_spec_apply_message -from parsl.serialize.errors import SerializationError, DeserializationError +from parsl.serialize.errors import DeserializationError, SerializationError +from parsl.utils import RepresentationMixin + +from .rpex_resources import ResourceConfig try: import radical.pilot as rp diff --git a/parsl/executors/radical/rpex_master.py b/parsl/executors/radical/rpex_master.py index 8094de6da9..6d3627e46f 100755 --- a/parsl/executors/radical/rpex_master.py +++ b/parsl/executors/radical/rpex_master.py @@ -2,9 +2,8 @@ import sys -import radical.utils as ru import radical.pilot as rp - +import radical.utils as ru # ------------------------------------------------------------------------------ # diff --git a/parsl/executors/radical/rpex_resources.py b/parsl/executors/radical/rpex_resources.py index 241392d8f2..f4daf9aa19 100644 --- a/parsl/executors/radical/rpex_resources.py +++ b/parsl/executors/radical/rpex_resources.py @@ -1,6 +1,5 @@ -import sys import json - +import sys from typing import List _setup_paths: List[str] = [] diff --git a/parsl/executors/radical/rpex_worker.py b/parsl/executors/radical/rpex_worker.py index 2225499e89..db1b7d2bea 100644 --- a/parsl/executors/radical/rpex_worker.py +++ b/parsl/executors/radical/rpex_worker.py @@ -1,10 +1,11 @@ import sys + import radical.pilot as rp import parsl.app.errors as pe from parsl.app.bash import remote_side_bash_executor -from parsl.serialize import unpack_res_spec_apply_message, serialize from parsl.executors.high_throughput.process_worker_pool import execute_task +from parsl.serialize import serialize, unpack_res_spec_apply_message class ParslWorker: diff --git a/parsl/executors/status_handling.py b/parsl/executors/status_handling.py index 9a4d48a2f4..4d29439670 100644 --- a/parsl/executors/status_handling.py +++ b/parsl/executors/status_handling.py @@ -1,17 +1,18 @@ from __future__ import annotations + import datetime import logging import threading import time -from itertools import compress from abc import abstractmethod, abstractproperty from concurrent.futures import Future -from typing import List, Any, Dict, Optional, Sequence, Tuple, Union, Callable +from itertools import compress +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union from parsl.executors.base import ParslExecutor from parsl.executors.errors import BadStateException, ScalingFailed -from parsl.jobs.states import JobStatus, JobState -from parsl.jobs.error_handlers import simple_error_handler, noop_error_handler +from parsl.jobs.error_handlers import noop_error_handler, simple_error_handler +from parsl.jobs.states import JobState, JobStatus from parsl.monitoring.message_type import MessageType from parsl.providers.base import ExecutionProvider from parsl.utils import AtomicIDCounter diff --git a/parsl/executors/taskvine/__init__.py b/parsl/executors/taskvine/__init__.py index 10f22696ec..a9f0c72360 100644 --- a/parsl/executors/taskvine/__init__.py +++ b/parsl/executors/taskvine/__init__.py @@ -1,5 +1,5 @@ from parsl.executors.taskvine.executor import TaskVineExecutor -from parsl.executors.taskvine.manager_config import TaskVineManagerConfig from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig +from parsl.executors.taskvine.manager_config import TaskVineManagerConfig __all__ = ['TaskVineExecutor', 'TaskVineManagerConfig', 'TaskVineFactoryConfig'] diff --git a/parsl/executors/taskvine/errors.py b/parsl/executors/taskvine/errors.py index 8a97c45d5e..3d7aac94b0 100644 --- a/parsl/executors/taskvine/errors.py +++ b/parsl/executors/taskvine/errors.py @@ -1,5 +1,5 @@ -from parsl.errors import ParslError from parsl.app.errors import AppException +from parsl.errors import ParslError class TaskVineTaskFailure(AppException): diff --git a/parsl/executors/taskvine/exec_parsl_function.py b/parsl/executors/taskvine/exec_parsl_function.py index 0bd86b7a99..cf02ced014 100644 --- a/parsl/executors/taskvine/exec_parsl_function.py +++ b/parsl/executors/taskvine/exec_parsl_function.py @@ -1,10 +1,10 @@ -import traceback import sys +import traceback from parsl.app.errors import RemoteExceptionWrapper from parsl.data_provider.files import File -from parsl.utils import get_std_fname_mode from parsl.serialize import deserialize, serialize +from parsl.utils import get_std_fname_mode # This scripts executes a parsl function which is pickled in 4 files: # diff --git a/parsl/executors/taskvine/executor.py b/parsl/executors/taskvine/executor.py index a2dea60170..6cfedf92bb 100644 --- a/parsl/executors/taskvine/executor.py +++ b/parsl/executors/taskvine/executor.py @@ -3,46 +3,45 @@ high-throughput system for delegating Parsl tasks to thousands of remote machines """ -# Import Python built-in libraries -import threading -import multiprocessing -import logging -import tempfile import hashlib -import subprocess +import inspect +import itertools +import logging +import multiprocessing import os import queue -import inspect import shutil -import itertools +import subprocess +import tempfile + +# Import Python built-in libraries +import threading import uuid from concurrent.futures import Future -from typing import List, Optional, Union, Literal +from typing import List, Literal, Optional, Union + +# Import other libraries +import typeguard # Import Parsl constructs import parsl.utils as putils -from parsl.data_provider.staging import Staging -from parsl.serialize import serialize, deserialize +from parsl.addresses import get_any_address from parsl.data_provider.files import File +from parsl.data_provider.staging import Staging from parsl.errors import OptionalModuleMissing -from parsl.providers.base import ExecutionProvider -from parsl.providers import LocalProvider, CondorProvider -from parsl.process_loggers import wrap_with_logs -from parsl.addresses import get_any_address from parsl.executors.errors import ExecutorError from parsl.executors.status_handling import BlockProviderExecutor from parsl.executors.taskvine import exec_parsl_function -from parsl.executors.taskvine.manager_config import TaskVineManagerConfig +from parsl.executors.taskvine.errors import TaskVineManagerFailure, TaskVineTaskFailure +from parsl.executors.taskvine.factory import _taskvine_factory from parsl.executors.taskvine.factory_config import TaskVineFactoryConfig -from parsl.executors.taskvine.errors import TaskVineTaskFailure -from parsl.executors.taskvine.errors import TaskVineManagerFailure -from parsl.executors.taskvine.utils import ParslTaskToVine -from parsl.executors.taskvine.utils import ParslFileToVine from parsl.executors.taskvine.manager import _taskvine_submit_wait -from parsl.executors.taskvine.factory import _taskvine_factory - -# Import other libraries -import typeguard +from parsl.executors.taskvine.manager_config import TaskVineManagerConfig +from parsl.executors.taskvine.utils import ParslFileToVine, ParslTaskToVine +from parsl.process_loggers import wrap_with_logs +from parsl.providers import CondorProvider, LocalProvider +from parsl.providers.base import ExecutionProvider +from parsl.serialize import deserialize, serialize logger = logging.getLogger(__name__) diff --git a/parsl/executors/taskvine/factory.py b/parsl/executors/taskvine/factory.py index 484877d1bd..20409efac8 100644 --- a/parsl/executors/taskvine/factory.py +++ b/parsl/executors/taskvine/factory.py @@ -1,7 +1,7 @@ import logging -from parsl.process_loggers import wrap_with_logs from parsl.executors.taskvine.errors import TaskVineFactoryFailure +from parsl.process_loggers import wrap_with_logs # This try except clause prevents import errors # when TaskVine is not used in Parsl. diff --git a/parsl/executors/taskvine/manager.py b/parsl/executors/taskvine/manager.py index c49fa454e4..e5a4986062 100644 --- a/parsl/executors/taskvine/manager.py +++ b/parsl/executors/taskvine/manager.py @@ -1,25 +1,23 @@ -import logging import hashlib -import subprocess +import logging import os import queue import shutil +import subprocess import uuid -from parsl.utils import setproctitle -from parsl.process_loggers import wrap_with_logs from parsl.executors.taskvine import exec_parsl_function -from parsl.executors.taskvine.utils import VineTaskToParsl -from parsl.executors.taskvine.utils import run_parsl_function +from parsl.executors.taskvine.utils import VineTaskToParsl, run_parsl_function +from parsl.process_loggers import wrap_with_logs +from parsl.utils import setproctitle try: - from ndcctools.taskvine import cvine - from ndcctools.taskvine import Manager - from ndcctools.taskvine import Task - from ndcctools.taskvine import FunctionCall - from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX_THROUGHPUT - from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING - from ndcctools.taskvine.cvine import VINE_ALLOCATION_MODE_MAX + from ndcctools.taskvine import FunctionCall, Manager, Task, cvine + from ndcctools.taskvine.cvine import ( + VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING, + VINE_ALLOCATION_MODE_MAX, + VINE_ALLOCATION_MODE_MAX_THROUGHPUT, + ) except ImportError: _taskvine_enabled = False else: diff --git a/parsl/executors/threads.py b/parsl/executors/threads.py index 97c26adb9e..cbc813d9d4 100644 --- a/parsl/executors/threads.py +++ b/parsl/executors/threads.py @@ -1,14 +1,13 @@ -import logging -import typeguard import concurrent.futures as cf - +import logging from typing import List, Optional +import typeguard + from parsl.data_provider.staging import Staging from parsl.executors.base import ParslExecutor -from parsl.utils import RepresentationMixin from parsl.executors.errors import UnsupportedFeatureError - +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/executors/workqueue/errors.py b/parsl/executors/workqueue/errors.py index 9c5da918bf..3b12d1ad7c 100644 --- a/parsl/executors/workqueue/errors.py +++ b/parsl/executors/workqueue/errors.py @@ -1,7 +1,7 @@ from typing import Optional -from parsl.errors import ParslError from parsl.app.errors import AppException +from parsl.errors import ParslError class WorkQueueTaskFailure(AppException): diff --git a/parsl/executors/workqueue/exec_parsl_function.py b/parsl/executors/workqueue/exec_parsl_function.py index 9aa8674677..06a86a5c8d 100644 --- a/parsl/executors/workqueue/exec_parsl_function.py +++ b/parsl/executors/workqueue/exec_parsl_function.py @@ -1,10 +1,11 @@ +import pickle +import sys +import traceback + from parsl.app.errors import RemoteExceptionWrapper from parsl.data_provider.files import File -from parsl.utils import get_std_fname_mode -import traceback -import sys -import pickle from parsl.serialize import serialize +from parsl.utils import get_std_fname_mode # This scripts executes a parsl function which is pickled in a file: # diff --git a/parsl/executors/workqueue/executor.py b/parsl/executors/workqueue/executor.py index 67628dacb2..0b931bbc31 100644 --- a/parsl/executors/workqueue/executor.py +++ b/parsl/executors/workqueue/executor.py @@ -3,50 +3,49 @@ high-throughput system for delegating Parsl tasks to thousands of remote machines """ -import threading -import multiprocessing -import logging -from concurrent.futures import Future -from ctypes import c_bool - -import tempfile import hashlib -import subprocess +import inspect +import itertools +import logging +import multiprocessing import os -import socket -import time import pickle import queue -import inspect import shutil -import itertools +import socket +import subprocess +import tempfile +import threading +import time +from collections import namedtuple +from concurrent.futures import Future +from ctypes import c_bool +from typing import Dict, List, Optional, Set, Union + +import typeguard -from parsl.serialize import pack_apply_message, deserialize import parsl.utils as putils -from parsl.executors.errors import ExecutorError from parsl.data_provider.files import File +from parsl.data_provider.staging import Staging from parsl.errors import OptionalModuleMissing +from parsl.executors.errors import ExecutorError from parsl.executors.status_handling import BlockProviderExecutor -from parsl.providers.base import ExecutionProvider -from parsl.providers import LocalProvider, CondorProvider from parsl.executors.workqueue import exec_parsl_function from parsl.process_loggers import wrap_with_logs +from parsl.providers import CondorProvider, LocalProvider +from parsl.providers.base import ExecutionProvider +from parsl.serialize import deserialize, pack_apply_message from parsl.utils import setproctitle -import typeguard -from typing import Dict, List, Optional, Set, Union -from parsl.data_provider.staging import Staging - -from .errors import WorkQueueTaskFailure -from .errors import WorkQueueFailure - -from collections import namedtuple +from .errors import WorkQueueFailure, WorkQueueTaskFailure try: import work_queue as wq - from work_queue import WorkQueue - from work_queue import WORK_QUEUE_DEFAULT_PORT - from work_queue import WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT + from work_queue import ( + WORK_QUEUE_ALLOCATION_MODE_MAX_THROUGHPUT, + WORK_QUEUE_DEFAULT_PORT, + WorkQueue, + ) except ImportError: _work_queue_enabled = False WORK_QUEUE_DEFAULT_PORT = 0 diff --git a/parsl/executors/workqueue/parsl_coprocess.py b/parsl/executors/workqueue/parsl_coprocess.py index 366dd8cdfb..0bbcde89cc 100755 --- a/parsl/executors/workqueue/parsl_coprocess.py +++ b/parsl/executors/workqueue/parsl_coprocess.py @@ -1,8 +1,8 @@ #! /usr/bin/env python3 -import socket import json import os +import socket import sys # If enabled, coprocess will print to stdout diff --git a/parsl/jobs/error_handlers.py b/parsl/jobs/error_handlers.py index 19f44eace3..2fb27b6959 100644 --- a/parsl/jobs/error_handlers.py +++ b/parsl/jobs/error_handlers.py @@ -3,8 +3,8 @@ from typing import Dict, Tuple import parsl.executors.status_handling as status_handling -from parsl.jobs.states import JobStatus, JobState from parsl.jobs.errors import TooManyJobFailuresError +from parsl.jobs.states import JobState, JobStatus def noop_error_handler(executor: status_handling.BlockProviderExecutor, status: Dict[str, JobStatus], threshold: int = 3) -> None: diff --git a/parsl/jobs/job_status_poller.py b/parsl/jobs/job_status_poller.py index b8e4625819..172d5195b6 100644 --- a/parsl/jobs/job_status_poller.py +++ b/parsl/jobs/job_status_poller.py @@ -1,13 +1,10 @@ import logging -from typing import List, Sequence, Optional, Union +from typing import List, Optional, Sequence, Union -from parsl.jobs.strategy import Strategy from parsl.executors.status_handling import BlockProviderExecutor - - +from parsl.jobs.strategy import Strategy from parsl.utils import Timer - logger = logging.getLogger(__name__) diff --git a/parsl/jobs/states.py b/parsl/jobs/states.py index a0d4ca504c..7ba4aae94e 100644 --- a/parsl/jobs/states.py +++ b/parsl/jobs/states.py @@ -1,6 +1,6 @@ +import logging import os from enum import IntEnum -import logging from typing import Optional logger = logging.getLogger(__name__) diff --git a/parsl/jobs/strategy.py b/parsl/jobs/strategy.py index cf10b3d2cc..e0898cccb1 100644 --- a/parsl/jobs/strategy.py +++ b/parsl/jobs/strategy.py @@ -1,7 +1,8 @@ from __future__ import annotations + import logging -import time import math +import time import warnings from typing import Dict, List, Optional, Sequence, TypedDict @@ -11,7 +12,6 @@ from parsl.jobs.states import JobState from parsl.process_loggers import wrap_with_logs - logger = logging.getLogger(__name__) diff --git a/parsl/launchers/__init__.py b/parsl/launchers/__init__.py index 80e8589022..155ed55637 100644 --- a/parsl/launchers/__init__.py +++ b/parsl/launchers/__init__.py @@ -1,6 +1,15 @@ -from parsl.launchers.launchers import SimpleLauncher, SingleNodeLauncher, \ - SrunLauncher, AprunLauncher, SrunMPILauncher, WrappedLauncher, \ - GnuParallelLauncher, MpiExecLauncher, MpiRunLauncher, JsrunLauncher +from parsl.launchers.launchers import ( + AprunLauncher, + GnuParallelLauncher, + JsrunLauncher, + MpiExecLauncher, + MpiRunLauncher, + SimpleLauncher, + SingleNodeLauncher, + SrunLauncher, + SrunMPILauncher, + WrappedLauncher, +) __all__ = ['SimpleLauncher', 'WrappedLauncher', diff --git a/parsl/launchers/errors.py b/parsl/launchers/errors.py index 002df6220e..2882a85aa4 100644 --- a/parsl/launchers/errors.py +++ b/parsl/launchers/errors.py @@ -1,5 +1,5 @@ -from parsl.providers.errors import ExecutionProviderException from parsl.launchers.base import Launcher +from parsl.providers.errors import ExecutionProviderException class BadLauncher(ExecutionProviderException, TypeError): diff --git a/parsl/log_utils.py b/parsl/log_utils.py index 35044bf335..ff25249b1b 100644 --- a/parsl/log_utils.py +++ b/parsl/log_utils.py @@ -12,10 +12,9 @@ """ import io import logging -import typeguard - from typing import Optional +import typeguard DEFAULT_FORMAT = ( "%(created)f %(asctime)s %(processName)s-%(process)d " diff --git a/parsl/monitoring/db_manager.py b/parsl/monitoring/db_manager.py index ff794bf60d..9f19cd9f4d 100644 --- a/parsl/monitoring/db_manager.py +++ b/parsl/monitoring/db_manager.py @@ -1,15 +1,14 @@ +import datetime import logging -import threading -import queue import os +import queue +import threading import time -import datetime - from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar, cast -from parsl.log_utils import set_file_logger from parsl.dataflow.states import States from parsl.errors import OptionalModuleMissing +from parsl.log_utils import set_file_logger from parsl.monitoring.message_type import MessageType from parsl.monitoring.types import MonitoringMessage, TaggedMonitoringMessage from parsl.process_loggers import wrap_with_logs @@ -21,11 +20,18 @@ try: import sqlalchemy as sa - from sqlalchemy import Column, Text, Float, Boolean, BigInteger, Integer, DateTime, PrimaryKeyConstraint, Table - from sqlalchemy.orm import Mapper - from sqlalchemy.orm import mapperlib - from sqlalchemy.orm import sessionmaker - from sqlalchemy.orm import declarative_base + from sqlalchemy import ( + BigInteger, + Boolean, + Column, + DateTime, + Float, + Integer, + PrimaryKeyConstraint, + Table, + Text, + ) + from sqlalchemy.orm import Mapper, declarative_base, mapperlib, sessionmaker except ImportError: _sqlalchemy_enabled = False else: diff --git a/parsl/monitoring/monitoring.py b/parsl/monitoring/monitoring.py index 2c28a132a8..8e4770a32a 100644 --- a/parsl/monitoring/monitoring.py +++ b/parsl/monitoring/monitoring.py @@ -1,29 +1,25 @@ from __future__ import annotations -import os -import time import logging import multiprocessing.synchronize as ms -import typeguard - +import os import queue - -from parsl.multiprocessing import ForkProcess, SizedQueue -from multiprocessing import Process -from multiprocessing import Event +import time +from multiprocessing import Event, Process from multiprocessing.queues import Queue -from parsl.log_utils import set_file_logger -from parsl.utils import RepresentationMixin -from parsl.process_loggers import wrap_with_logs -from parsl.utils import setproctitle +from typing import TYPE_CHECKING, Any, Optional, Tuple, Union, cast -from parsl.serialize import deserialize +import typeguard +from parsl.log_utils import set_file_logger +from parsl.monitoring.message_type import MessageType from parsl.monitoring.radios import MultiprocessingQueueRadio from parsl.monitoring.router import router_starter -from parsl.monitoring.message_type import MessageType from parsl.monitoring.types import AddressedMonitoringMessage -from typing import cast, Any, Optional, Tuple, Union, TYPE_CHECKING +from parsl.multiprocessing import ForkProcess, SizedQueue +from parsl.process_loggers import wrap_with_logs +from parsl.serialize import deserialize +from parsl.utils import RepresentationMixin, setproctitle _db_manager_excepts: Optional[Exception] diff --git a/parsl/monitoring/queries/pandas.py b/parsl/monitoring/queries/pandas.py index 8de537f7c9..9bda8422e7 100644 --- a/parsl/monitoring/queries/pandas.py +++ b/parsl/monitoring/queries/pandas.py @@ -1,7 +1,6 @@ -import pandas as pd - from typing import Any +import pandas as pd # pandas can take several different types of database connection, # and itself exposes its connection parameters as "Any". diff --git a/parsl/monitoring/radios.py b/parsl/monitoring/radios.py index 33fab59e71..070869bdba 100644 --- a/parsl/monitoring/radios.py +++ b/parsl/monitoring/radios.py @@ -1,11 +1,9 @@ +import logging import os -import socket import pickle +import socket import uuid -import logging - from abc import ABCMeta, abstractmethod - from multiprocessing.queues import Queue from typing import Optional diff --git a/parsl/monitoring/remote.py b/parsl/monitoring/remote.py index 8d773b7157..98168aa858 100644 --- a/parsl/monitoring/remote.py +++ b/parsl/monitoring/remote.py @@ -1,16 +1,20 @@ +import datetime +import logging import os import time -import logging -import datetime from functools import wraps - -from parsl.multiprocessing import ForkProcess from multiprocessing import Event -from parsl.process_loggers import wrap_with_logs +from typing import Any, Callable, Dict, List, Sequence, Tuple from parsl.monitoring.message_type import MessageType -from parsl.monitoring.radios import MonitoringRadio, UDPRadio, HTEXRadio, FilesystemRadio -from typing import Any, Callable, Dict, List, Sequence, Tuple +from parsl.monitoring.radios import ( + FilesystemRadio, + HTEXRadio, + MonitoringRadio, + UDPRadio, +) +from parsl.multiprocessing import ForkProcess +from parsl.process_loggers import wrap_with_logs logger = logging.getLogger(__name__) @@ -135,8 +139,8 @@ def send_first_last_message(try_id: int, monitoring_hub_url: str, run_id: str, radio_mode: str, run_dir: str, is_last: bool) -> None: - import platform import os + import platform radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir) @@ -177,6 +181,7 @@ def monitor(pid: int, """ import logging import platform + import psutil from parsl.utils import setproctitle diff --git a/parsl/monitoring/router.py b/parsl/monitoring/router.py index d367ed6a1d..70b4862295 100644 --- a/parsl/monitoring/router.py +++ b/parsl/monitoring/router.py @@ -1,24 +1,21 @@ from __future__ import annotations +import logging import os +import pickle +import queue import socket import time -import pickle -import logging -import zmq +from multiprocessing.synchronize import Event +from typing import Optional, Tuple, Union -import queue +import zmq from parsl.log_utils import set_file_logger -from parsl.process_loggers import wrap_with_logs -from parsl.utils import setproctitle - from parsl.monitoring.message_type import MessageType from parsl.monitoring.types import AddressedMonitoringMessage, TaggedMonitoringMessage - -from multiprocessing.synchronize import Event -from typing import Optional, Tuple, Union - +from parsl.process_loggers import wrap_with_logs +from parsl.utils import setproctitle logger = logging.getLogger(__name__) diff --git a/parsl/monitoring/types.py b/parsl/monitoring/types.py index ab3483133c..c64dd22e7d 100644 --- a/parsl/monitoring/types.py +++ b/parsl/monitoring/types.py @@ -1,5 +1,7 @@ from typing import Any, Dict, Tuple, Union + from typing_extensions import TypeAlias + from parsl.monitoring.message_type import MessageType # A basic parsl monitoring message is wrapped by up to two wrappers: diff --git a/parsl/monitoring/visualization/app.py b/parsl/monitoring/visualization/app.py index 3bb844e144..e0c9510ee8 100644 --- a/parsl/monitoring/visualization/app.py +++ b/parsl/monitoring/visualization/app.py @@ -1,8 +1,10 @@ -from flask import Flask -from parsl.monitoring.visualization.models import db import argparse import os +from flask import Flask + +from parsl.monitoring.visualization.models import db + def cli_run(): """ Instantiates the Monitoring viz server diff --git a/parsl/monitoring/visualization/models.py b/parsl/monitoring/visualization/models.py index 9ac4a0d8b2..dd39c1d961 100644 --- a/parsl/monitoring/visualization/models.py +++ b/parsl/monitoring/visualization/models.py @@ -1,6 +1,5 @@ from flask_sqlalchemy import SQLAlchemy - WORKFLOW = 'workflow' # Workflow table includes workflow metadata TASK = 'task' # Task table includes task metadata STATUS = 'status' # Status table includes task status diff --git a/parsl/monitoring/visualization/plots/default/workflow_plots.py b/parsl/monitoring/visualization/plots/default/workflow_plots.py index 2c98d59081..7fc0620f90 100644 --- a/parsl/monitoring/visualization/plots/default/workflow_plots.py +++ b/parsl/monitoring/visualization/plots/default/workflow_plots.py @@ -1,13 +1,17 @@ import math + +import networkx as nx import numpy as np import pandas as pd -import plotly.graph_objs as go import plotly.figure_factory as ff +import plotly.graph_objs as go from plotly.offline import plot -import networkx as nx - -from parsl.monitoring.visualization.utils import timestamp_to_int, num_to_timestamp, DB_DATE_FORMAT +from parsl.monitoring.visualization.utils import ( + DB_DATE_FORMAT, + num_to_timestamp, + timestamp_to_int, +) # gantt_colors must assign a color value for every state name defined # in parsl/dataflow/states.py diff --git a/parsl/monitoring/visualization/plots/default/workflow_resource_plots.py b/parsl/monitoring/visualization/plots/default/workflow_resource_plots.py index 44ffe8ed98..c7cd776d81 100644 --- a/parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +++ b/parsl/monitoring/visualization/plots/default/workflow_resource_plots.py @@ -1,4 +1,5 @@ import math + import numpy as np import pandas as pd import plotly.graph_objs as go diff --git a/parsl/monitoring/visualization/utils.py b/parsl/monitoring/visualization/utils.py index e189e99fc0..a2438c5c2b 100644 --- a/parsl/monitoring/visualization/utils.py +++ b/parsl/monitoring/visualization/utils.py @@ -1,6 +1,5 @@ from datetime import datetime - DB_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/parsl/monitoring/visualization/views.py b/parsl/monitoring/visualization/views.py index 5489da2a03..8e34119143 100644 --- a/parsl/monitoring/visualization/views.py +++ b/parsl/monitoring/visualization/views.py @@ -1,15 +1,22 @@ -from flask import render_template -from flask import current_app as app import pandas as pd +from flask import current_app as app +from flask import render_template import parsl.monitoring.queries.pandas as queries - -from parsl.monitoring.visualization.models import Workflow, Task, Status, db - -from parsl.monitoring.visualization.plots.default.workflow_plots import task_gantt_plot, task_per_app_plot, workflow_dag_plot -from parsl.monitoring.visualization.plots.default.task_plots import time_series_memory_per_task_plot -from parsl.monitoring.visualization.plots.default.workflow_resource_plots import (resource_distribution_plot, - resource_efficiency, worker_efficiency) +from parsl.monitoring.visualization.models import Status, Task, Workflow, db +from parsl.monitoring.visualization.plots.default.task_plots import ( + time_series_memory_per_task_plot, +) +from parsl.monitoring.visualization.plots.default.workflow_plots import ( + task_gantt_plot, + task_per_app_plot, + workflow_dag_plot, +) +from parsl.monitoring.visualization.plots.default.workflow_resource_plots import ( + resource_distribution_plot, + resource_efficiency, + worker_efficiency, +) dummy = True diff --git a/parsl/multiprocessing.py b/parsl/multiprocessing.py index f09bc2211f..a468d840ec 100644 --- a/parsl/multiprocessing.py +++ b/parsl/multiprocessing.py @@ -6,7 +6,6 @@ import multiprocessing.queues import platform from multiprocessing.context import ForkProcess as ForkProcessType - from typing import Callable logger = logging.getLogger(__name__) diff --git a/parsl/process_loggers.py b/parsl/process_loggers.py index d224b4e38d..a7352de70f 100644 --- a/parsl/process_loggers.py +++ b/parsl/process_loggers.py @@ -1,7 +1,6 @@ +import functools import logging import threading -import functools - from typing import Callable, Optional diff --git a/parsl/providers/__init__.py b/parsl/providers/__init__.py index 4a3bb95efd..6a40b38ab6 100644 --- a/parsl/providers/__init__.py +++ b/parsl/providers/__init__.py @@ -1,24 +1,26 @@ # Workstation Provider -from parsl.providers.local.local import LocalProvider - -# Cluster Providers - -from parsl.providers.cobalt.cobalt import CobaltProvider -from parsl.providers.condor.condor import CondorProvider -from parsl.providers.grid_engine.grid_engine import GridEngineProvider -from parsl.providers.slurm.slurm import SlurmProvider -from parsl.providers.torque.torque import TorqueProvider -from parsl.providers.pbspro.pbspro import PBSProProvider -from parsl.providers.lsf.lsf import LSFProvider from parsl.providers.ad_hoc.ad_hoc import AdHocProvider # Cloud Providers from parsl.providers.aws.aws import AWSProvider -from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider from parsl.providers.azure.azure import AzureProvider +from parsl.providers.cobalt.cobalt import CobaltProvider +from parsl.providers.condor.condor import CondorProvider +from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider +from parsl.providers.grid_engine.grid_engine import GridEngineProvider # Kubernetes from parsl.providers.kubernetes.kube import KubernetesProvider +from parsl.providers.local.local import LocalProvider +from parsl.providers.lsf.lsf import LSFProvider +from parsl.providers.pbspro.pbspro import PBSProProvider +from parsl.providers.slurm.slurm import SlurmProvider +from parsl.providers.torque.torque import TorqueProvider + +# Cluster Providers + + + __all__ = ['LocalProvider', 'CobaltProvider', diff --git a/parsl/providers/ad_hoc/ad_hoc.py b/parsl/providers/ad_hoc/ad_hoc.py index 63784998ab..207dd55738 100644 --- a/parsl/providers/ad_hoc/ad_hoc.py +++ b/parsl/providers/ad_hoc/ad_hoc.py @@ -3,7 +3,7 @@ import time from parsl.channels import LocalChannel -from parsl.jobs.states import JobStatus, JobState +from parsl.jobs.states import JobState, JobStatus from parsl.launchers import SimpleLauncher from parsl.providers.base import ExecutionProvider from parsl.providers.errors import ScriptPathError diff --git a/parsl/providers/aws/aws.py b/parsl/providers/aws/aws.py index e3217dfd3e..d380e26872 100644 --- a/parsl/providers/aws/aws.py +++ b/parsl/providers/aws/aws.py @@ -4,13 +4,12 @@ import time from string import Template -from parsl.errors import ConfigurationError +from parsl.errors import ConfigurationError, OptionalModuleMissing from parsl.jobs.states import JobState, JobStatus +from parsl.launchers import SingleNodeLauncher from parsl.providers.aws.template import template_string from parsl.providers.base import ExecutionProvider -from parsl.errors import OptionalModuleMissing from parsl.utils import RepresentationMixin -from parsl.launchers import SingleNodeLauncher logger = logging.getLogger(__name__) diff --git a/parsl/providers/azure/azure.py b/parsl/providers/azure/azure.py index b33e9be237..bc7d2cad16 100644 --- a/parsl/providers/azure/azure.py +++ b/parsl/providers/azure/azure.py @@ -4,22 +4,21 @@ import time from string import Template -from parsl.errors import ConfigurationError +from parsl.errors import ConfigurationError, OptionalModuleMissing from parsl.jobs.states import JobState, JobStatus +from parsl.launchers import SingleNodeLauncher from parsl.providers.azure.template import template_string from parsl.providers.base import ExecutionProvider -from parsl.errors import OptionalModuleMissing from parsl.utils import RepresentationMixin -from parsl.launchers import SingleNodeLauncher logger = logging.getLogger(__name__) try: from azure.common.credentials import ServicePrincipalCredentials - from azure.mgmt.resource import ResourceManagementClient - from azure.mgmt.network import NetworkManagementClient from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.compute.models import DiskCreateOption + from azure.mgmt.network import NetworkManagementClient + from azure.mgmt.resource import ResourceManagementClient from msrestazure.azure_exceptions import CloudError _api_enabled = True diff --git a/parsl/providers/base.py b/parsl/providers/base.py index e4c7355445..5475f86d4c 100644 --- a/parsl/providers/base.py +++ b/parsl/providers/base.py @@ -1,5 +1,5 @@ -from abc import ABCMeta, abstractmethod, abstractproperty import logging +from abc import ABCMeta, abstractmethod, abstractproperty from typing import Any, Dict, List, Optional from parsl.channels.base import Channel diff --git a/parsl/providers/cluster_provider.py b/parsl/providers/cluster_provider.py index 20fbe62645..50f38a6136 100644 --- a/parsl/providers/cluster_provider.py +++ b/parsl/providers/cluster_provider.py @@ -2,10 +2,10 @@ from abc import abstractmethod from string import Template -from parsl.providers.errors import SchedulerMissingArgs, ScriptPathError from parsl.launchers.base import Launcher from parsl.launchers.errors import BadLauncher from parsl.providers.base import ExecutionProvider +from parsl.providers.errors import SchedulerMissingArgs, ScriptPathError logger = logging.getLogger(__name__) diff --git a/parsl/providers/cobalt/cobalt.py b/parsl/providers/cobalt/cobalt.py index 308ff98ced..4039dfcbea 100644 --- a/parsl/providers/cobalt/cobalt.py +++ b/parsl/providers/cobalt/cobalt.py @@ -3,12 +3,12 @@ import time import warnings -from parsl.providers.errors import ScaleOutFailed from parsl.channels import LocalChannel +from parsl.jobs.states import JobState, JobStatus from parsl.launchers import AprunLauncher -from parsl.providers.cobalt.template import template_string from parsl.providers.cluster_provider import ClusterProvider -from parsl.jobs.states import JobState, JobStatus +from parsl.providers.cobalt.template import template_string +from parsl.providers.errors import ScaleOutFailed from parsl.utils import RepresentationMixin, wtime_to_minutes logger = logging.getLogger(__name__) diff --git a/parsl/providers/condor/condor.py b/parsl/providers/condor/condor.py index af875db457..25266d3a2e 100644 --- a/parsl/providers/condor/condor.py +++ b/parsl/providers/condor/condor.py @@ -2,20 +2,22 @@ import os import re import time + import typeguard from parsl.channels import LocalChannel from parsl.jobs.states import JobState, JobStatus -from parsl.utils import RepresentationMixin from parsl.launchers import SingleNodeLauncher from parsl.launchers.base import Launcher -from parsl.providers.condor.template import template_string from parsl.providers.cluster_provider import ClusterProvider +from parsl.providers.condor.template import template_string from parsl.providers.errors import ScaleOutFailed +from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) from typing import Dict, List, Optional + from parsl.channels.base import Channel # See http://pages.cs.wisc.edu/~adesmet/status.html diff --git a/parsl/providers/errors.py b/parsl/providers/errors.py index 03573d3b55..c3aa7e5954 100644 --- a/parsl/providers/errors.py +++ b/parsl/providers/errors.py @@ -1,7 +1,7 @@ -from parsl.errors import ParslError - import warnings +from parsl.errors import ParslError + class ExecutionProviderException(ParslError): """ Base class for all exceptions diff --git a/parsl/providers/googlecloud/googlecloud.py b/parsl/providers/googlecloud/googlecloud.py index 74ebe44a82..93993285bd 100644 --- a/parsl/providers/googlecloud/googlecloud.py +++ b/parsl/providers/googlecloud/googlecloud.py @@ -1,8 +1,9 @@ import atexit import logging import os -from parsl.launchers import SingleNodeLauncher + from parsl.jobs.states import JobState, JobStatus +from parsl.launchers import SingleNodeLauncher logger = logging.getLogger(__name__) diff --git a/parsl/providers/grid_engine/grid_engine.py b/parsl/providers/grid_engine/grid_engine.py index 5277ef10ae..8e231637a9 100644 --- a/parsl/providers/grid_engine/grid_engine.py +++ b/parsl/providers/grid_engine/grid_engine.py @@ -3,10 +3,10 @@ import time from parsl.channels import LocalChannel +from parsl.jobs.states import JobState, JobStatus +from parsl.launchers import SingleNodeLauncher from parsl.providers.cluster_provider import ClusterProvider from parsl.providers.grid_engine.template import template_string -from parsl.launchers import SingleNodeLauncher -from parsl.jobs.states import JobState, JobStatus from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/providers/kubernetes/kube.py b/parsl/providers/kubernetes/kube.py index 0efccf88ec..0b53881702 100644 --- a/parsl/providers/kubernetes/kube.py +++ b/parsl/providers/kubernetes/kube.py @@ -1,17 +1,19 @@ import logging import time + from parsl.providers.kubernetes.template import template_string logger = logging.getLogger(__name__) +from typing import Any, Dict, List, Optional, Tuple + +import typeguard + from parsl.errors import OptionalModuleMissing from parsl.jobs.states import JobState, JobStatus from parsl.providers.base import ExecutionProvider from parsl.utils import RepresentationMixin -import typeguard -from typing import Any, Dict, List, Optional, Tuple - try: from kubernetes import client, config _kubernetes_enabled = True diff --git a/parsl/providers/local/local.py b/parsl/providers/local/local.py index 7312da317c..f13521466a 100644 --- a/parsl/providers/local/local.py +++ b/parsl/providers/local/local.py @@ -6,7 +6,11 @@ from parsl.jobs.states import JobState, JobStatus from parsl.launchers import SingleNodeLauncher from parsl.providers.base import ExecutionProvider -from parsl.providers.errors import SchedulerMissingArgs, ScriptPathError, SubmitException +from parsl.providers.errors import ( + SchedulerMissingArgs, + ScriptPathError, + SubmitException, +) from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/providers/lsf/lsf.py b/parsl/providers/lsf/lsf.py index 189a71c56d..c993b58c7b 100644 --- a/parsl/providers/lsf/lsf.py +++ b/parsl/providers/lsf/lsf.py @@ -1,7 +1,7 @@ -import os -import time import logging import math +import os +import time from parsl.channels import LocalChannel from parsl.jobs.states import JobState, JobStatus diff --git a/parsl/providers/pbspro/pbspro.py b/parsl/providers/pbspro/pbspro.py index ca0be9db96..c998602152 100644 --- a/parsl/providers/pbspro/pbspro.py +++ b/parsl/providers/pbspro/pbspro.py @@ -1,7 +1,7 @@ +import json import logging import os import time -import json from parsl.channels import LocalChannel from parsl.jobs.states import JobState, JobStatus diff --git a/parsl/providers/slurm/slurm.py b/parsl/providers/slurm/slurm.py index 2542b54d42..fead7143d5 100644 --- a/parsl/providers/slurm/slurm.py +++ b/parsl/providers/slurm/slurm.py @@ -1,12 +1,12 @@ -import os -import math -import time import logging +import math +import os import re -import typeguard - +import time from typing import Optional +import typeguard + from parsl.channels import LocalChannel from parsl.channels.base import Channel from parsl.jobs.states import JobState, JobStatus diff --git a/parsl/providers/torque/torque.py b/parsl/providers/torque/torque.py index bf170c8d40..dfe267dbd3 100644 --- a/parsl/providers/torque/torque.py +++ b/parsl/providers/torque/torque.py @@ -5,8 +5,8 @@ from parsl.channels import LocalChannel from parsl.jobs.states import JobState, JobStatus from parsl.launchers import AprunLauncher -from parsl.providers.torque.template import template_string from parsl.providers.cluster_provider import ClusterProvider +from parsl.providers.torque.template import template_string from parsl.utils import RepresentationMixin logger = logging.getLogger(__name__) diff --git a/parsl/serialize/__init__.py b/parsl/serialize/__init__.py index 90585bda90..dfbf7f79d9 100644 --- a/parsl/serialize/__init__.py +++ b/parsl/serialize/__init__.py @@ -1,6 +1,11 @@ -from parsl.serialize.facade import (serialize, deserialize, pack_apply_message, - unpack_apply_message, unpack_res_spec_apply_message, - pack_res_spec_apply_message) +from parsl.serialize.facade import ( + deserialize, + pack_apply_message, + pack_res_spec_apply_message, + serialize, + unpack_apply_message, + unpack_res_spec_apply_message, +) __all__ = ['serialize', 'deserialize', diff --git a/parsl/serialize/base.py b/parsl/serialize/base.py index 77060d4acf..188c68db14 100644 --- a/parsl/serialize/base.py +++ b/parsl/serialize/base.py @@ -1,7 +1,6 @@ +import logging from abc import abstractmethod from functools import cached_property -import logging - from typing import Any logger = logging.getLogger(__name__) diff --git a/parsl/serialize/concretes.py b/parsl/serialize/concretes.py index fb0363c046..40ad08be2a 100644 --- a/parsl/serialize/concretes.py +++ b/parsl/serialize/concretes.py @@ -1,13 +1,14 @@ -import dill import functools -import pickle import logging +import pickle -logger = logging.getLogger(__name__) -from parsl.serialize.base import SerializerBase +import dill +logger = logging.getLogger(__name__) from typing import Any +from parsl.serialize.base import SerializerBase + class PickleSerializer(SerializerBase): """ Pickle serialization covers most python objects, with some notable exceptions: diff --git a/parsl/serialize/proxystore.py b/parsl/serialize/proxystore.py index dcecfbc8d8..fb1d77e019 100644 --- a/parsl/serialize/proxystore.py +++ b/parsl/serialize/proxystore.py @@ -1,10 +1,11 @@ -import dill import io import typing as t -from parsl.serialize.base import SerializerBase +import dill from proxystore.store import Store +from parsl.serialize.base import SerializerBase + class ProxyStoreDeepPickler(dill.Pickler): """This class extends dill so that certain objects will be stored into diff --git a/parsl/tests/__init__.py b/parsl/tests/__init__.py index fc6c25a930..7018ff6aeb 100644 --- a/parsl/tests/__init__.py +++ b/parsl/tests/__init__.py @@ -1,5 +1,5 @@ -from parsl.dataflow.memoization import id_for_memo from parsl.data_provider.files import File +from parsl.dataflow.memoization import id_for_memo @id_for_memo.register(File) diff --git a/parsl/tests/configs/ad_hoc_cluster_htex.py b/parsl/tests/configs/ad_hoc_cluster_htex.py index 1f6dbcaaf0..0949b82392 100644 --- a/parsl/tests/configs/ad_hoc_cluster_htex.py +++ b/parsl/tests/configs/ad_hoc_cluster_htex.py @@ -1,9 +1,9 @@ -from parsl.providers import AdHocProvider +from typing import Any, Dict + from parsl.channels import SSHChannel -from parsl.executors import HighThroughputExecutor from parsl.config import Config - -from typing import Any, Dict +from parsl.executors import HighThroughputExecutor +from parsl.providers import AdHocProvider user_opts = {'adhoc': {'username': 'YOUR_USERNAME', diff --git a/parsl/tests/configs/azure_single_node.py b/parsl/tests/configs/azure_single_node.py index 17f3e7def9..32b6a5d348 100644 --- a/parsl/tests/configs/azure_single_node.py +++ b/parsl/tests/configs/azure_single_node.py @@ -1,13 +1,12 @@ """Config for Azure""" -from parsl.providers import AzureProvider +import getpass from parsl.config import Config -from parsl.executors import HighThroughputExecutor -from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.rsync import RSyncStaging - -import getpass +from parsl.executors import HighThroughputExecutor +from parsl.providers import AzureProvider # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py # If you are a user copying-and-pasting this as an example, make sure to either diff --git a/parsl/tests/configs/bridges.py b/parsl/tests/configs/bridges.py index 09fb507288..21feec1925 100644 --- a/parsl/tests/configs/bridges.py +++ b/parsl/tests/configs/bridges.py @@ -1,7 +1,8 @@ from parsl.config import Config -from parsl.providers import SlurmProvider -from parsl.launchers import SrunLauncher from parsl.executors import HighThroughputExecutor +from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider + from .user_opts import user_opts diff --git a/parsl/tests/configs/cc_in2p3.py b/parsl/tests/configs/cc_in2p3.py index 3131519d84..38c817cccd 100644 --- a/parsl/tests/configs/cc_in2p3.py +++ b/parsl/tests/configs/cc_in2p3.py @@ -1,7 +1,7 @@ -from parsl.config import Config from parsl.channels import LocalChannel -from parsl.providers import GridEngineProvider +from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import GridEngineProvider from .user_opts import user_opts diff --git a/parsl/tests/configs/comet.py b/parsl/tests/configs/comet.py index 033539da8a..3a946b5c8f 100644 --- a/parsl/tests/configs/comet.py +++ b/parsl/tests/configs/comet.py @@ -1,7 +1,8 @@ from parsl.config import Config +from parsl.executors import HighThroughputExecutor from parsl.launchers import SrunLauncher from parsl.providers import SlurmProvider -from parsl.executors import HighThroughputExecutor + from .user_opts import user_opts diff --git a/parsl/tests/configs/ec2_single_node.py b/parsl/tests/configs/ec2_single_node.py index 92c8108add..6222dbfabd 100644 --- a/parsl/tests/configs/ec2_single_node.py +++ b/parsl/tests/configs/ec2_single_node.py @@ -11,10 +11,9 @@ ================== """ -from parsl.providers import AWSProvider - from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import AWSProvider # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py # If you are a user copying-and-pasting this as an example, make sure to either diff --git a/parsl/tests/configs/ec2_spot.py b/parsl/tests/configs/ec2_spot.py index 37f272e4de..56084a9172 100644 --- a/parsl/tests/configs/ec2_spot.py +++ b/parsl/tests/configs/ec2_spot.py @@ -1,7 +1,6 @@ -from parsl.providers import AWSProvider - from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import AWSProvider # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py # If you are a user copying-and-pasting this as an example, make sure to either diff --git a/parsl/tests/configs/frontera.py b/parsl/tests/configs/frontera.py index d00e8ea737..537d6f78e0 100644 --- a/parsl/tests/configs/frontera.py +++ b/parsl/tests/configs/frontera.py @@ -1,10 +1,11 @@ -from parsl.config import Config from parsl.channels import LocalChannel -from parsl.providers import SlurmProvider +from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider from .user_opts import user_opts + """ This config assumes that it is used to launch parsl tasks from the login nodes of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes. """ diff --git a/parsl/tests/configs/htex_ad_hoc_cluster.py b/parsl/tests/configs/htex_ad_hoc_cluster.py index 82ae7bf621..db24b42ab2 100644 --- a/parsl/tests/configs/htex_ad_hoc_cluster.py +++ b/parsl/tests/configs/htex_ad_hoc_cluster.py @@ -1,9 +1,7 @@ -from parsl.providers import AdHocProvider from parsl.channels import SSHChannel -from parsl.executors import HighThroughputExecutor - from parsl.config import Config - +from parsl.executors import HighThroughputExecutor +from parsl.providers import AdHocProvider from parsl.tests.configs.user_opts import user_opts config = Config( diff --git a/parsl/tests/configs/htex_local.py b/parsl/tests/configs/htex_local.py index 0f09fce9dc..5553a7f70a 100644 --- a/parsl/tests/configs/htex_local.py +++ b/parsl/tests/configs/htex_local.py @@ -1,9 +1,8 @@ -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher - from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider def fresh_config(): diff --git a/parsl/tests/configs/htex_local_alternate.py b/parsl/tests/configs/htex_local_alternate.py index 3ae6df12ac..52124211bc 100644 --- a/parsl/tests/configs/htex_local_alternate.py +++ b/parsl/tests/configs/htex_local_alternate.py @@ -15,23 +15,20 @@ those timing parameters control. """ -# imports for monitoring: -from parsl.monitoring import MonitoringHub - import os -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SingleNodeLauncher - from parsl.config import Config -from parsl.executors import HighThroughputExecutor - - -from parsl.data_provider.http import HTTPInTaskStaging -from parsl.data_provider.ftp import FTPInTaskStaging from parsl.data_provider.file_noop import NoOpFileStaging +from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.zip import ZipFileStaging +from parsl.executors import HighThroughputExecutor +from parsl.launchers import SingleNodeLauncher + +# imports for monitoring: +from parsl.monitoring import MonitoringHub +from parsl.providers import LocalProvider working_dir = os.getcwd() + "/" + "test_htex_alternate" diff --git a/parsl/tests/configs/htex_local_intask_staging.py b/parsl/tests/configs/htex_local_intask_staging.py index 634cbf1654..eef88ea2e1 100644 --- a/parsl/tests/configs/htex_local_intask_staging.py +++ b/parsl/tests/configs/htex_local_intask_staging.py @@ -1,13 +1,11 @@ -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher - -from parsl.data_provider.http import HTTPInTaskStaging -from parsl.data_provider.ftp import FTPInTaskStaging -from parsl.data_provider.file_noop import NoOpFileStaging - from parsl.config import Config +from parsl.data_provider.file_noop import NoOpFileStaging +from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging from parsl.executors import HighThroughputExecutor +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider config = Config( executors=[ diff --git a/parsl/tests/configs/htex_local_rsync_staging.py b/parsl/tests/configs/htex_local_rsync_staging.py index 6cb47e55e9..044d2b9f89 100644 --- a/parsl/tests/configs/htex_local_rsync_staging.py +++ b/parsl/tests/configs/htex_local_rsync_staging.py @@ -1,13 +1,11 @@ -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher - -from parsl.data_provider.http import HTTPInTaskStaging +from parsl.config import Config from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.rsync import RSyncStaging - -from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider config = Config( executors=[ diff --git a/parsl/tests/configs/local_adhoc.py b/parsl/tests/configs/local_adhoc.py index 96c08ee309..25b1f38d61 100644 --- a/parsl/tests/configs/local_adhoc.py +++ b/parsl/tests/configs/local_adhoc.py @@ -1,5 +1,5 @@ -from parsl.config import Config from parsl.channels import LocalChannel +from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.providers import AdHocProvider diff --git a/parsl/tests/configs/local_radical.py b/parsl/tests/configs/local_radical.py index c52537bbf6..ec146f5050 100644 --- a/parsl/tests/configs/local_radical.py +++ b/parsl/tests/configs/local_radical.py @@ -1,9 +1,7 @@ import os from parsl.config import Config -from parsl.executors.radical import RadicalPilotExecutor -from parsl.executors.radical import ResourceConfig - +from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig rpex_cfg = ResourceConfig() diff --git a/parsl/tests/configs/local_radical_mpi.py b/parsl/tests/configs/local_radical_mpi.py index 3801dfd680..582f406e45 100644 --- a/parsl/tests/configs/local_radical_mpi.py +++ b/parsl/tests/configs/local_radical_mpi.py @@ -1,10 +1,10 @@ import os + from parsl.config import Config def fresh_config(): - from parsl.executors.radical import ResourceConfig - from parsl.executors.radical import RadicalPilotExecutor + from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig rpex_cfg = ResourceConfig() rpex_cfg.worker_type = "MPI" diff --git a/parsl/tests/configs/midway.py b/parsl/tests/configs/midway.py index 410e680c01..01cc49fd28 100644 --- a/parsl/tests/configs/midway.py +++ b/parsl/tests/configs/midway.py @@ -1,7 +1,7 @@ from parsl.config import Config -from parsl.providers import SlurmProvider -from parsl.launchers import SrunLauncher from parsl.executors import HighThroughputExecutor +from parsl.launchers import SrunLauncher +from parsl.providers import SlurmProvider from .user_opts import user_opts diff --git a/parsl/tests/configs/nscc_singapore.py b/parsl/tests/configs/nscc_singapore.py index ae5cc6d6d4..8260fbb601 100644 --- a/parsl/tests/configs/nscc_singapore.py +++ b/parsl/tests/configs/nscc_singapore.py @@ -1,8 +1,8 @@ -from parsl.providers import PBSProProvider -from parsl.executors import HighThroughputExecutor -from parsl.launchers import MpiRunLauncher from parsl.addresses import address_by_interface from parsl.config import Config +from parsl.executors import HighThroughputExecutor +from parsl.launchers import MpiRunLauncher +from parsl.providers import PBSProProvider from .user_opts import user_opts diff --git a/parsl/tests/configs/osg_htex.py b/parsl/tests/configs/osg_htex.py index 6e05cb17e1..3e0658b30e 100644 --- a/parsl/tests/configs/osg_htex.py +++ b/parsl/tests/configs/osg_htex.py @@ -1,6 +1,6 @@ from parsl.config import Config -from parsl.providers import CondorProvider from parsl.executors import HighThroughputExecutor +from parsl.providers import CondorProvider # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py # If you are a user copying-and-pasting this as an example, make sure to either diff --git a/parsl/tests/configs/petrelkube.py b/parsl/tests/configs/petrelkube.py index 451f374f20..7fe35854ca 100644 --- a/parsl/tests/configs/petrelkube.py +++ b/parsl/tests/configs/petrelkube.py @@ -1,10 +1,11 @@ +import os + +from parsl.addresses import address_by_route from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.providers import KubernetesProvider -from parsl.addresses import address_by_route from .user_opts import user_opts -import os def fresh_config(): diff --git a/parsl/tests/configs/summit.py b/parsl/tests/configs/summit.py index 60e668f41e..69487f780e 100644 --- a/parsl/tests/configs/summit.py +++ b/parsl/tests/configs/summit.py @@ -4,6 +4,7 @@ from parsl.providers import LSFProvider from .user_opts import user_opts + """ This config assumes that it is used to launch parsl tasks from the login nodes of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes. """ diff --git a/parsl/tests/configs/swan_htex.py b/parsl/tests/configs/swan_htex.py index 4884703a2a..3b1b6785ab 100644 --- a/parsl/tests/configs/swan_htex.py +++ b/parsl/tests/configs/swan_htex.py @@ -8,10 +8,10 @@ ================== """ from parsl.channels import SSHChannel -from parsl.launchers import AprunLauncher -from parsl.providers import TorqueProvider from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.launchers import AprunLauncher +from parsl.providers import TorqueProvider # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py # If you are a user copying-and-pasting this as an example, make sure to either diff --git a/parsl/tests/configs/taskvine_ex.py b/parsl/tests/configs/taskvine_ex.py index da5cb7bbe8..276e37cf4e 100644 --- a/parsl/tests/configs/taskvine_ex.py +++ b/parsl/tests/configs/taskvine_ex.py @@ -1,10 +1,8 @@ from parsl.config import Config -from parsl.executors.taskvine import TaskVineExecutor -from parsl.executors.taskvine import TaskVineManagerConfig - -from parsl.data_provider.http import HTTPInTaskStaging -from parsl.data_provider.ftp import FTPInTaskStaging from parsl.data_provider.file_noop import NoOpFileStaging +from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging +from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig def fresh_config(): diff --git a/parsl/tests/configs/theta.py b/parsl/tests/configs/theta.py index c6e8d9192d..71ed25142b 100644 --- a/parsl/tests/configs/theta.py +++ b/parsl/tests/configs/theta.py @@ -1,7 +1,7 @@ from parsl.config import Config -from parsl.providers import CobaltProvider -from parsl.launchers import AprunLauncher from parsl.executors import HighThroughputExecutor +from parsl.launchers import AprunLauncher +from parsl.providers import CobaltProvider from .user_opts import user_opts diff --git a/parsl/tests/configs/workqueue_ex.py b/parsl/tests/configs/workqueue_ex.py index 08c3ba71b7..7e9a696777 100644 --- a/parsl/tests/configs/workqueue_ex.py +++ b/parsl/tests/configs/workqueue_ex.py @@ -1,9 +1,8 @@ from parsl.config import Config -from parsl.executors import WorkQueueExecutor - -from parsl.data_provider.http import HTTPInTaskStaging -from parsl.data_provider.ftp import FTPInTaskStaging from parsl.data_provider.file_noop import NoOpFileStaging +from parsl.data_provider.ftp import FTPInTaskStaging +from parsl.data_provider.http import HTTPInTaskStaging +from parsl.executors import WorkQueueExecutor def fresh_config(): diff --git a/parsl/tests/conftest.py b/parsl/tests/conftest.py index 3e21712582..80b9e000cd 100644 --- a/parsl/tests/conftest.py +++ b/parsl/tests/conftest.py @@ -6,21 +6,21 @@ import random import re import shutil -import string -import time -import types import signal +import string import sys import tempfile import threading +import time import traceback +import types import typing as t from datetime import datetime from glob import glob from itertools import chain -import pytest import _pytest.runner as runner +import pytest import parsl from parsl.dataflow.dflow import DataFlowKernelLoader diff --git a/parsl/tests/integration/test_channels/test_ssh_errors.py b/parsl/tests/integration/test_channels/test_ssh_errors.py index 7da7cc34ef..7483e30a5c 100644 --- a/parsl/tests/integration/test_channels/test_ssh_errors.py +++ b/parsl/tests/integration/test_channels/test_ssh_errors.py @@ -1,4 +1,4 @@ -from parsl.channels.errors import SSHException, BadHostKeyException +from parsl.channels.errors import BadHostKeyException, SSHException from parsl.channels.ssh.ssh import SSHChannel as SSH diff --git a/parsl/tests/integration/test_stress/test_python_simple.py b/parsl/tests/integration/test_stress/test_python_simple.py index 3666ecd3ce..10cc4b6092 100644 --- a/parsl/tests/integration/test_stress/test_python_simple.py +++ b/parsl/tests/integration/test_stress/test_python_simple.py @@ -1,10 +1,12 @@ +import argparse +import time + import parsl from parsl import python_app from parsl.configs.htex_local import config + # from parsl.configs.local_threads import config -import time -import argparse @python_app diff --git a/parsl/tests/integration/test_stress/test_python_threads.py b/parsl/tests/integration/test_stress/test_python_threads.py index 11309851ba..30f6ff9b5d 100644 --- a/parsl/tests/integration/test_stress/test_python_threads.py +++ b/parsl/tests/integration/test_stress/test_python_threads.py @@ -1,14 +1,12 @@ ''' Testing bash apps ''' -import parsl -from parsl import python_app - -import time import argparse +import time +import parsl +from parsl import python_app from parsl.tests.configs.local_threads import config - local_config = config diff --git a/parsl/tests/manual_tests/htex_local.py b/parsl/tests/manual_tests/htex_local.py index a5c0449be1..bcfdbf34ec 100644 --- a/parsl/tests/manual_tests/htex_local.py +++ b/parsl/tests/manual_tests/htex_local.py @@ -1,11 +1,11 @@ -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -# from parsl.launchers import SimpleLauncher -from parsl.launchers import SingleNodeLauncher - from parsl.config import Config from parsl.executors import HighThroughputExecutor +# from parsl.launchers import SimpleLauncher +from parsl.launchers import SingleNodeLauncher +from parsl.providers import LocalProvider + config = Config( executors=[ HighThroughputExecutor( diff --git a/parsl/tests/manual_tests/test_ad_hoc_htex.py b/parsl/tests/manual_tests/test_ad_hoc_htex.py index f5525b8f32..dfa34ec0d1 100644 --- a/parsl/tests/manual_tests/test_ad_hoc_htex.py +++ b/parsl/tests/manual_tests/test_ad_hoc_htex.py @@ -1,11 +1,12 @@ import parsl from parsl import python_app + parsl.set_stream_logger() -from parsl.providers import AdHocProvider from parsl.channels import SSHChannel from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import AdHocProvider remotes = ['midway2-login2.rcc.uchicago.edu', 'midway2-login1.rcc.uchicago.edu'] diff --git a/parsl/tests/manual_tests/test_basic.py b/parsl/tests/manual_tests/test_basic.py index 9200936eb2..2a9fe93e4e 100644 --- a/parsl/tests/manual_tests/test_basic.py +++ b/parsl/tests/manual_tests/test_basic.py @@ -2,6 +2,7 @@ import time import parsl + # Tested. Confirmed. Local X Local X SingleNodeLauncher # from parsl.tests.configs.local_ipp import config diff --git a/parsl/tests/manual_tests/test_fan_in_out_htex_remote.py b/parsl/tests/manual_tests/test_fan_in_out_htex_remote.py index ce09e4bcad..b780fd47b4 100644 --- a/parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +++ b/parsl/tests/manual_tests/test_fan_in_out_htex_remote.py @@ -1,13 +1,13 @@ +import logging + import parsl -from parsl.monitoring.monitoring import MonitoringHub +from parsl.app.app import python_app from parsl.config import Config from parsl.executors import HighThroughputExecutor from parsl.launchers import AprunLauncher +from parsl.monitoring.monitoring import MonitoringHub from parsl.providers import CobaltProvider -import logging -from parsl.app.app import python_app - def local_setup(): threads_config = Config( diff --git a/parsl/tests/manual_tests/test_log_filter.py b/parsl/tests/manual_tests/test_log_filter.py index 4a1d738690..ff620633b4 100644 --- a/parsl/tests/manual_tests/test_log_filter.py +++ b/parsl/tests/manual_tests/test_log_filter.py @@ -1,6 +1,8 @@ import argparse -import parsl import logging + +import parsl + parsl.load() from parsl import python_app diff --git a/parsl/tests/manual_tests/test_memory_limits.py b/parsl/tests/manual_tests/test_memory_limits.py index be4507808f..b353ba0929 100644 --- a/parsl/tests/manual_tests/test_memory_limits.py +++ b/parsl/tests/manual_tests/test_memory_limits.py @@ -1,15 +1,15 @@ import argparse -import parsl -import psutil import multiprocessing -from parsl.providers import LocalProvider +import psutil + +import parsl +from parsl.app.app import python_app # , bash_app from parsl.channels import LocalChannel -from parsl.launchers import SingleNodeLauncher from parsl.config import Config from parsl.executors import HighThroughputExecutor - -from parsl.app.app import python_app # , bash_app +from parsl.launchers import SingleNodeLauncher +from parsl.providers import LocalProvider @python_app diff --git a/parsl/tests/manual_tests/test_regression_220.py b/parsl/tests/manual_tests/test_regression_220.py index 3b7a629c85..7221c62499 100644 --- a/parsl/tests/manual_tests/test_regression_220.py +++ b/parsl/tests/manual_tests/test_regression_220.py @@ -1,7 +1,8 @@ +import time + import pytest from parsl import DataFlowKernel, set_stream_logger -import time from parsl.tests.configs.local_threads import config diff --git a/parsl/tests/manual_tests/test_udp_simple.py b/parsl/tests/manual_tests/test_udp_simple.py index 81138d0ed4..8de257d8fa 100644 --- a/parsl/tests/manual_tests/test_udp_simple.py +++ b/parsl/tests/manual_tests/test_udp_simple.py @@ -1,9 +1,10 @@ +import logging + +import parsl from parsl import python_app -from parsl.monitoring.monitoring import MonitoringHub from parsl.config import Config from parsl.executors import ThreadPoolExecutor -import parsl -import logging +from parsl.monitoring.monitoring import MonitoringHub def local_setup(): diff --git a/parsl/tests/manual_tests/test_worker_count.py b/parsl/tests/manual_tests/test_worker_count.py index cf3a89f262..4f9d2d0d0b 100644 --- a/parsl/tests/manual_tests/test_worker_count.py +++ b/parsl/tests/manual_tests/test_worker_count.py @@ -1,7 +1,7 @@ import argparse -import time import math import multiprocessing +import time import parsl @@ -10,10 +10,11 @@ EXPECTED_WORKERS = math.floor(CORES / CORES_PER_WORKER) +from parsl.executors import HighThroughputExecutor + # from parsl.tests.configs.htex_local import config from parsl.tests.manual_tests.htex_local import config -from parsl.executors import HighThroughputExecutor assert isinstance(config.executors[0], HighThroughputExecutor) config.executors[0].cores_per_worker = CORES_PER_WORKER config.executors[0].provider.init_blocks = 1 diff --git a/parsl/tests/scaling_tests/htex_local.py b/parsl/tests/scaling_tests/htex_local.py index 9f61cc7617..f2bcf86963 100644 --- a/parsl/tests/scaling_tests/htex_local.py +++ b/parsl/tests/scaling_tests/htex_local.py @@ -1,8 +1,8 @@ -from parsl.providers import LocalProvider from parsl.channels import LocalChannel - from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.providers import LocalProvider + # import os config = Config( executors=[ diff --git a/parsl/tests/scaling_tests/test_scale.py b/parsl/tests/scaling_tests/test_scale.py index f024051ee0..4331d0db53 100755 --- a/parsl/tests/scaling_tests/test_scale.py +++ b/parsl/tests/scaling_tests/test_scale.py @@ -5,15 +5,16 @@ import parsl +# parsl.set_stream_logger() +# config.executors[0].provider.tasks_per_node = 4 +# parsl.load(config) +from parsl.app.app import python_app # , bash_app + # from parsl.tests.configs.htex_local import config # from htex_local import config # from parsl.configs.local_threads import config # from parsl.configs.local_ipp import config -# parsl.set_stream_logger() -# config.executors[0].provider.tasks_per_node = 4 -# parsl.load(config) -from parsl.app.app import python_app # , bash_app @python_app diff --git a/parsl/tests/scaling_tests/vineex_condor.py b/parsl/tests/scaling_tests/vineex_condor.py index 59f8e7b07d..956121ca16 100644 --- a/parsl/tests/scaling_tests/vineex_condor.py +++ b/parsl/tests/scaling_tests/vineex_condor.py @@ -1,6 +1,5 @@ from parsl.config import Config -from parsl.executors.taskvine import TaskVineExecutor -from parsl.executors.taskvine import TaskVineManagerConfig +from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig from parsl.providers import CondorProvider config = Config( diff --git a/parsl/tests/scaling_tests/vineex_local.py b/parsl/tests/scaling_tests/vineex_local.py index 4ade645f31..53edbfe2d5 100644 --- a/parsl/tests/scaling_tests/vineex_local.py +++ b/parsl/tests/scaling_tests/vineex_local.py @@ -1,6 +1,5 @@ from parsl.config import Config -from parsl.executors.taskvine import TaskVineExecutor -from parsl.executors.taskvine import TaskVineManagerConfig +from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig from parsl.providers import LocalProvider config = Config( diff --git a/parsl/tests/site_tests/test_provider.py b/parsl/tests/site_tests/test_provider.py index eef0420583..43a8c592d6 100644 --- a/parsl/tests/site_tests/test_provider.py +++ b/parsl/tests/site_tests/test_provider.py @@ -1,8 +1,10 @@ import argparse import logging +import time + import pytest + import parsl -import time from parsl.app.app import python_app # , bash_app from parsl.jobs.states import JobState from parsl.tests.site_tests.site_config_selector import fresh_config diff --git a/parsl/tests/site_tests/test_site.py b/parsl/tests/site_tests/test_site.py index 991a03fe46..6ce3bc2e43 100644 --- a/parsl/tests/site_tests/test_site.py +++ b/parsl/tests/site_tests/test_site.py @@ -1,5 +1,7 @@ import argparse + import pytest + import parsl from parsl.app.app import python_app # , bash_app from parsl.tests.site_tests.site_config_selector import fresh_config diff --git a/parsl/tests/sites/test_affinity.py b/parsl/tests/sites/test_affinity.py index 41f2f3a876..50d08ce830 100644 --- a/parsl/tests/sites/test_affinity.py +++ b/parsl/tests/sites/test_affinity.py @@ -1,12 +1,14 @@ """Tests related to assigning workers to specific compute units""" -from parsl.providers import LocalProvider +import os + +import pytest + +from parsl import python_app from parsl.channels import LocalChannel from parsl.config import Config from parsl.executors import HighThroughputExecutor -from parsl import python_app -import pytest -import os +from parsl.providers import LocalProvider def local_config(): @@ -32,8 +34,8 @@ def local_config(): @python_app def get_worker_info(): - from time import sleep import os + from time import sleep rank = int(os.environ['PARSL_WORKER_RANK']) aff = os.sched_getaffinity(0) device = os.environ.get('CUDA_VISIBLE_DEVICES') diff --git a/parsl/tests/sites/test_dynamic_executor.py b/parsl/tests/sites/test_dynamic_executor.py index f81f76a645..26a6eb9375 100644 --- a/parsl/tests/sites/test_dynamic_executor.py +++ b/parsl/tests/sites/test_dynamic_executor.py @@ -1,10 +1,10 @@ +import pytest + import parsl +from parsl.app.app import python_app from parsl.executors import HighThroughputExecutor from parsl.executors.threads import ThreadPoolExecutor from parsl.providers import LocalProvider -from parsl.app.app import python_app - -import pytest @python_app(executors=['threads']) diff --git a/parsl/tests/sites/test_ec2.py b/parsl/tests/sites/test_ec2.py index 6ab13b1ef7..7cb089a0c7 100644 --- a/parsl/tests/sites/test_ec2.py +++ b/parsl/tests/sites/test_ec2.py @@ -1,10 +1,11 @@ -import parsl +import logging + import pytest +import parsl from parsl.app.app import python_app from parsl.tests.configs.ec2_single_node import config -import logging logger = logging.getLogger(__name__) local_config = config diff --git a/parsl/tests/sites/test_local_adhoc.py b/parsl/tests/sites/test_local_adhoc.py index 4af935a4a3..bdb922773c 100644 --- a/parsl/tests/sites/test_local_adhoc.py +++ b/parsl/tests/sites/test_local_adhoc.py @@ -1,9 +1,10 @@ +import logging + import pytest from parsl import python_app from parsl.tests.configs.local_adhoc import fresh_config as local_config -import logging logger = logging.getLogger(__name__) diff --git a/parsl/tests/sites/test_worker_info.py b/parsl/tests/sites/test_worker_info.py index 01666d92f9..4d9865da84 100644 --- a/parsl/tests/sites/test_worker_info.py +++ b/parsl/tests/sites/test_worker_info.py @@ -1,11 +1,12 @@ """Tests related to Parsl workers being able to access their worker ID""" -from parsl.providers import LocalProvider +import pytest + +from parsl import python_app from parsl.channels import LocalChannel from parsl.config import Config from parsl.executors import HighThroughputExecutor -from parsl import python_app -import pytest +from parsl.providers import LocalProvider def local_config(): diff --git a/parsl/tests/test_aalst_patterns.py b/parsl/tests/test_aalst_patterns.py index 3e6311e350..e410a7243a 100644 --- a/parsl/tests/test_aalst_patterns.py +++ b/parsl/tests/test_aalst_patterns.py @@ -6,7 +6,6 @@ from parsl.app.app import python_app from parsl.tests.configs.local_threads import config - pytestmark = pytest.mark.skip('not asserting anything') diff --git a/parsl/tests/test_bash_apps/test_apptimeout.py b/parsl/tests/test_bash_apps/test_apptimeout.py index 331082d39b..1c8eebaf5e 100644 --- a/parsl/tests/test_bash_apps/test_apptimeout.py +++ b/parsl/tests/test_bash_apps/test_apptimeout.py @@ -1,9 +1,9 @@ -import parsl import pytest +import parsl from parsl.app.app import bash_app -from parsl.tests.configs.local_threads import config from parsl.app.errors import AppTimeout +from parsl.tests.configs.local_threads import config @bash_app diff --git a/parsl/tests/test_bash_apps/test_error_codes.py b/parsl/tests/test_bash_apps/test_error_codes.py index ebabb4437d..4a0b835728 100644 --- a/parsl/tests/test_bash_apps/test_error_codes.py +++ b/parsl/tests/test_bash_apps/test_error_codes.py @@ -4,12 +4,9 @@ import pytest import parsl -from parsl.app.app import bash_app import parsl.app.errors as pe - - +from parsl.app.app import bash_app from parsl.app.errors import BashExitFailure - from parsl.tests.configs.local_threads import fresh_config as local_config diff --git a/parsl/tests/test_bash_apps/test_memoize_ignore_args.py b/parsl/tests/test_bash_apps/test_memoize_ignore_args.py index ba00524b19..0439bfb163 100644 --- a/parsl/tests/test_bash_apps/test_memoize_ignore_args.py +++ b/parsl/tests/test_bash_apps/test_memoize_ignore_args.py @@ -1,4 +1,5 @@ import os + import pytest import parsl diff --git a/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py b/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py index 9c57383c4a..3c9b51e980 100644 --- a/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +++ b/parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py @@ -1,9 +1,9 @@ import copy import os -import pytest - from typing import List +import pytest + import parsl from parsl.app.app import bash_app diff --git a/parsl/tests/test_bash_apps/test_pipeline.py b/parsl/tests/test_bash_apps/test_pipeline.py index 6cefe77223..75f16237d8 100644 --- a/parsl/tests/test_bash_apps/test_pipeline.py +++ b/parsl/tests/test_bash_apps/test_pipeline.py @@ -1,8 +1,8 @@ import pytest from parsl.app.app import bash_app -from parsl.data_provider.files import File from parsl.app.futures import DataFuture +from parsl.data_provider.files import File @bash_app diff --git a/parsl/tests/test_bash_apps/test_std_uri.py b/parsl/tests/test_bash_apps/test_std_uri.py index 7b24b3aecb..3da1fd0d88 100644 --- a/parsl/tests/test_bash_apps/test_std_uri.py +++ b/parsl/tests/test_bash_apps/test_std_uri.py @@ -1,9 +1,10 @@ import logging -import parsl -import pytest import zipfile - from functools import partial + +import pytest + +import parsl from parsl.app.futures import DataFuture from parsl.data_provider.files import File from parsl.executors import ThreadPoolExecutor diff --git a/parsl/tests/test_callables.py b/parsl/tests/test_callables.py index 206693fe2b..14b533d98b 100644 --- a/parsl/tests/test_callables.py +++ b/parsl/tests/test_callables.py @@ -5,10 +5,10 @@ import importlib import pathlib -import parsl - from functools import partial +import parsl + @parsl.python_app def app(x): diff --git a/parsl/tests/test_checkpointing/test_python_checkpoint_1.py b/parsl/tests/test_checkpointing/test_python_checkpoint_1.py index 3c8693969e..7539af7f24 100644 --- a/parsl/tests/test_checkpointing/test_python_checkpoint_1.py +++ b/parsl/tests/test_checkpointing/test_python_checkpoint_1.py @@ -1,4 +1,5 @@ import os + import pytest import parsl diff --git a/parsl/tests/test_checkpointing/test_python_checkpoint_2.py b/parsl/tests/test_checkpointing/test_python_checkpoint_2.py index 328eab5ca2..f219b704e4 100644 --- a/parsl/tests/test_checkpointing/test_python_checkpoint_2.py +++ b/parsl/tests/test_checkpointing/test_python_checkpoint_2.py @@ -1,9 +1,10 @@ import contextlib import os + import pytest + import parsl from parsl import python_app - from parsl.tests.configs.local_threads_checkpoint import fresh_config diff --git a/parsl/tests/test_checkpointing/test_regression_239.py b/parsl/tests/test_checkpointing/test_regression_239.py index 240267dbf7..749bd3842e 100644 --- a/parsl/tests/test_checkpointing/test_regression_239.py +++ b/parsl/tests/test_checkpointing/test_regression_239.py @@ -2,7 +2,7 @@ import pytest -from parsl import python_app, DataFlowKernel +from parsl import DataFlowKernel, python_app from parsl.utils import time_limited_open diff --git a/parsl/tests/test_checkpointing/test_task_exit.py b/parsl/tests/test_checkpointing/test_task_exit.py index 31bad799ca..cc1ffd8445 100644 --- a/parsl/tests/test_checkpointing/test_task_exit.py +++ b/parsl/tests/test_checkpointing/test_task_exit.py @@ -5,8 +5,8 @@ import parsl from parsl.app.app import python_app -from parsl.utils import time_limited_open from parsl.tests.configs.local_threads_checkpoint_task_exit import config +from parsl.utils import time_limited_open def local_setup(): diff --git a/parsl/tests/test_docs/test_from_slides.py b/parsl/tests/test_docs/test_from_slides.py index 09f5e86f93..b07092b4ae 100644 --- a/parsl/tests/test_docs/test_from_slides.py +++ b/parsl/tests/test_docs/test_from_slides.py @@ -1,8 +1,8 @@ +import os + from parsl.app.app import bash_app, python_app from parsl.data_provider.files import File -import os - @bash_app def echo(message, outputs=[]): diff --git a/parsl/tests/test_docs/test_kwargs.py b/parsl/tests/test_docs/test_kwargs.py index 922fd39e94..5524c0b819 100644 --- a/parsl/tests/test_docs/test_kwargs.py +++ b/parsl/tests/test_docs/test_kwargs.py @@ -1,7 +1,7 @@ """Functions used to explain kwargs""" from pathlib import Path -from parsl import python_app, File +from parsl import File, python_app def test_inputs(): diff --git a/parsl/tests/test_docs/test_tutorial_1.py b/parsl/tests/test_docs/test_tutorial_1.py index 5f8906d4ca..cffc03e33d 100644 --- a/parsl/tests/test_docs/test_tutorial_1.py +++ b/parsl/tests/test_docs/test_tutorial_1.py @@ -1,12 +1,11 @@ import argparse -import parsl import pytest +import parsl from parsl.app.app import bash_app from parsl.tests.configs.local_threads import config - local_config = config diff --git a/parsl/tests/test_docs/test_workflow1.py b/parsl/tests/test_docs/test_workflow1.py index 2678794a3c..271baab4d8 100644 --- a/parsl/tests/test_docs/test_workflow1.py +++ b/parsl/tests/test_docs/test_workflow1.py @@ -1,12 +1,12 @@ import os + import pytest -import parsl +import parsl from parsl.app.app import bash_app, python_app from parsl.data_provider.files import File from parsl.tests.configs.local_threads import config - # parsl.set_stream_logger() diff --git a/parsl/tests/test_docs/test_workflow2.py b/parsl/tests/test_docs/test_workflow2.py index 7878a7fa7f..1c6db0cff5 100644 --- a/parsl/tests/test_docs/test_workflow2.py +++ b/parsl/tests/test_docs/test_workflow2.py @@ -5,7 +5,6 @@ from parsl.app.app import python_app from parsl.tests.configs.local_threads import config - local_config = config diff --git a/parsl/tests/test_error_handling/test_rand_fail.py b/parsl/tests/test_error_handling/test_rand_fail.py index 50d58a500d..4046b8ffb6 100644 --- a/parsl/tests/test_error_handling/test_rand_fail.py +++ b/parsl/tests/test_error_handling/test_rand_fail.py @@ -15,8 +15,8 @@ def local_config(): @python_app def sleep_fail(sleep_dur, sleep_rand_max, fail_prob, inputs=[]): - import time import random + import time s = sleep_dur + random.randint(-sleep_rand_max, sleep_rand_max) @@ -144,8 +144,8 @@ def test_deps(numtasks=10): @python_app def sleep_then_fail(sleep_dur=0.1): - import time import math + import time time.sleep(sleep_dur) math.ceil("Trigger TypeError") return 0 diff --git a/parsl/tests/test_error_handling/test_resource_spec.py b/parsl/tests/test_error_handling/test_resource_spec.py index 336080afae..871df68512 100644 --- a/parsl/tests/test_error_handling/test_resource_spec.py +++ b/parsl/tests/test_error_handling/test_resource_spec.py @@ -1,9 +1,11 @@ import parsl from parsl.app.app import python_app -from parsl.executors.errors import UnsupportedFeatureError, ExecutorError from parsl.executors import WorkQueueExecutor -from parsl.executors.high_throughput.mpi_prefix_composer import InvalidResourceSpecification +from parsl.executors.errors import ExecutorError, UnsupportedFeatureError from parsl.executors.high_throughput.executor import HighThroughputExecutor +from parsl.executors.high_throughput.mpi_prefix_composer import ( + InvalidResourceSpecification, +) @python_app diff --git a/parsl/tests/test_error_handling/test_retries.py b/parsl/tests/test_error_handling/test_retries.py index c99edee2f6..c069ee7ba7 100644 --- a/parsl/tests/test_error_handling/test_retries.py +++ b/parsl/tests/test_error_handling/test_retries.py @@ -1,5 +1,6 @@ import argparse import os + import pytest import parsl @@ -15,8 +16,8 @@ def local_config(): @python_app def sleep_then_fail(inputs=[], sleep_dur=0.1): - import time import math + import time time.sleep(sleep_dur) math.ceil("Trigger TypeError") return 0 diff --git a/parsl/tests/test_error_handling/test_retry_handler.py b/parsl/tests/test_error_handling/test_retry_handler.py index 4eceddc9c9..788c6063e5 100644 --- a/parsl/tests/test_error_handling/test_retry_handler.py +++ b/parsl/tests/test_error_handling/test_retry_handler.py @@ -1,4 +1,5 @@ import os + import pytest import parsl diff --git a/parsl/tests/test_error_handling/test_retry_handler_failure.py b/parsl/tests/test_error_handling/test_retry_handler_failure.py index 9ec73f8442..8d5406bb95 100644 --- a/parsl/tests/test_error_handling/test_retry_handler_failure.py +++ b/parsl/tests/test_error_handling/test_retry_handler_failure.py @@ -1,6 +1,7 @@ -import parsl import pytest +import parsl + @parsl.python_app def always_fails(): diff --git a/parsl/tests/test_error_handling/test_serialization_fail.py b/parsl/tests/test_error_handling/test_serialization_fail.py index 1f62c199bf..60b54afb7a 100644 --- a/parsl/tests/test_error_handling/test_serialization_fail.py +++ b/parsl/tests/test_error_handling/test_serialization_fail.py @@ -1,8 +1,8 @@ import pytest from parsl import python_app -from parsl.tests.configs.htex_local import fresh_config from parsl.serialize.errors import SerializationError +from parsl.tests.configs.htex_local import fresh_config def local_config(): diff --git a/parsl/tests/test_error_handling/test_wrap_with_logs.py b/parsl/tests/test_error_handling/test_wrap_with_logs.py index 606b6fb7e0..28200b6538 100644 --- a/parsl/tests/test_error_handling/test_wrap_with_logs.py +++ b/parsl/tests/test_error_handling/test_wrap_with_logs.py @@ -1,4 +1,5 @@ import logging + import pytest from parsl.process_loggers import wrap_with_logs diff --git a/parsl/tests/test_flux.py b/parsl/tests/test_flux.py index 4b99d2f182..993d870239 100644 --- a/parsl/tests/test_flux.py +++ b/parsl/tests/test_flux.py @@ -1,5 +1,5 @@ -import os import concurrent.futures as cf +import os import pytest diff --git a/parsl/tests/test_htex/test_command_client_timeout.py b/parsl/tests/test_htex/test_command_client_timeout.py index 85e44f1470..f003950ac0 100644 --- a/parsl/tests/test_htex/test_command_client_timeout.py +++ b/parsl/tests/test_htex/test_command_client_timeout.py @@ -1,11 +1,15 @@ -import pytest import threading import time + +import pytest import zmq + from parsl import curvezmq +from parsl.executors.high_throughput.errors import ( + CommandClientBadError, + CommandClientTimeoutError, +) from parsl.executors.high_throughput.zmq_pipes import CommandClient -from parsl.executors.high_throughput.errors import CommandClientTimeoutError, CommandClientBadError - # Time constant used for timeout tests: various delays and # timeouts will be appropriate multiples of this, but the diff --git a/parsl/tests/test_htex/test_connected_blocks.py b/parsl/tests/test_htex/test_connected_blocks.py index 1e53d84dcd..94f36b3578 100644 --- a/parsl/tests/test_htex/test_connected_blocks.py +++ b/parsl/tests/test_htex/test_connected_blocks.py @@ -1,7 +1,8 @@ -import parsl import pytest -from parsl.executors import HighThroughputExecutor + +import parsl from parsl import Config +from parsl.executors import HighThroughputExecutor from parsl.providers import LocalProvider diff --git a/parsl/tests/test_htex/test_cpu_affinity_explicit.py b/parsl/tests/test_htex/test_cpu_affinity_explicit.py index 4b50b04242..0dfef43283 100644 --- a/parsl/tests/test_htex/test_cpu_affinity_explicit.py +++ b/parsl/tests/test_htex/test_cpu_affinity_explicit.py @@ -1,8 +1,10 @@ import logging import os -import parsl -import pytest import random + +import pytest + +import parsl from parsl.tests.configs.htex_local import fresh_config logger = logging.getLogger(__name__) diff --git a/parsl/tests/test_htex/test_disconnected_blocks.py b/parsl/tests/test_htex/test_disconnected_blocks.py index adf414a8a1..2fc6c627e0 100644 --- a/parsl/tests/test_htex/test_disconnected_blocks.py +++ b/parsl/tests/test_htex/test_disconnected_blocks.py @@ -1,11 +1,13 @@ import logging -import parsl + import pytest -from parsl.executors import HighThroughputExecutor + +import parsl from parsl import Config -from parsl.providers import LocalProvider +from parsl.executors import HighThroughputExecutor from parsl.executors.errors import BadStateException -from parsl.jobs.states import JobStatus, JobState +from parsl.jobs.states import JobState, JobStatus +from parsl.providers import LocalProvider def local_config(): diff --git a/parsl/tests/test_htex/test_drain.py b/parsl/tests/test_htex/test_drain.py index 4787d97e88..da115b42f0 100644 --- a/parsl/tests/test_htex/test_drain.py +++ b/parsl/tests/test_htex/test_drain.py @@ -1,13 +1,13 @@ -import parsl -import pytest import time -from parsl.providers import LocalProvider -from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher +import pytest +import parsl +from parsl.channels import LocalChannel from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider # this constant is used to scale some durations that happen # based around the expected drain period: the drain period diff --git a/parsl/tests/test_htex/test_htex.py b/parsl/tests/test_htex/test_htex.py index 8850a9cc3e..ca95773e1b 100644 --- a/parsl/tests/test_htex/test_htex.py +++ b/parsl/tests/test_htex/test_htex.py @@ -4,8 +4,7 @@ import pytest -from parsl import curvezmq -from parsl import HighThroughputExecutor +from parsl import HighThroughputExecutor, curvezmq from parsl.multiprocessing import ForkProcess _MOCK_BASE = "parsl.executors.high_throughput.executor" diff --git a/parsl/tests/test_htex/test_managers_command.py b/parsl/tests/test_htex/test_managers_command.py index 99d896f5ff..272f24f21c 100644 --- a/parsl/tests/test_htex/test_managers_command.py +++ b/parsl/tests/test_htex/test_managers_command.py @@ -1,7 +1,8 @@ -import pytest import logging - import sys + +import pytest + import parsl from parsl.app.app import python_app from parsl.tests.configs.htex_local import fresh_config diff --git a/parsl/tests/test_htex/test_multiple_disconnected_blocks.py b/parsl/tests/test_htex/test_multiple_disconnected_blocks.py index 616eefbb35..159c20f58d 100644 --- a/parsl/tests/test_htex/test_multiple_disconnected_blocks.py +++ b/parsl/tests/test_htex/test_multiple_disconnected_blocks.py @@ -1,11 +1,13 @@ import logging -import parsl + import pytest -from parsl.executors import HighThroughputExecutor + +import parsl from parsl import Config -from parsl.providers import LocalProvider +from parsl.executors import HighThroughputExecutor from parsl.executors.errors import BadStateException -from parsl.jobs.states import JobStatus, JobState +from parsl.jobs.states import JobState, JobStatus +from parsl.providers import LocalProvider def local_config(): diff --git a/parsl/tests/test_monitoring/test_app_names.py b/parsl/tests/test_monitoring/test_app_names.py index 62f1f6eb35..1c3b6fef11 100644 --- a/parsl/tests/test_monitoring/test_app_names.py +++ b/parsl/tests/test_monitoring/test_app_names.py @@ -2,10 +2,11 @@ """ import os -import parsl -import pytest import time +import pytest + +import parsl from parsl.tests.configs.htex_local_alternate import fresh_config diff --git a/parsl/tests/test_monitoring/test_basic.py b/parsl/tests/test_monitoring/test_basic.py index e777b15922..c900670ec8 100644 --- a/parsl/tests/test_monitoring/test_basic.py +++ b/parsl/tests/test_monitoring/test_basic.py @@ -1,12 +1,12 @@ import os -import parsl -import pytest import time +import pytest + +import parsl from parsl import HighThroughputExecutor from parsl.config import Config -from parsl.executors.taskvine import TaskVineExecutor -from parsl.executors.taskvine import TaskVineManagerConfig +from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig from parsl.monitoring import MonitoringHub diff --git a/parsl/tests/test_monitoring/test_db_locks.py b/parsl/tests/test_monitoring/test_db_locks.py index 5a4f761a54..e1d821a40e 100644 --- a/parsl/tests/test_monitoring/test_db_locks.py +++ b/parsl/tests/test_monitoring/test_db_locks.py @@ -1,9 +1,11 @@ import logging import os -import parsl -import pytest import time +import pytest + +import parsl + logger = logging.getLogger(__name__) @@ -14,9 +16,10 @@ def this_app(): @pytest.mark.local def test_row_counts(): - from parsl.tests.configs.htex_local_alternate import fresh_config import sqlalchemy from sqlalchemy import text + + from parsl.tests.configs.htex_local_alternate import fresh_config if os.path.exists("runinfo/monitoring.db"): logger.info("Monitoring database already exists - deleting") os.remove("runinfo/monitoring.db") diff --git a/parsl/tests/test_monitoring/test_fuzz_zmq.py b/parsl/tests/test_monitoring/test_fuzz_zmq.py index 16f89ea560..36f048efb3 100644 --- a/parsl/tests/test_monitoring/test_fuzz_zmq.py +++ b/parsl/tests/test_monitoring/test_fuzz_zmq.py @@ -1,11 +1,13 @@ import logging import os -import parsl -import pytest import socket import time + +import pytest import zmq +import parsl + logger = logging.getLogger(__name__) @@ -16,10 +18,11 @@ def this_app(): @pytest.mark.local def test_row_counts(): - from parsl.tests.configs.htex_local_alternate import fresh_config import sqlalchemy from sqlalchemy import text + from parsl.tests.configs.htex_local_alternate import fresh_config + if os.path.exists("runinfo/monitoring.db"): logger.info("Monitoring database already exists - deleting") os.remove("runinfo/monitoring.db") diff --git a/parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py b/parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py index be9599ef39..eb7a25003b 100644 --- a/parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +++ b/parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py @@ -1,16 +1,16 @@ import logging import os -import parsl -import pytest import time -from parsl.providers import LocalProvider -from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher +import pytest +import parsl +from parsl.channels import LocalChannel from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.launchers import SimpleLauncher from parsl.monitoring import MonitoringHub +from parsl.providers import LocalProvider def fresh_config(run_dir, strategy, db_url): diff --git a/parsl/tests/test_monitoring/test_incomplete_futures.py b/parsl/tests/test_monitoring/test_incomplete_futures.py index 7068821055..c73d973150 100644 --- a/parsl/tests/test_monitoring/test_incomplete_futures.py +++ b/parsl/tests/test_monitoring/test_incomplete_futures.py @@ -1,11 +1,12 @@ import logging import os -import parsl -import pytest import random - from concurrent.futures import Future +import pytest + +import parsl + @parsl.python_app def this_app(inputs=()): @@ -16,6 +17,7 @@ def this_app(inputs=()): def test_future_representation(tmpd_cwd): import sqlalchemy from sqlalchemy import text + from parsl.tests.configs.htex_local_alternate import fresh_config monitoring_db = str(tmpd_cwd / "monitoring.db") diff --git a/parsl/tests/test_monitoring/test_memoization_representation.py b/parsl/tests/test_monitoring/test_memoization_representation.py index 497d0d2ccd..08d12bb113 100644 --- a/parsl/tests/test_monitoring/test_memoization_representation.py +++ b/parsl/tests/test_monitoring/test_memoization_representation.py @@ -1,9 +1,11 @@ import logging import os -import parsl + import pytest +import parsl + logger = logging.getLogger(__name__) @@ -16,6 +18,7 @@ def this_app(x): def test_hashsum(): import sqlalchemy from sqlalchemy import text + from parsl.tests.configs.htex_local_alternate import fresh_config if os.path.exists("runinfo/monitoring.db"): diff --git a/parsl/tests/test_monitoring/test_stdouterr.py b/parsl/tests/test_monitoring/test_stdouterr.py index a05900c7f2..d1817164c0 100644 --- a/parsl/tests/test_monitoring/test_stdouterr.py +++ b/parsl/tests/test_monitoring/test_stdouterr.py @@ -3,16 +3,16 @@ import logging import os -import parsl -import pytest import re import time - from typing import Union +import pytest + +import parsl from parsl.config import Config -from parsl.data_provider.files import File from parsl.data_provider.data_manager import default_staging +from parsl.data_provider.files import File from parsl.data_provider.staging import Staging from parsl.executors import HighThroughputExecutor from parsl.monitoring import MonitoringHub diff --git a/parsl/tests/test_monitoring/test_viz_colouring.py b/parsl/tests/test_monitoring/test_viz_colouring.py index bcbfd3d5e7..5d7d82f9f3 100644 --- a/parsl/tests/test_monitoring/test_viz_colouring.py +++ b/parsl/tests/test_monitoring/test_viz_colouring.py @@ -1,4 +1,5 @@ import pytest + from parsl.dataflow.states import States diff --git a/parsl/tests/test_mpi_apps/test_bad_mpi_config.py b/parsl/tests/test_mpi_apps/test_bad_mpi_config.py index edfd54226c..336bf87703 100644 --- a/parsl/tests/test_mpi_apps/test_bad_mpi_config.py +++ b/parsl/tests/test_mpi_apps/test_bad_mpi_config.py @@ -2,7 +2,7 @@ from parsl import Config from parsl.executors import HighThroughputExecutor -from parsl.launchers import SrunLauncher, AprunLauncher, SimpleLauncher +from parsl.launchers import AprunLauncher, SimpleLauncher, SrunLauncher from parsl.providers import SlurmProvider diff --git a/parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py b/parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py index a6218f4902..e1e5c70883 100644 --- a/parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +++ b/parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py @@ -1,5 +1,7 @@ from typing import Dict + import pytest + import parsl from parsl import python_app from parsl.tests.configs.htex_local import fresh_config diff --git a/parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py b/parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py index c3d1bcfb26..6743d40eba 100644 --- a/parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +++ b/parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py @@ -1,15 +1,17 @@ import logging +import os import random from typing import Dict + import pytest + import parsl -from parsl import python_app, bash_app +from parsl import bash_app, python_app +from parsl.executors.high_throughput.mpi_prefix_composer import ( + MissingResourceSpecification, +) from parsl.tests.configs.htex_local import fresh_config -from parsl.executors.high_throughput.mpi_prefix_composer import MissingResourceSpecification - -import os - EXECUTOR_LABEL = "MPI_TEST" diff --git a/parsl/tests/test_mpi_apps/test_mpi_prefix.py b/parsl/tests/test_mpi_apps/test_mpi_prefix.py index 37c5dc7553..48740950a0 100644 --- a/parsl/tests/test_mpi_apps/test_mpi_prefix.py +++ b/parsl/tests/test_mpi_apps/test_mpi_prefix.py @@ -1,14 +1,14 @@ import logging + import pytest -from parsl.executors.high_throughput.mpi_resource_management import Scheduler from parsl.executors.high_throughput.mpi_prefix_composer import ( - compose_srun_launch_cmd, + compose_all, compose_aprun_launch_cmd, compose_mpiexec_launch_cmd, - compose_all, + compose_srun_launch_cmd, ) - +from parsl.executors.high_throughput.mpi_resource_management import Scheduler resource_spec = {"num_nodes": 2, "num_ranks": 8, diff --git a/parsl/tests/test_mpi_apps/test_mpi_scheduler.py b/parsl/tests/test_mpi_apps/test_mpi_scheduler.py index 4e77231b77..92248cafe9 100644 --- a/parsl/tests/test_mpi_apps/test_mpi_scheduler.py +++ b/parsl/tests/test_mpi_apps/test_mpi_scheduler.py @@ -1,9 +1,14 @@ import logging import os +import pickle from unittest import mock + import pytest -import pickle -from parsl.executors.high_throughput.mpi_resource_management import TaskScheduler, MPITaskScheduler + +from parsl.executors.high_throughput.mpi_resource_management import ( + MPITaskScheduler, + TaskScheduler, +) from parsl.multiprocessing import SpawnContext from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message diff --git a/parsl/tests/test_mpi_apps/test_mpiex.py b/parsl/tests/test_mpi_apps/test_mpiex.py index b9f1e41072..1b3e86e0b8 100644 --- a/parsl/tests/test_mpi_apps/test_mpiex.py +++ b/parsl/tests/test_mpi_apps/test_mpiex.py @@ -5,11 +5,12 @@ import pytest import parsl -from .test_mpi_mode_enabled import get_env_vars -from parsl import HighThroughputExecutor, Config +from parsl import Config, HighThroughputExecutor +from parsl.executors.high_throughput.mpi_executor import MPIExecutor from parsl.launchers import SimpleLauncher from parsl.providers import LocalProvider -from parsl.executors.high_throughput.mpi_executor import MPIExecutor + +from .test_mpi_mode_enabled import get_env_vars cwd = Path(__file__).parent.absolute() pbs_nodefile = cwd.joinpath("mocks", "pbs_nodefile") diff --git a/parsl/tests/test_mpi_apps/test_resource_spec.py b/parsl/tests/test_mpi_apps/test_resource_spec.py index 7e533c128f..99d0187ccd 100644 --- a/parsl/tests/test_mpi_apps/test_resource_spec.py +++ b/parsl/tests/test_mpi_apps/test_resource_spec.py @@ -2,26 +2,25 @@ import logging import os import typing - +import unittest +from typing import Dict import pytest -import unittest import parsl from parsl.app.app import python_app -from parsl.tests.configs.htex_local import fresh_config -from typing import Dict +from parsl.executors.high_throughput.mpi_prefix_composer import ( + InvalidResourceSpecification, + MissingResourceSpecification, + validate_resource_spec, +) from parsl.executors.high_throughput.mpi_resource_management import ( + get_nodes_in_batchjob, get_pbs_hosts_list, get_slurm_hosts_list, - get_nodes_in_batchjob, identify_scheduler, ) -from parsl.executors.high_throughput.mpi_prefix_composer import ( - validate_resource_spec, - InvalidResourceSpecification, - MissingResourceSpecification -) +from parsl.tests.configs.htex_local import fresh_config EXECUTOR_LABEL = "MPI_TEST" diff --git a/parsl/tests/test_providers/test_cobalt_deprecation_warning.py b/parsl/tests/test_providers/test_cobalt_deprecation_warning.py index 0ae39439f3..249a4d90b9 100644 --- a/parsl/tests/test_providers/test_cobalt_deprecation_warning.py +++ b/parsl/tests/test_providers/test_cobalt_deprecation_warning.py @@ -1,5 +1,7 @@ import warnings + import pytest + from parsl.providers import CobaltProvider diff --git a/parsl/tests/test_providers/test_local_provider.py b/parsl/tests/test_providers/test_local_provider.py index 293884b062..29907ec47d 100644 --- a/parsl/tests/test_providers/test_local_provider.py +++ b/parsl/tests/test_providers/test_local_provider.py @@ -1,7 +1,6 @@ import logging import os import pathlib -import pytest import random import shutil import socket @@ -10,6 +9,8 @@ import threading import time +import pytest + from parsl.channels import LocalChannel, SSHChannel from parsl.jobs.states import JobState from parsl.launchers import SingleNodeLauncher diff --git a/parsl/tests/test_providers/test_pbspro_template.py b/parsl/tests/test_providers/test_pbspro_template.py index 6269da06c0..1a61118047 100644 --- a/parsl/tests/test_providers/test_pbspro_template.py +++ b/parsl/tests/test_providers/test_pbspro_template.py @@ -1,6 +1,6 @@ import random - from unittest import mock + import pytest from parsl.channels import LocalChannel diff --git a/parsl/tests/test_providers/test_slurm_template.py b/parsl/tests/test_providers/test_slurm_template.py index 6b799466ce..72bacf4e1d 100644 --- a/parsl/tests/test_providers/test_slurm_template.py +++ b/parsl/tests/test_providers/test_slurm_template.py @@ -1,7 +1,7 @@ import logging import random - from unittest import mock + import pytest from parsl.channels import LocalChannel diff --git a/parsl/tests/test_providers/test_submiterror_deprecation.py b/parsl/tests/test_providers/test_submiterror_deprecation.py index 38151fd234..352e68441b 100644 --- a/parsl/tests/test_providers/test_submiterror_deprecation.py +++ b/parsl/tests/test_providers/test_submiterror_deprecation.py @@ -1,7 +1,8 @@ -import pytest import random import string +import pytest + from parsl.providers.errors import SubmitException diff --git a/parsl/tests/test_python_apps/test_context_manager.py b/parsl/tests/test_python_apps/test_context_manager.py index 1ee45adca3..a314c0d362 100644 --- a/parsl/tests/test_python_apps/test_context_manager.py +++ b/parsl/tests/test_python_apps/test_context_manager.py @@ -1,8 +1,9 @@ -import parsl -from parsl.tests.configs.local_threads import fresh_config import pytest -from parsl.errors import NoDataFlowKernelError + +import parsl from parsl.dataflow.dflow import DataFlowKernel +from parsl.errors import NoDataFlowKernelError +from parsl.tests.configs.local_threads import fresh_config @parsl.python_app diff --git a/parsl/tests/test_python_apps/test_dep_standard_futures.py b/parsl/tests/test_python_apps/test_dep_standard_futures.py index 7b239de7b9..4856888d35 100644 --- a/parsl/tests/test_python_apps/test_dep_standard_futures.py +++ b/parsl/tests/test_python_apps/test_dep_standard_futures.py @@ -1,6 +1,7 @@ +from concurrent.futures import Future + import parsl from parsl.dataflow.errors import DependencyError -from concurrent.futures import Future @parsl.python_app diff --git a/parsl/tests/test_python_apps/test_futures.py b/parsl/tests/test_python_apps/test_futures.py index 5b914dfe0a..500845f73b 100644 --- a/parsl/tests/test_python_apps/test_futures.py +++ b/parsl/tests/test_python_apps/test_futures.py @@ -11,9 +11,10 @@ 2. done() called on 1, vs 2 """ -import pytest from os.path import basename +import pytest + from parsl.app.app import python_app from parsl.data_provider.files import File diff --git a/parsl/tests/test_python_apps/test_join.py b/parsl/tests/test_python_apps/test_join.py index 3a812a1400..dcb855a6f7 100644 --- a/parsl/tests/test_python_apps/test_join.py +++ b/parsl/tests/test_python_apps/test_join.py @@ -3,7 +3,6 @@ from parsl import join_app, python_app from parsl.dataflow.errors import JoinError - RESULT_CONSTANT = 3 diff --git a/parsl/tests/test_python_apps/test_lifted.py b/parsl/tests/test_python_apps/test_lifted.py index 57ea3b5152..77084ac03f 100644 --- a/parsl/tests/test_python_apps/test_lifted.py +++ b/parsl/tests/test_python_apps/test_lifted.py @@ -1,10 +1,10 @@ +from concurrent.futures import Future +from typing import TypeVar + import pytest -from concurrent.futures import Future from parsl import python_app -from typing import TypeVar - T = TypeVar('T') diff --git a/parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py b/parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py index e9db0f0452..d423545773 100644 --- a/parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +++ b/parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py @@ -1,4 +1,5 @@ import pytest + from parsl import python_app from parsl.dataflow.memoization import id_for_memo diff --git a/parsl/tests/test_python_apps/test_pluggable_future_resolution.py b/parsl/tests/test_python_apps/test_pluggable_future_resolution.py index b219c9384d..4793c0ac19 100644 --- a/parsl/tests/test_python_apps/test_pluggable_future_resolution.py +++ b/parsl/tests/test_python_apps/test_pluggable_future_resolution.py @@ -7,8 +7,8 @@ import parsl from parsl.config import Config -from parsl.dataflow.errors import DependencyError from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER +from parsl.dataflow.errors import DependencyError def local_config(): diff --git a/parsl/tests/test_radical/test_mpi_funcs.py b/parsl/tests/test_radical/test_mpi_funcs.py index 57215287f0..2c69e26dd8 100644 --- a/parsl/tests/test_radical/test_mpi_funcs.py +++ b/parsl/tests/test_radical/test_mpi_funcs.py @@ -1,6 +1,6 @@ -import parsl import pytest +import parsl from parsl.tests.configs.local_radical_mpi import fresh_config as local_config diff --git a/parsl/tests/test_regression/test_1480.py b/parsl/tests/test_regression/test_1480.py index 93ed765b65..a52a1f3e59 100644 --- a/parsl/tests/test_regression/test_1480.py +++ b/parsl/tests/test_regression/test_1480.py @@ -1,5 +1,6 @@ -from parsl import python_app import pytest + +from parsl import python_app from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_regression/test_1653.py b/parsl/tests/test_regression/test_1653.py index 79c58fdc3b..61ab1bd4f8 100644 --- a/parsl/tests/test_regression/test_1653.py +++ b/parsl/tests/test_regression/test_1653.py @@ -1,5 +1,6 @@ -from parsl import python_app import pytest + +from parsl import python_app from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_regression/test_2652.py b/parsl/tests/test_regression/test_2652.py index b3cbcf58b8..42cdae9ca3 100644 --- a/parsl/tests/test_regression/test_2652.py +++ b/parsl/tests/test_regression/test_2652.py @@ -1,4 +1,5 @@ import pytest + from parsl.jobs.states import JobState diff --git a/parsl/tests/test_regression/test_69a.py b/parsl/tests/test_regression/test_69a.py index 3433f54d0f..fa6fbb4116 100644 --- a/parsl/tests/test_regression/test_69a.py +++ b/parsl/tests/test_regression/test_69a.py @@ -6,7 +6,6 @@ from parsl.app.app import bash_app, python_app from parsl.tests.configs.local_threads import config - local_config = config diff --git a/parsl/tests/test_regression/test_854.py b/parsl/tests/test_regression/test_854.py index 2450e0763c..e11d7fe8ac 100644 --- a/parsl/tests/test_regression/test_854.py +++ b/parsl/tests/test_regression/test_854.py @@ -1,8 +1,10 @@ -import time import multiprocessing +import random +import time + import pytest + from parsl.multiprocessing import MacSafeQueue -import random def consumer(in_q, out_q, delay=0): diff --git a/parsl/tests/test_regression/test_97_parallelism_0.py b/parsl/tests/test_regression/test_97_parallelism_0.py index de06bcd9df..a83a5b795a 100644 --- a/parsl/tests/test_regression/test_97_parallelism_0.py +++ b/parsl/tests/test_regression/test_97_parallelism_0.py @@ -1,11 +1,10 @@ import pytest import parsl - from parsl.config import Config from parsl.executors import HighThroughputExecutor -from parsl.providers import LocalProvider from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider def local_config() -> Config: diff --git a/parsl/tests/test_regression/test_98.py b/parsl/tests/test_regression/test_98.py index 3f3bacd079..486e33f11b 100644 --- a/parsl/tests/test_regression/test_98.py +++ b/parsl/tests/test_regression/test_98.py @@ -6,7 +6,6 @@ import pytest import parsl - from parsl.dataflow.dflow import DataFlowKernel from parsl.tests.configs.local_threads import config diff --git a/parsl/tests/test_scaling/test_block_error_handler.py b/parsl/tests/test_scaling/test_block_error_handler.py index 7d9ffedd17..3a7beb9f8b 100644 --- a/parsl/tests/test_scaling/test_block_error_handler.py +++ b/parsl/tests/test_scaling/test_block_error_handler.py @@ -1,11 +1,16 @@ +from functools import partial +from unittest.mock import Mock + import pytest from parsl.executors import HighThroughputExecutor +from parsl.jobs.error_handlers import ( + noop_error_handler, + simple_error_handler, + windowed_error_handler, +) +from parsl.jobs.states import JobState, JobStatus from parsl.providers import LocalProvider -from unittest.mock import Mock -from parsl.jobs.states import JobStatus, JobState -from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler -from functools import partial @pytest.mark.local diff --git a/parsl/tests/test_scaling/test_scale_down.py b/parsl/tests/test_scaling/test_scale_down.py index 42ced21291..7fb72ba507 100644 --- a/parsl/tests/test_scaling/test_scale_down.py +++ b/parsl/tests/test_scaling/test_scale_down.py @@ -4,13 +4,12 @@ import pytest import parsl - from parsl import File, python_app -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SingleNodeLauncher from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.launchers import SingleNodeLauncher +from parsl.providers import LocalProvider logger = logging.getLogger(__name__) diff --git a/parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py b/parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py index f3d581a177..424caac452 100644 --- a/parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +++ b/parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py @@ -1,15 +1,14 @@ +from threading import Event + import pytest import parsl - from parsl import File, python_app -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SingleNodeLauncher from parsl.config import Config from parsl.executors import HighThroughputExecutor - -from threading import Event +from parsl.launchers import SingleNodeLauncher +from parsl.providers import LocalProvider _max_blocks = 5 _min_blocks = 0 diff --git a/parsl/tests/test_scaling/test_scale_down_htex_unregistered.py b/parsl/tests/test_scaling/test_scale_down_htex_unregistered.py index 7d50aecd8a..dd1b164478 100644 --- a/parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +++ b/parsl/tests/test_scaling/test_scale_down_htex_unregistered.py @@ -4,14 +4,13 @@ import pytest import parsl - from parsl import File, python_app -from parsl.jobs.states import JobState, TERMINAL_STATES -from parsl.providers import LocalProvider from parsl.channels import LocalChannel -from parsl.launchers import SingleNodeLauncher from parsl.config import Config from parsl.executors import HighThroughputExecutor +from parsl.jobs.states import TERMINAL_STATES, JobState +from parsl.launchers import SingleNodeLauncher +from parsl.providers import LocalProvider logger = logging.getLogger(__name__) diff --git a/parsl/tests/test_scaling/test_shutdown_scalein.py b/parsl/tests/test_scaling/test_shutdown_scalein.py index 05c70e451b..1d1557ebb1 100644 --- a/parsl/tests/test_scaling/test_shutdown_scalein.py +++ b/parsl/tests/test_scaling/test_shutdown_scalein.py @@ -1,3 +1,4 @@ +import random import threading import pytest @@ -9,8 +10,6 @@ from parsl.launchers import SimpleLauncher from parsl.providers import LocalProvider -import random - # we need some blocks, but it doesn't matter too much how many, as long # as they can all start up and get registered within the try_assert # timeout later on. diff --git a/parsl/tests/test_serialization/test_2555_caching_deserializer.py b/parsl/tests/test_serialization/test_2555_caching_deserializer.py index 8438d94dfb..7a46731e2f 100644 --- a/parsl/tests/test_serialization/test_2555_caching_deserializer.py +++ b/parsl/tests/test_serialization/test_2555_caching_deserializer.py @@ -1,6 +1,6 @@ -import parsl import pytest +import parsl from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_serialization/test_basic.py b/parsl/tests/test_serialization/test_basic.py index ad89d1eccc..15f506fbe1 100644 --- a/parsl/tests/test_serialization/test_basic.py +++ b/parsl/tests/test_serialization/test_basic.py @@ -1,5 +1,6 @@ import pytest -from parsl.serialize import serialize, deserialize + +from parsl.serialize import deserialize, serialize from parsl.serialize.concretes import DillSerializer, PickleSerializer diff --git a/parsl/tests/test_serialization/test_htex_code_cache.py b/parsl/tests/test_serialization/test_htex_code_cache.py index 6695c5f460..65cb72527a 100644 --- a/parsl/tests/test_serialization/test_htex_code_cache.py +++ b/parsl/tests/test_serialization/test_htex_code_cache.py @@ -1,10 +1,9 @@ -import parsl -import pytest - from typing import Any -from parsl.serialize.facade import methods_for_code +import pytest +import parsl +from parsl.serialize.facade import methods_for_code from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_serialization/test_pack_resource_spec.py b/parsl/tests/test_serialization/test_pack_resource_spec.py index 922e124cfc..6020871e0b 100644 --- a/parsl/tests/test_serialization/test_pack_resource_spec.py +++ b/parsl/tests/test_serialization/test_pack_resource_spec.py @@ -1,5 +1,6 @@ import pytest -from parsl.serialize import unpack_res_spec_apply_message, pack_res_spec_apply_message + +from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message def double(x: int, y: int = 2) -> int: diff --git a/parsl/tests/test_serialization/test_proxystore_configured.py b/parsl/tests/test_serialization/test_proxystore_configured.py index f03713a4f9..90bea12ac2 100644 --- a/parsl/tests/test_serialization/test_proxystore_configured.py +++ b/parsl/tests/test_serialization/test_proxystore_configured.py @@ -1,19 +1,24 @@ import logging -import pytest import uuid +import pytest + import parsl -from parsl.serialize.facade import additional_methods_for_deserialization, methods_for_data, register_method_for_data +from parsl.serialize.facade import ( + additional_methods_for_deserialization, + methods_for_data, + register_method_for_data, +) from parsl.tests.configs.htex_local import fresh_config - logger = logging.getLogger(__name__) def local_setup(): - from parsl.serialize.proxystore import ProxyStoreSerializer - from proxystore.store import Store, register_store from proxystore.connectors.file import FileConnector + from proxystore.store import Store, register_store + + from parsl.serialize.proxystore import ProxyStoreSerializer parsl.load(fresh_config()) diff --git a/parsl/tests/test_serialization/test_proxystore_impl.py b/parsl/tests/test_serialization/test_proxystore_impl.py index fefc4e2a6e..5a04af0faf 100644 --- a/parsl/tests/test_serialization/test_proxystore_impl.py +++ b/parsl/tests/test_serialization/test_proxystore_impl.py @@ -1,6 +1,7 @@ -import pytest import uuid +import pytest + def policy_example(o): """Example policy will proxy only lists.""" @@ -13,10 +14,11 @@ def test_proxystore_nonglobal(): """ # import in function, because proxystore is not importable in base parsl # installation. - from parsl.serialize.proxystore import ProxyStoreSerializer + from proxystore.connectors.file import FileConnector from proxystore.proxy import Proxy from proxystore.store import Store, register_store - from proxystore.connectors.file import FileConnector + + from parsl.serialize.proxystore import ProxyStoreSerializer store = Store(name='parsl_store_' + str(uuid.uuid4()), connector=FileConnector(store_dir="/tmp")) register_store(store) diff --git a/parsl/tests/test_shutdown/test_kill_monitoring.py b/parsl/tests/test_shutdown/test_kill_monitoring.py index 3a4b8778c6..0fb56f08d8 100644 --- a/parsl/tests/test_shutdown/test_kill_monitoring.py +++ b/parsl/tests/test_shutdown/test_kill_monitoring.py @@ -1,9 +1,10 @@ import os -import parsl -import pytest import signal import time +import pytest + +import parsl from parsl.tests.configs.htex_local_alternate import fresh_config # This is a very generous upper bound on expected shutdown time of target diff --git a/parsl/tests/test_staging/staging_provider.py b/parsl/tests/test_staging/staging_provider.py index 8006ef118b..d309e8d305 100644 --- a/parsl/tests/test_staging/staging_provider.py +++ b/parsl/tests/test_staging/staging_provider.py @@ -62,8 +62,8 @@ def make_stage_out_app(executor, dfk): def stage_out_noop(app_fu, inputs=[], _parsl_staging_inhibit=True): - import time import logging + import time logger = logging.getLogger(__name__) logger.info("stage_out_noop") time.sleep(1) @@ -75,8 +75,8 @@ def make_stage_in_app(executor, dfk): def stage_in_noop(parent_fut=None, outputs=[], _parsl_staging_inhibit=True): - import time import logging + import time logger = logging.getLogger(__name__) logger.info("stage_in_noop") time.sleep(1) diff --git a/parsl/tests/test_staging/test_1316.py b/parsl/tests/test_staging/test_1316.py index 4a90da0f2b..9d5b7a33b2 100644 --- a/parsl/tests/test_staging/test_1316.py +++ b/parsl/tests/test_staging/test_1316.py @@ -1,8 +1,9 @@ +import time + import pytest import parsl -import time -from parsl import python_app, ThreadPoolExecutor +from parsl import ThreadPoolExecutor, python_app from parsl.config import Config from parsl.data_provider.files import File from parsl.data_provider.staging import Staging diff --git a/parsl/tests/test_staging/test_docs_1.py b/parsl/tests/test_staging/test_docs_1.py index e44c3b7344..8f549ae9b3 100644 --- a/parsl/tests/test_staging/test_docs_1.py +++ b/parsl/tests/test_staging/test_docs_1.py @@ -1,6 +1,6 @@ import pytest -from parsl import python_app, File +from parsl import File, python_app @python_app diff --git a/parsl/tests/test_staging/test_docs_2.py b/parsl/tests/test_staging/test_docs_2.py index 12c48b8c7f..a985872bce 100644 --- a/parsl/tests/test_staging/test_docs_2.py +++ b/parsl/tests/test_staging/test_docs_2.py @@ -1,5 +1,6 @@ import pytest -from parsl import bash_app, File + +from parsl import File, bash_app from parsl.tests.configs.local_threads import fresh_config as local_config diff --git a/parsl/tests/test_staging/test_elaborate_noop_file.py b/parsl/tests/test_staging/test_elaborate_noop_file.py index 87d960a652..6f52536059 100644 --- a/parsl/tests/test_staging/test_elaborate_noop_file.py +++ b/parsl/tests/test_staging/test_elaborate_noop_file.py @@ -6,15 +6,15 @@ # of the globus staging provider import logging + import pytest import parsl - from parsl import bash_app, python_app from parsl.config import Config from parsl.data_provider.files import File from parsl.executors.threads import ThreadPoolExecutor -from parsl.tests.test_staging.staging_provider import NoOpTestingFileStaging, NoOpError +from parsl.tests.test_staging.staging_provider import NoOpError, NoOpTestingFileStaging logger = logging.getLogger(__name__) diff --git a/parsl/tests/test_staging/test_staging_https.py b/parsl/tests/test_staging/test_staging_https.py index 6169a264ef..4a68e66a5c 100644 --- a/parsl/tests/test_staging/test_staging_https.py +++ b/parsl/tests/test_staging/test_staging_https.py @@ -1,9 +1,9 @@ +import pytest + import parsl from parsl.app.app import python_app from parsl.data_provider.files import File -import pytest - # This config is for the local test which will adding an executor. # Most tests in this file should be non-local and use the configuration # specificed with --config, not this one. diff --git a/parsl/tests/test_staging/test_staging_stdout.py b/parsl/tests/test_staging/test_staging_stdout.py index 616bd40448..aaa45440a7 100644 --- a/parsl/tests/test_staging/test_staging_stdout.py +++ b/parsl/tests/test_staging/test_staging_stdout.py @@ -1,12 +1,13 @@ import logging import os -import parsl -import pytest import zipfile +import pytest + +import parsl from parsl.app.futures import DataFuture -from parsl.tests.configs.htex_local import fresh_config as local_config from parsl.data_provider.files import File +from parsl.tests.configs.htex_local import fresh_config as local_config @parsl.bash_app diff --git a/parsl/tests/test_staging/test_zip_in.py b/parsl/tests/test_staging/test_zip_in.py index 69f3d9f534..1f74f7e11b 100644 --- a/parsl/tests/test_staging/test_zip_in.py +++ b/parsl/tests/test_staging/test_zip_in.py @@ -1,18 +1,16 @@ -import parsl -import pytest import random import zipfile -from parsl.data_provider.files import File -from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging +import pytest -from parsl.providers import LocalProvider +import parsl from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher - from parsl.config import Config +from parsl.data_provider.files import File +from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging from parsl.executors import HighThroughputExecutor - +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_staging/test_zip_out.py b/parsl/tests/test_staging/test_zip_out.py index cfdeb14f9f..e369031033 100644 --- a/parsl/tests/test_staging/test_zip_out.py +++ b/parsl/tests/test_staging/test_zip_out.py @@ -1,18 +1,16 @@ -import parsl -import pytest import zipfile -from parsl.data_provider.files import File -from parsl.data_provider.data_manager import default_staging -from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging +import pytest -from parsl.providers import LocalProvider +import parsl from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher - from parsl.config import Config +from parsl.data_provider.data_manager import default_staging +from parsl.data_provider.files import File +from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging from parsl.executors import HighThroughputExecutor - +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_staging/test_zip_to_zip.py b/parsl/tests/test_staging/test_zip_to_zip.py index 6c3b2d371c..2c78e3bec2 100644 --- a/parsl/tests/test_staging/test_zip_to_zip.py +++ b/parsl/tests/test_staging/test_zip_to_zip.py @@ -1,18 +1,16 @@ -import parsl -import pytest import random import zipfile -from parsl.data_provider.files import File -from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging +import pytest -from parsl.providers import LocalProvider +import parsl from parsl.channels import LocalChannel -from parsl.launchers import SimpleLauncher - from parsl.config import Config +from parsl.data_provider.files import File +from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging from parsl.executors import HighThroughputExecutor - +from parsl.launchers import SimpleLauncher +from parsl.providers import LocalProvider from parsl.tests.configs.htex_local import fresh_config as local_config diff --git a/parsl/tests/test_summary.py b/parsl/tests/test_summary.py index 15ce3559cd..e98f393980 100644 --- a/parsl/tests/test_summary.py +++ b/parsl/tests/test_summary.py @@ -1,5 +1,6 @@ -import parsl import pytest + +import parsl from parsl.tests.configs.local_threads import fresh_config diff --git a/parsl/tests/test_thread_parallelism.py b/parsl/tests/test_thread_parallelism.py index 102396243e..e91b0631f0 100644 --- a/parsl/tests/test_thread_parallelism.py +++ b/parsl/tests/test_thread_parallelism.py @@ -5,7 +5,6 @@ from parsl.app.app import bash_app, python_app from parsl.tests.configs.local_threads import config - local_config = config diff --git a/parsl/tests/test_threads/test_configs.py b/parsl/tests/test_threads/test_configs.py index 183ae54abb..022f8825b0 100644 --- a/parsl/tests/test_threads/test_configs.py +++ b/parsl/tests/test_threads/test_configs.py @@ -7,9 +7,9 @@ @python_app def worker_identify(x, sleep_dur=0.2): - import time import os import threading + import time time.sleep(sleep_dur) return {"pid": os.getpid(), "tid": threading.current_thread()} diff --git a/parsl/tests/test_threads/test_lazy_errors.py b/parsl/tests/test_threads/test_lazy_errors.py index 9284d64e5f..a070487eb0 100644 --- a/parsl/tests/test_threads/test_lazy_errors.py +++ b/parsl/tests/test_threads/test_lazy_errors.py @@ -1,5 +1,6 @@ -import parsl import pytest + +import parsl from parsl import python_app from parsl.tests.configs.local_threads import fresh_config diff --git a/parsl/usage_tracking/api.py b/parsl/usage_tracking/api.py index cbac746906..6b31bd8caf 100644 --- a/parsl/usage_tracking/api.py +++ b/parsl/usage_tracking/api.py @@ -1,11 +1,9 @@ import inspect - -from parsl.utils import RepresentationMixin - from abc import abstractmethod from functools import singledispatch from typing import Any, List, Sequence +from parsl.utils import RepresentationMixin # Traverse the configuration hierarchy, returning a JSON component # for each one. Configuration components which implement diff --git a/parsl/usage_tracking/usage.py b/parsl/usage_tracking/usage.py index 78d9f30d31..0aafd98d21 100644 --- a/parsl/usage_tracking/usage.py +++ b/parsl/usage_tracking/usage.py @@ -1,21 +1,22 @@ -import uuid -import time -import os import json import logging +import os +import platform import socket import sys -import platform +import time +import uuid +from parsl.dataflow.states import States +from parsl.multiprocessing import ForkProcess from parsl.usage_tracking.api import get_parsl_usage from parsl.utils import setproctitle -from parsl.multiprocessing import ForkProcess -from parsl.dataflow.states import States from parsl.version import VERSION as PARSL_VERSION logger = logging.getLogger(__name__) from typing import Callable + from typing_extensions import ParamSpec # protocol version byte: when (for example) compression parameters are changed diff --git a/parsl/utils.py b/parsl/utils.py index be115158e3..6f36d4506a 100644 --- a/parsl/utils.py +++ b/parsl/utils.py @@ -7,7 +7,19 @@ import time from contextlib import contextmanager from types import TracebackType -from typing import Any, Callable, List, Sequence, Tuple, Union, Generator, IO, AnyStr, Dict, Optional +from typing import ( + IO, + Any, + AnyStr, + Callable, + Dict, + Generator, + List, + Optional, + Sequence, + Tuple, + Union, +) import typeguard from typing_extensions import Type @@ -16,7 +28,6 @@ from parsl.app.errors import BadStdStreamFile from parsl.version import VERSION - try: import setproctitle as setproctitle_module except ImportError: diff --git a/setup.py b/setup.py index 2c829ac994..dae3e64ca4 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import find_packages, setup with open('parsl/version.py') as f: exec(f.read()) From 7acf67b5d8ffb75cd77de956fe5c1ce3fe086b80 Mon Sep 17 00:00:00 2001 From: Yadu Babuji Date: Mon, 3 Jun 2024 12:10:15 -0500 Subject: [PATCH 3/3] Post isort cleanup Whitespace changes & removing dead comments --- parsl/providers/__init__.py | 5 ----- .../integration/test_stress/test_python_simple.py | 3 --- parsl/tests/scaling_tests/test_scale.py | 10 ---------- parsl/usage_tracking/api.py | 1 + 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/parsl/providers/__init__.py b/parsl/providers/__init__.py index 6a40b38ab6..475737f1f9 100644 --- a/parsl/providers/__init__.py +++ b/parsl/providers/__init__.py @@ -17,11 +17,6 @@ from parsl.providers.slurm.slurm import SlurmProvider from parsl.providers.torque.torque import TorqueProvider -# Cluster Providers - - - - __all__ = ['LocalProvider', 'CobaltProvider', 'CondorProvider', diff --git a/parsl/tests/integration/test_stress/test_python_simple.py b/parsl/tests/integration/test_stress/test_python_simple.py index 10cc4b6092..7d104ba79f 100644 --- a/parsl/tests/integration/test_stress/test_python_simple.py +++ b/parsl/tests/integration/test_stress/test_python_simple.py @@ -5,9 +5,6 @@ from parsl import python_app from parsl.configs.htex_local import config -# from parsl.configs.local_threads import config - - @python_app def increment(x): diff --git a/parsl/tests/scaling_tests/test_scale.py b/parsl/tests/scaling_tests/test_scale.py index 4331d0db53..ad940b99eb 100755 --- a/parsl/tests/scaling_tests/test_scale.py +++ b/parsl/tests/scaling_tests/test_scale.py @@ -4,18 +4,8 @@ import time import parsl - -# parsl.set_stream_logger() -# config.executors[0].provider.tasks_per_node = 4 -# parsl.load(config) from parsl.app.app import python_app # , bash_app -# from parsl.tests.configs.htex_local import config -# from htex_local import config -# from parsl.configs.local_threads import config -# from parsl.configs.local_ipp import config - - @python_app def double(x): diff --git a/parsl/usage_tracking/api.py b/parsl/usage_tracking/api.py index 6b31bd8caf..2d8b081113 100644 --- a/parsl/usage_tracking/api.py +++ b/parsl/usage_tracking/api.py @@ -12,6 +12,7 @@ # are traversed in sequence. Other types default to reporting no # usage information. + @singledispatch def get_parsl_usage(obj) -> List[Any]: return []