Merge branch 'master' into benc-flake8-bugbear

mypy.ini

@@ -32,11 +32,6 @@ disallow_untyped_defs = True
 disallow_any_expr = True
 disallow_any_decorated = True
 
-[mypy-parsl.dataflow.executor_status.*]
-disallow_untyped_defs = True
-disallow_any_expr = True
-disallow_any_decorated = True
-
 [mypy-parsl.dataflow.futures.*]
 disallow_untyped_defs = True
 disallow_any_decorated = True

parsl/app/app.py

@@ -73,15 +73,14 @@ def python_app(function=None,
                data_flow_kernel: Optional[DataFlowKernel] = None,
                cache: bool = False,
                executors: Union[List[str], Literal['all']] = 'all',
-               ignore_for_cache: Optional[List[str]] = None,
-               join: bool = False):
+               ignore_for_cache: Optional[List[str]] = None):
     """Decorator function for making python apps.
 
     Parameters
     ----------
     function : function
         Do not pass this keyword argument directly. This is needed in order to allow for omitted parenthesis,
-        for example, ``@join_app`` if using all defaults or ``@python_app(walltime=120)``. If the
+        for example, ``@python_app`` if using all defaults or ``@python_app(walltime=120)``. If the
         decorator is used alone, function will be the actual function being decorated, whereas if it
         is called with arguments, function will be None. Default is None.
     data_flow_kernel : DataFlowKernel
@@ -112,7 +111,6 @@ def wrapper(f):
 def join_app(function=None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
-             executors: Union[List[str], Literal['all']] = 'all',
              ignore_for_cache: Optional[List[str]] = None):
     """Decorator function for making join apps
 

parsl/app/python.py

@@ -36,7 +36,7 @@ def inject_exception(thread):
 class PythonApp(AppBase):
     """Extends AppBase to cover the Python App."""
 
-    def __init__(self, func, data_flow_kernel=None, cache=False, executors='all', ignore_for_cache=[], join=False):
+    def __init__(self, func, data_flow_kernel=None, cache=False, executors='all', ignore_for_cache=None, join=False):
         super().__init__(
             wrap_error(func),
             data_flow_kernel=data_flow_kernel,

parsl/jobs/strategy.py

@@ -3,12 +3,11 @@
 import time
 import math
 import warnings
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, TypedDict
 
 import parsl.jobs.job_status_poller as jsp
 
 from parsl.executors import HighThroughputExecutor
-from parsl.executors.base import ParslExecutor
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import JobState
 from parsl.process_loggers import wrap_with_logs
@@ -17,6 +16,16 @@
 logger = logging.getLogger(__name__)
 
 
+class ExecutorState(TypedDict):
+    """Strategy relevant state for an executor
+    """
+
+    idle_since: Optional[float]
+    """The timestamp at which an executor became idle.
+    If the executor is not idle, then None.
+    """
+
+
 class Strategy:
     """Scaling strategy.
 
@@ -115,7 +124,7 @@ class Strategy:
 
     def __init__(self, *, strategy: Optional[str], max_idletime: float):
         """Initialize strategy."""
-        self.executors: Dict[str, ParslExecutor]
+        self.executors: Dict[str, ExecutorState]
         self.executors = {}
         self.max_idletime = max_idletime