Squash benc-mypy history

.github/workflows/ci.yaml

@@ -10,7 +10,7 @@ jobs:
   main-test-suite:
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.10"]
     runs-on: ubuntu-20.04
     timeout-minutes: 30
 

Makefile

@@ -46,11 +46,10 @@ clean_coverage:
 
 .PHONY: mypy
 mypy: ## run mypy checks
-	MYPYPATH=$(CWD)/mypy-stubs mypy parsl/
-	# process worker pool is explicitly listed to check, because it is not
-	# imported from anywhere in core parsl python code.
-	# so anything explicitly invoked (rather than imported) might need listing here
-	MYPYPATH=$(CWD)/mypy-stubs mypy parsl/executors/high_throughput/process_worker_pool.py parsl/executors/high_throughput/interchange.py
+	mypy --version
+	which mypy
+	echo $(PYTHONPATH)
+	PYTHONPATH=$(CWD)/mypy-plugins:$(PYTHONPATH) MYPYPATH=$(CWD)/mypy-stubs mypy --no-incremental parsl/
 
 .PHONY: local_thread_test
 local_thread_test: ## run all tests with local_thread config
@@ -65,11 +64,12 @@ htex_local_alternate_test: ## run all tests with htex_local config
 	pip3 install ".[monitoring]"
 	PYTHONPATH=.  pytest parsl/tests/ -k "not cleannet" --config parsl/tests/configs/htex_local_alternate.py --random-order
 
-$(WORKQUEUE_INSTALL):
+$(WORKQUEUE_INSTALL)/installed.parsl:
 	parsl/executors/workqueue/install-workqueue.sh
+	touch $(WORKQUEUE_INSTALL)/installed.parsl
 
 .PHONY: workqueue_ex_test
-workqueue_ex_test: $(WORKQUEUE_INSTALL)  ## run all tests with workqueue_ex config
+workqueue_ex_test: $(WORKQUEUE_INSTALL)/installed.parsl  ## run all tests with workqueue_ex config
 	PYTHONPATH=.:/tmp/cctools/lib/python3.8/site-packages  pytest parsl/tests/ -k "not cleannet and not issue363" --config parsl/tests/configs/workqueue_ex.py --random-order
 
 .PHONY: config_local_test

docs/reference.rst

@@ -78,7 +78,6 @@ Executors
     parsl.executors.WorkQueueExecutor
     parsl.executors.ExtremeScaleExecutor
     parsl.executors.FluxExecutor
-    parsl.executors.swift_t.TurbineExecutor
 
 Launchers
 =========

mypy-plugins/parsl_mypy.py

@@ -0,0 +1,22 @@
+from mypy.plugin import FunctionContext, Plugin
+from mypy.types import Type
+
+def plugin(v):
+    return ParslMypyPlugin
+
+class ParslMypyPlugin(Plugin):
+    def get_type_analyze_hook(self, t):
+        # print("BENC: gtah t={}".format(t))
+        return None
+
+    def get_function_hook(self, f):
+        if f == "parsl.app.errors.wrap_error":
+            return wrap_error_hook
+        else:
+            return None
+
+def wrap_error_hook(ctx: FunctionContext) -> Type:
+    print("inside wrap_error_hook for parsl-mypy")
+    print("ctx = {}".format(ctx))
+    print("ctx.default_return_type = {}".format(ctx.default_return_type))
+    return ctx.default_return_type

mypy.ini

@@ -1,18 +1,22 @@
 [mypy]
-plugins = sqlmypy
+plugins = sqlmypy, parsl_mypy
+enable_error_code = ignore-without-code
 
 # globally disabled error codes:
 #   str-bytes-safe warns that a byte string is formatted into a string.
 #                  which is commonly done with manager IDs in the parsl
 #                  codebase.
-disable_error_code = str-bytes-safe
+# disable_error_code = str-bytes-safe
 no_implicit_reexport = True
 warn_redundant_casts = True
 
 no_implicit_optional = True
 
 strict_equality = True
 
+# there are some exceptions to this even in the more-strongly-typed sections
+warn_unreachable = True
+
 [mypy-non_existent.*]
 ignore_missing_imports = True
 
@@ -21,15 +25,141 @@ ignore_missing_imports = True
 # parsl to be checked that way, so these options could be default instead of
 # listed per package; but unless/until that happens, the more-strictly-checked
 # code should be listed here:
+
 [mypy-parsl.providers.base.*]
-no_implicit_optional = True
+disallow_untyped_defs = True
 disallow_untyped_decorators = True
 warn_unused_ignores = True
 check_untyped_defs = True
 disallow_subclassing_any = True
-warn_unreachable = True
+
+
+# modules to be checked mostly more strongly than default:
+
+[mypy-parsl.addresses.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.dflow.*]
+disallow_untyped_defs = True
+
+warn_unreachable = False
+# this is because of some tangle of types
+# to do with channel script dirs: script dir
+# can be none at the start before initialisation,
+# but must be set later on, and I can't
+# represent that session type in mypy.
+# (or can I?!)
+
+[mypy-parsl.dataflow.error.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.executor_status.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.flow_control.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.futures.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.memoization.*]
+disallow_untyped_defs = True
+
+[mypy-parsl.dataflow.rundirs.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.strategy.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.states.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.taskrecord.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+disallow_any_decorated = True
+
+[mypy-parsl.dataflow.task_status_poller.*]
+disallow_untyped_defs = True
+
+# merge of #1877 introduced stuff that violates this so disabling pending perhaps further investigation
+# disallow_any_expr = True
+
+disallow_any_decorated = True
+
+[mypy-parsl.config.*]
+disallow_untyped_defs = True
+# Any has to be allowed because TaskRecord now forms part of the type signature of config,
+# and task record has Any from the type of tasks args
+#disallow_any_expr = True
+#disallow_any_decorated = True
+
+[mypy-parsl.channels.base.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+
+[mypy-parsl.channels.ssh.*]
+disallow_untyped_defs = True
+
+[mypy-parsl.launchers.*]
+disallow_untyped_defs = True
+disallow_any_decorated = True
+
+
+
+[mypy-parsl.executors.base.*]
+disallow_untyped_defs = True
+disallow_any_expr = True
+[mypy-parsl.serialize.*]
 disallow_untyped_defs = True
 
+
+# modules to be checked more weakly than default:
+
+[mypy-parsl.executors.flux.*]
+ignore_errors = True
+
+[mypy-parsl.executors.extreme_scale.*]
+ignore_errors = True
+
+
+[mypy-parsl.executors.low_latency.*]
+check_untyped_defs = False
+
+
+[mypy-parsl.providers.aws.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.pbspro.pbspro.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.lsf.lsf.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.torque.torque.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.grid_engine.grid_engine.*]
+check_untyped_defs = False
+
+[mypy-parsl.providers.googlecloud.*]
+check_untyped_defs = False
+
+[mypy-parsl.monitoring.db_manager.*]
+check_untyped_defs = False
+
 [mypy-parsl.executors.high_throughput.interchange.*]
 check_untyped_defs = True
 
@@ -40,12 +170,10 @@ check_untyped_defs = True
 ignore_errors = True
 
 [mypy-parsl.monitoring.*]
-no_implicit_optional = True
 disallow_untyped_decorators = True
 warn_unused_ignores = True
 check_untyped_defs = True
 disallow_subclassing_any = True
-warn_unreachable = True
 disallow_untyped_defs = True
 
 # visualization typechecks much less well than the rest of monitoring,
@@ -56,6 +184,9 @@ ignore_errors = True
 [mypy-parsl.tests.configs.local_user_opts]
 ignore_missing_imports = True
 
+
+# imports from elsewhere that there are no stubs for:
+
 [mypy-flask_sqlalchemy.*]
 ignore_missing_imports = True
 
@@ -119,25 +250,6 @@ ignore_missing_imports = True
 [mypy-zmq.*]
 ignore_missing_imports = True
 
-[mypy-mpi4py.*]
-ignore_missing_imports = True
-
-[mypy-flask.*]
-ignore_missing_imports = True
-
-# this is an internal undocumentated package
-# of multiprocessing - trying to get Event
-# to typecheck in monitoring, but it's not
-# a top level class as far as mypy is concerned.
-# but... when commented out seems ok?
-# so lets see when happens when I try to merge
-# in clean CI
-#[mypy-multiprocessing.synchronization.*]
-#ignore_missing_imports = True
-
-[mypy-pandas.*]
-ignore_missing_imports = True
-
 [mypy-work_queue.*]
 ignore_missing_imports = True
 
@@ -149,3 +261,6 @@ ignore_missing_imports = True
 
 [mypy-setproctitle.*]
 ignore_missing_imports = True
+
+[mypy-pandas.*]
+ignore_missing_imports = True

parsl/addresses.py

@@ -13,7 +13,7 @@
 try:
     import fcntl
 except ImportError:
-    fcntl = None  # type: ignore
+    fcntl = None  # type: ignore[assignment]
 import struct
 import typeguard
 import psutil

parsl/app/app.py

@@ -11,6 +11,16 @@
 
 from parsl.dataflow.dflow import DataFlowKernel
 
+from typing import TYPE_CHECKING
+from typing import Callable
+
+if TYPE_CHECKING:
+    from typing import Dict
+    from typing import Any
+
+from parsl.dataflow.futures import AppFuture
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -22,7 +32,12 @@ class AppBase(metaclass=ABCMeta):
 
     """
 
-    def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ignore_for_cache=None):
+    @typeguard.typechecked
+    def __init__(self, func: Callable,
+                 data_flow_kernel: Optional[DataFlowKernel] = None,
+                 executors: Union[List[str], Literal['all']] = 'all',
+                 cache: bool = False,
+                 ignore_for_cache=None) -> None:
         """Construct the App object.
 
         Args:
@@ -45,13 +60,15 @@ def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ig
         self.executors = executors
         self.cache = cache
         self.ignore_for_cache = ignore_for_cache
-        if not (isinstance(executors, list) or isinstance(executors, str)):
-            logger.error("App {} specifies invalid executor option, expects string or list".format(
-                func.__name__))
+
+        # unreachable if properly typechecked
+        # if not (isinstance(executors, list) or isinstance(executors, str)):
+        #    logger.error("App {} specifies invalid executor option, expects string or list".format(
+        #        func.__name__))
 
         params = signature(func).parameters
 
-        self.kwargs = {}
+        self.kwargs = {}  # type: Dict[str, Any]
         if 'stdout' in params:
             self.kwargs['stdout'] = params['stdout'].default
         if 'stderr' in params:
@@ -64,7 +81,7 @@ def __init__(self, func, data_flow_kernel=None, executors='all', cache=False, ig
         self.inputs = params['inputs'].default if 'inputs' in params else []
 
     @abstractmethod
-    def __call__(self, *args, **kwargs):
+    def __call__(self, *args, **kwargs) -> AppFuture:
         pass
 
 
@@ -74,7 +91,7 @@ def python_app(function=None,
                cache: bool = False,
                executors: Union[List[str], Literal['all']] = 'all',
                ignore_for_cache: Optional[List[str]] = None,
-               join: bool = False):
+               join: bool = False) -> Callable:
     """Decorator function for making python apps.
 
     Parameters
@@ -116,7 +133,7 @@ def wrapper(f):
 def join_app(function=None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[List[str]] = None) -> Callable:
     return python_app(function=function,
                       data_flow_kernel=data_flow_kernel,
                       cache=cache,
@@ -130,7 +147,7 @@ def bash_app(function=None,
              data_flow_kernel: Optional[DataFlowKernel] = None,
              cache: bool = False,
              executors: Union[List[str], Literal['all']] = 'all',
-             ignore_for_cache: Optional[List[str]] = None):
+             ignore_for_cache: Optional[List[str]] = None) -> Callable:
     """Decorator function for making bash apps.
 
     Parameters