From 694d874bd439375f35636a492f23d2058a84ce4b Mon Sep 17 00:00:00 2001 From: Andrew Rosen Date: Thu, 12 Oct 2023 15:37:24 -0700 Subject: [PATCH 1/6] Add support for `__getattr__` --- parsl/dataflow/futures.py | 24 +++++++++ parsl/tests/test_python_apps/test_lifted.py | 54 +++++++++++++++++++ .../test_python_apps/test_lifted_dict.py | 23 -------- 3 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 parsl/tests/test_python_apps/test_lifted.py delete mode 100644 parsl/tests/test_python_apps/test_lifted_dict.py diff --git a/parsl/dataflow/futures.py b/parsl/dataflow/futures.py index d12a3a7db7..c2f8fb08ff 100644 --- a/parsl/dataflow/futures.py +++ b/parsl/dataflow/futures.py @@ -139,9 +139,33 @@ def __getitem__(self, key: Any) -> AppFuture: return deferred_getitem_app(self, key) + def __getattr__(self, name: str) -> AppFuture: + # hack around circular imports for python_app + from parsl.app.app import python_app + + # TODO: it would be nice to avoid redecorating this each time, + # which was done to avoid import loops here -- but the DFK + # is not defined at import time, and so this decoration needs + # to happen at least once per DFK. So perhaps for implementation + # simplicity, this redecoration should always happen, as happens + # for example with the globus data provider. + + # TODO: this should be run on the same DFK as is executing the + # task that is associated with this future. That value isn't + # easily available here (although probably the right thing to + # do is add it to self.task_def) + deferred_getattr_app = python_app(deferred_getattr, executors=['_parsl_internal']) + + return deferred_getattr_app(self, name) # this needs python_app to be importable, but three's an import loop # if so... so hack around it for prototyping. # @python_app def deferred_getitem(o: Any, k: Any) -> Any: return o[k] + +# this needs python_app to be importable, but three's an import loop +# if so... so hack around it for prototyping. +# @python_app +def deferred_getattr(o: Any, name: str) -> Any: + return getattr(o, name) \ No newline at end of file diff --git a/parsl/tests/test_python_apps/test_lifted.py b/parsl/tests/test_python_apps/test_lifted.py new file mode 100644 index 0000000000..7061454719 --- /dev/null +++ b/parsl/tests/test_python_apps/test_lifted.py @@ -0,0 +1,54 @@ +from parsl import python_app + + +@python_app +def returns_a_dict(): + return {"a": "X", "b": "Y"} + +@python_app +def returns_a_list(): + return ["X", "Y"] + +@python_app +def returns_a_class(): + from dataclasses import dataclass + + @dataclass + class MyClass: + a: str = "X" + b: str = "Y" + + return MyClass + +def test_returns_a_dict(): + + # precondition that returns_a_dict behaves + # correctly + assert returns_a_dict().result()["a"] == "X" + + # check that the deferred __getitem__ functionality works, + # allowing [] to be used on an AppFuture + assert returns_a_dict()["a"].result() == "X" + +def test_returns_a_list(): + + # precondition that returns_a_list behaves + # correctly + assert returns_a_list().result()[0] == "X" + + # check that the deferred __getitem__ functionality works, + # allowing [] to be used on an AppFuture + assert returns_a_list()[0].result() == "X" + +def test_returns_a_class(): + + # precondition that returns_a_class behaves + # correctly + assert returns_a_class().result().a == "X" + + # check that the deferred __getitem__ functionality works, + # allowing [] to be used on an AppFuture + assert returns_a_class().a.result() == "X" + + # when the result is not indexable, a sensible error should + # appear in the appropriate future diff --git a/parsl/tests/test_python_apps/test_lifted_dict.py b/parsl/tests/test_python_apps/test_lifted_dict.py deleted file mode 100644 index acf3d6389f..0000000000 --- a/parsl/tests/test_python_apps/test_lifted_dict.py +++ /dev/null @@ -1,23 +0,0 @@ -from parsl import python_app - - -@python_app -def returns_a_dict(): - return {"a": "X", "b": "Y"} - - -def test_returns_a_dict(): - - # precondition that returns_a_dict behaves - # correctly - assert returns_a_dict().result()["a"] == "X" - - # check that the deferred __getitem__ functionality works, - # allowing [] to be used on an AppFuture - assert returns_a_dict()["a"].result() == "X" - - # other things to test: when the result is a sequence, so that - # [] is a position - - # when the result is not indexable, a sensible error should - # appear in the appropriate future From 2ee5b08f8c4b44ca99d659e82444a0c7dc2c84a5 Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Thu, 12 Oct 2023 15:39:44 -0700 Subject: [PATCH 2/6] Formatting patch --- parsl/dataflow/futures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsl/dataflow/futures.py b/parsl/dataflow/futures.py index c2f8fb08ff..a111357298 100644 --- a/parsl/dataflow/futures.py +++ b/parsl/dataflow/futures.py @@ -168,4 +168,4 @@ def deferred_getitem(o: Any, k: Any) -> Any: # if so... so hack around it for prototyping. # @python_app def deferred_getattr(o: Any, name: str) -> Any: - return getattr(o, name) \ No newline at end of file + return getattr(o, name) From f190653b9c763efeaf728fb7dc6821039d965489 Mon Sep 17 00:00:00 2001 From: Andrew Rosen Date: Thu, 12 Oct 2023 15:44:16 -0700 Subject: [PATCH 3/6] E302 lint fix --- parsl/dataflow/futures.py | 2 ++ parsl/tests/test_python_apps/test_lifted.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/parsl/dataflow/futures.py b/parsl/dataflow/futures.py index a111357298..9dbbf6237b 100644 --- a/parsl/dataflow/futures.py +++ b/parsl/dataflow/futures.py @@ -158,12 +158,14 @@ def __getattr__(self, name: str) -> AppFuture: return deferred_getattr_app(self, name) + # this needs python_app to be importable, but three's an import loop # if so... so hack around it for prototyping. # @python_app def deferred_getitem(o: Any, k: Any) -> Any: return o[k] + # this needs python_app to be importable, but three's an import loop # if so... so hack around it for prototyping. # @python_app diff --git a/parsl/tests/test_python_apps/test_lifted.py b/parsl/tests/test_python_apps/test_lifted.py index 7061454719..e88a2cb555 100644 --- a/parsl/tests/test_python_apps/test_lifted.py +++ b/parsl/tests/test_python_apps/test_lifted.py @@ -5,10 +5,12 @@ def returns_a_dict(): return {"a": "X", "b": "Y"} + @python_app def returns_a_list(): return ["X", "Y"] + @python_app def returns_a_class(): from dataclasses import dataclass @@ -20,6 +22,7 @@ class MyClass: return MyClass + def test_returns_a_dict(): # precondition that returns_a_dict behaves @@ -30,6 +33,7 @@ def test_returns_a_dict(): # allowing [] to be used on an AppFuture assert returns_a_dict()["a"].result() == "X" + def test_returns_a_list(): # precondition that returns_a_list behaves @@ -40,6 +44,7 @@ def test_returns_a_list(): # allowing [] to be used on an AppFuture assert returns_a_list()[0].result() == "X" + def test_returns_a_class(): # precondition that returns_a_class behaves From d215f7f37fc243df9cb451065df548cffda72db8 Mon Sep 17 00:00:00 2001 From: Andrew Rosen Date: Thu, 12 Oct 2023 15:47:51 -0700 Subject: [PATCH 4/6] Add a test for a set --- parsl/tests/test_python_apps/test_lifted.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/parsl/tests/test_python_apps/test_lifted.py b/parsl/tests/test_python_apps/test_lifted.py index e88a2cb555..2dc1c06a50 100644 --- a/parsl/tests/test_python_apps/test_lifted.py +++ b/parsl/tests/test_python_apps/test_lifted.py @@ -11,6 +11,11 @@ def returns_a_list(): return ["X", "Y"] +@python_app +def returns_a_set(): + return ("X", "Y") + + @python_app def returns_a_class(): from dataclasses import dataclass @@ -45,6 +50,17 @@ def test_returns_a_list(): assert returns_a_list()[0].result() == "X" +def test_returns_a_set(): + + # precondition that returns_a_set behaves + # correctly + assert returns_a_set().result()[0] == "X" + + # check that the deferred __getitem__ functionality works, + # allowing [] to be used on an AppFuture + assert returns_a_set()[0].result() == "X" + + def test_returns_a_class(): # precondition that returns_a_class behaves From ee6266189bbc133cb3c3e67fe497c4dc7fb7bd62 Mon Sep 17 00:00:00 2001 From: Andrew Rosen Date: Thu, 12 Oct 2023 15:48:37 -0700 Subject: [PATCH 5/6] Rename test --- parsl/tests/test_python_apps/test_lifted.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/parsl/tests/test_python_apps/test_lifted.py b/parsl/tests/test_python_apps/test_lifted.py index 2dc1c06a50..a544cacd31 100644 --- a/parsl/tests/test_python_apps/test_lifted.py +++ b/parsl/tests/test_python_apps/test_lifted.py @@ -12,7 +12,7 @@ def returns_a_list(): @python_app -def returns_a_set(): +def returns_a_tuple(): return ("X", "Y") @@ -50,15 +50,15 @@ def test_returns_a_list(): assert returns_a_list()[0].result() == "X" -def test_returns_a_set(): +def test_returns_a_tuple(): - # precondition that returns_a_set behaves + # precondition that returns_a_tuple behaves # correctly - assert returns_a_set().result()[0] == "X" + assert returns_a_tuple().result()[0] == "X" # check that the deferred __getitem__ functionality works, # allowing [] to be used on an AppFuture - assert returns_a_set()[0].result() == "X" + assert returns_a_tuple()[0].result() == "X" def test_returns_a_class(): From 4557822df3eba63e8e0ff3717a32a4615e9708f6 Mon Sep 17 00:00:00 2001 From: Andrew Rosen Date: Thu, 12 Oct 2023 15:49:10 -0700 Subject: [PATCH 6/6] Fix code comment --- parsl/tests/test_python_apps/test_lifted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsl/tests/test_python_apps/test_lifted.py b/parsl/tests/test_python_apps/test_lifted.py index a544cacd31..d83c48815c 100644 --- a/parsl/tests/test_python_apps/test_lifted.py +++ b/parsl/tests/test_python_apps/test_lifted.py @@ -67,7 +67,7 @@ def test_returns_a_class(): # correctly assert returns_a_class().result().a == "X" - # check that the deferred __getitem__ functionality works, + # check that the deferred __getattr__ functionality works, # allowing [] to be used on an AppFuture assert returns_a_class().a.result() == "X"