diff --git a/parsl/dataflow/dflow.py b/parsl/dataflow/dflow.py index 707e2abc23..f45ac37629 100644 --- a/parsl/dataflow/dflow.py +++ b/parsl/dataflow/dflow.py @@ -730,7 +730,8 @@ def launch_task(self, task_record: TaskRecord) -> Future: return exec_fu - def _add_input_deps(self, executor: str, args: Sequence[Any], kwargs: Dict[str, Any], func: Callable) -> Tuple[Sequence[Any], Dict[str, Any], Callable]: + def _add_input_deps(self, executor: str, args: Sequence[Any], kwargs: Dict[str, Any], func: Callable) -> Tuple[Sequence[Any], Dict[str, Any], + Callable]: """Look for inputs of the app that are files. Give the data manager the opportunity to replace a file with a data future for that file, for example wrapping the result of a staging action. diff --git a/parsl/dataflow/memoization.py b/parsl/dataflow/memoization.py index e4d657ccaa..2f2126ecba 100644 --- a/parsl/dataflow/memoization.py +++ b/parsl/dataflow/memoization.py @@ -5,7 +5,7 @@ import pickle from parsl.dataflow.taskrecord import TaskRecord -from typing import Dict, Any, List, Optional, TYPE_CHECKING +from typing import Dict, Any, List, Optional, TYPE_CHECKING # avoid circular imports if TYPE_CHECKING: from parsl import DataFlowKernel # import loop at runtime - needed for typechecking - TODO turn into "if typing:" @@ -14,7 +14,9 @@ import types -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) # logger named name for logging purposes + +# memoization function with a single dispatch decorator @singledispatch @@ -49,6 +51,8 @@ def id_for_memo(obj: object, output_ref: bool = False) -> bytes: logger.error("id_for_memo attempted on unknown type {}".format(type(obj))) raise ValueError("unknown type for memoization: {}".format(type(obj))) +# type specific implementations - handle how each type should be serialized for memoization + @id_for_memo.register(str) @id_for_memo.register(int) @@ -94,10 +98,13 @@ def id_for_memo_dict(denormalized_dict: dict, output_ref: bool = False) -> bytes if type(denormalized_dict) is not dict: raise ValueError("id_for_memo_dict cannot work on subclasses of dict") - keys = sorted(denormalized_dict) + # keys = sorted(denormalized_dict) Line that sirosen commented on + # Proposed solution was to normalize the keys and then sort them + keymap = {id_for_memo(k): k for k in denormalized_dict} + normed_keys = sorted(keymap.values()) normalized_list = [] - for k in keys: + for k in normed_keys: normalized_list.append(id_for_memo(k)) normalized_list.append(id_for_memo(denormalized_dict[k], output_ref=output_ref)) return pickle.dumps(normalized_list) diff --git a/parsl/tests/test_python_apps/test_memoize_3.py b/parsl/tests/test_python_apps/test_memoize_3.py new file mode 100644 index 0000000000..4047189853 --- /dev/null +++ b/parsl/tests/test_python_apps/test_memoize_3.py @@ -0,0 +1,32 @@ +import pytest +import enum + +# Define an enum - collection of related consonants + + +class Foo(enum.Enum): + x = enum.auto() + y = enum.auto() + + +# Test function demonstrating the issue with unstable sorting when keys +# are hashable but not comparable. + + +def test_unstable_sorting(): + # Functions + def foo(): + return 1 + + def bar(): + return 2 + + # Dictionary with problematic keys + d = {foo: 1, bar: 2} + + # Sort the dictionary, it should raise a TypeError + with pytest.raises(TypeError): + sorted(d) + + # Create a dictionary with enum keys + d_enum = {Foo.x: 1, Foo.y: 2} diff --git a/runinfo/000/checkpoint/dfk.pkl b/runinfo/000/checkpoint/dfk.pkl new file mode 100644 index 0000000000..42edab3d84 Binary files /dev/null and b/runinfo/000/checkpoint/dfk.pkl differ diff --git a/runinfo/000/checkpoint/tasks.pkl b/runinfo/000/checkpoint/tasks.pkl new file mode 100644 index 0000000000..43e10cac9d Binary files /dev/null and b/runinfo/000/checkpoint/tasks.pkl differ diff --git a/runinfo/001/checkpoint/dfk.pkl b/runinfo/001/checkpoint/dfk.pkl new file mode 100644 index 0000000000..3b662b2a7c Binary files /dev/null and b/runinfo/001/checkpoint/dfk.pkl differ diff --git a/runinfo/001/checkpoint/tasks.pkl b/runinfo/001/checkpoint/tasks.pkl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/runinfo/002/checkpoint/dfk.pkl b/runinfo/002/checkpoint/dfk.pkl new file mode 100644 index 0000000000..b7795f1404 Binary files /dev/null and b/runinfo/002/checkpoint/dfk.pkl differ diff --git a/runinfo/002/checkpoint/tasks.pkl b/runinfo/002/checkpoint/tasks.pkl new file mode 100644 index 0000000000..7b870b571e Binary files /dev/null and b/runinfo/002/checkpoint/tasks.pkl differ diff --git a/runinfo/003/checkpoint/dfk.pkl b/runinfo/003/checkpoint/dfk.pkl new file mode 100644 index 0000000000..82262c5168 Binary files /dev/null and b/runinfo/003/checkpoint/dfk.pkl differ diff --git a/runinfo/003/checkpoint/tasks.pkl b/runinfo/003/checkpoint/tasks.pkl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/runinfo/004/checkpoint/dfk.pkl b/runinfo/004/checkpoint/dfk.pkl new file mode 100644 index 0000000000..bf23f7ba14 Binary files /dev/null and b/runinfo/004/checkpoint/dfk.pkl differ diff --git a/runinfo/004/checkpoint/tasks.pkl b/runinfo/004/checkpoint/tasks.pkl new file mode 100644 index 0000000000..6d31d3eac9 Binary files /dev/null and b/runinfo/004/checkpoint/tasks.pkl differ diff --git a/runinfo/005/checkpoint/dfk.pkl b/runinfo/005/checkpoint/dfk.pkl new file mode 100644 index 0000000000..4a1d3a2945 Binary files /dev/null and b/runinfo/005/checkpoint/dfk.pkl differ diff --git a/runinfo/005/checkpoint/tasks.pkl b/runinfo/005/checkpoint/tasks.pkl new file mode 100644 index 0000000000..6d31d3eac9 Binary files /dev/null and b/runinfo/005/checkpoint/tasks.pkl differ diff --git a/runinfo/006/checkpoint/dfk.pkl b/runinfo/006/checkpoint/dfk.pkl new file mode 100644 index 0000000000..874a78154e Binary files /dev/null and b/runinfo/006/checkpoint/dfk.pkl differ diff --git a/runinfo/006/checkpoint/tasks.pkl b/runinfo/006/checkpoint/tasks.pkl new file mode 100644 index 0000000000..71fbd07ddd Binary files /dev/null and b/runinfo/006/checkpoint/tasks.pkl differ diff --git a/runinfo/007/checkpoint/dfk.pkl b/runinfo/007/checkpoint/dfk.pkl new file mode 100644 index 0000000000..8236f8554a Binary files /dev/null and b/runinfo/007/checkpoint/dfk.pkl differ diff --git a/runinfo/007/checkpoint/tasks.pkl b/runinfo/007/checkpoint/tasks.pkl new file mode 100644 index 0000000000..43e10cac9d Binary files /dev/null and b/runinfo/007/checkpoint/tasks.pkl differ diff --git a/runinfo/008/checkpoint/dfk.pkl b/runinfo/008/checkpoint/dfk.pkl new file mode 100644 index 0000000000..2914824d73 Binary files /dev/null and b/runinfo/008/checkpoint/dfk.pkl differ diff --git a/runinfo/008/checkpoint/tasks.pkl b/runinfo/008/checkpoint/tasks.pkl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/runinfo/009/checkpoint/dfk.pkl b/runinfo/009/checkpoint/dfk.pkl new file mode 100644 index 0000000000..316ee2c87b Binary files /dev/null and b/runinfo/009/checkpoint/dfk.pkl differ diff --git a/runinfo/009/checkpoint/tasks.pkl b/runinfo/009/checkpoint/tasks.pkl new file mode 100644 index 0000000000..44f968dcff Binary files /dev/null and b/runinfo/009/checkpoint/tasks.pkl differ diff --git a/runinfo/010/checkpoint/dfk.pkl b/runinfo/010/checkpoint/dfk.pkl new file mode 100644 index 0000000000..85fbf53152 Binary files /dev/null and b/runinfo/010/checkpoint/dfk.pkl differ diff --git a/runinfo/010/checkpoint/tasks.pkl b/runinfo/010/checkpoint/tasks.pkl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/runinfo/011/checkpoint/dfk.pkl b/runinfo/011/checkpoint/dfk.pkl new file mode 100644 index 0000000000..ac39ce6997 Binary files /dev/null and b/runinfo/011/checkpoint/dfk.pkl differ diff --git a/runinfo/011/checkpoint/tasks.pkl b/runinfo/011/checkpoint/tasks.pkl new file mode 100644 index 0000000000..900a1d7324 Binary files /dev/null and b/runinfo/011/checkpoint/tasks.pkl differ diff --git a/runinfo/012/checkpoint/dfk.pkl b/runinfo/012/checkpoint/dfk.pkl new file mode 100644 index 0000000000..6a2fb46970 Binary files /dev/null and b/runinfo/012/checkpoint/dfk.pkl differ diff --git a/runinfo/012/checkpoint/tasks.pkl b/runinfo/012/checkpoint/tasks.pkl new file mode 100644 index 0000000000..900a1d7324 Binary files /dev/null and b/runinfo/012/checkpoint/tasks.pkl differ diff --git a/runinfo/013/checkpoint/dfk.pkl b/runinfo/013/checkpoint/dfk.pkl new file mode 100644 index 0000000000..0dbeb2acde Binary files /dev/null and b/runinfo/013/checkpoint/dfk.pkl differ diff --git a/runinfo/013/checkpoint/tasks.pkl b/runinfo/013/checkpoint/tasks.pkl new file mode 100644 index 0000000000..71fbd07ddd Binary files /dev/null and b/runinfo/013/checkpoint/tasks.pkl differ diff --git a/runinfo/014/checkpoint/dfk.pkl b/runinfo/014/checkpoint/dfk.pkl new file mode 100644 index 0000000000..61cda9a00c Binary files /dev/null and b/runinfo/014/checkpoint/dfk.pkl differ diff --git a/runinfo/014/checkpoint/tasks.pkl b/runinfo/014/checkpoint/tasks.pkl new file mode 100644 index 0000000000..43e10cac9d Binary files /dev/null and b/runinfo/014/checkpoint/tasks.pkl differ diff --git a/runinfo/015/checkpoint/dfk.pkl b/runinfo/015/checkpoint/dfk.pkl new file mode 100644 index 0000000000..be417e7a29 Binary files /dev/null and b/runinfo/015/checkpoint/dfk.pkl differ diff --git a/runinfo/015/checkpoint/tasks.pkl b/runinfo/015/checkpoint/tasks.pkl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/runinfo/016/checkpoint/dfk.pkl b/runinfo/016/checkpoint/dfk.pkl new file mode 100644 index 0000000000..235a8b3f11 Binary files /dev/null and b/runinfo/016/checkpoint/dfk.pkl differ diff --git a/runinfo/016/checkpoint/tasks.pkl b/runinfo/016/checkpoint/tasks.pkl new file mode 100644 index 0000000000..2a6331874d Binary files /dev/null and b/runinfo/016/checkpoint/tasks.pkl differ diff --git a/runinfo/017/checkpoint/dfk.pkl b/runinfo/017/checkpoint/dfk.pkl new file mode 100644 index 0000000000..dc3c30d19d Binary files /dev/null and b/runinfo/017/checkpoint/dfk.pkl differ diff --git a/runinfo/017/checkpoint/tasks.pkl b/runinfo/017/checkpoint/tasks.pkl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/runinfo/018/checkpoint/dfk.pkl b/runinfo/018/checkpoint/dfk.pkl new file mode 100644 index 0000000000..ff38200ea6 Binary files /dev/null and b/runinfo/018/checkpoint/dfk.pkl differ diff --git a/runinfo/018/checkpoint/tasks.pkl b/runinfo/018/checkpoint/tasks.pkl new file mode 100644 index 0000000000..f13c7af35c Binary files /dev/null and b/runinfo/018/checkpoint/tasks.pkl differ diff --git a/runinfo/019/checkpoint/dfk.pkl b/runinfo/019/checkpoint/dfk.pkl new file mode 100644 index 0000000000..33601045bc Binary files /dev/null and b/runinfo/019/checkpoint/dfk.pkl differ diff --git a/runinfo/019/checkpoint/tasks.pkl b/runinfo/019/checkpoint/tasks.pkl new file mode 100644 index 0000000000..f13c7af35c Binary files /dev/null and b/runinfo/019/checkpoint/tasks.pkl differ diff --git a/runinfo/020/checkpoint/dfk.pkl b/runinfo/020/checkpoint/dfk.pkl new file mode 100644 index 0000000000..eb770c8ed3 Binary files /dev/null and b/runinfo/020/checkpoint/dfk.pkl differ diff --git a/runinfo/020/checkpoint/tasks.pkl b/runinfo/020/checkpoint/tasks.pkl new file mode 100644 index 0000000000..5ea3527e19 Binary files /dev/null and b/runinfo/020/checkpoint/tasks.pkl differ