Skip to content

Commit

Permalink
restored functionality from specs
Browse files Browse the repository at this point in the history
  • Loading branch information
tclose committed Dec 8, 2024
1 parent 553bb2f commit 032fd4e
Show file tree
Hide file tree
Showing 13 changed files with 629 additions and 1,118 deletions.
39 changes: 26 additions & 13 deletions pydra/design/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,18 @@
ensure_list,
PYDRA_ATTR_METADATA,
list_fields,
is_lazy,
)
from pydra.utils.typing import (
MultiInputObj,
MultiInputFile,
MultiOutputObj,
MultiOutputFile,
)
from pydra.engine.workflow.lazy import LazyField


if ty.TYPE_CHECKING:
from pydra.engine.specs import OutputsSpec
from pydra.engine.specs import TaskSpec, OutSpec
from pydra.engine.core import Task

__all__ = [
Expand Down Expand Up @@ -84,7 +84,9 @@ class Field:
validator=is_type, default=ty.Any, converter=default_if_none(ty.Any)
)
help_string: str = ""
requires: list | None = None
requires: list[str] | list[list[str]] = attrs.field(
factory=list, converter=ensure_list
)
converter: ty.Callable | None = None
validator: ty.Callable | None = None

Expand Down Expand Up @@ -240,6 +242,8 @@ def get_fields(klass, field_type, auto_attribs, helps) -> dict[str, Field]:


def make_task_spec(
spec_type: type["TaskSpec"],
out_type: type["OutSpec"],
task_type: type["Task"],
inputs: dict[str, Arg],
outputs: dict[str, Out],
Expand Down Expand Up @@ -281,14 +285,16 @@ def make_task_spec(

if name is None and klass is not None:
name = klass.__name__
outputs_klass = make_outputs_spec(outputs, outputs_bases, name)
if klass is None or not issubclass(klass, TaskSpec):
outputs_klass = make_outputs_spec(out_type, outputs, outputs_bases, name)
if klass is None or not issubclass(klass, spec_type):
if name is None:
raise ValueError("name must be provided if klass is not")
if klass is not None and issubclass(klass, TaskSpec):
raise ValueError(f"Cannot change type of spec {klass} to {spec_type}")
bases = tuple(bases)
# Ensure that TaskSpec is a base class
if not any(issubclass(b, TaskSpec) for b in bases):
bases = bases + (TaskSpec,)
if not any(issubclass(b, spec_type) for b in bases):
bases = bases + (spec_type,)
# If building from a decorated class (as opposed to dynamically from a function
# or shell-template), add any base classes not already in the bases tuple
if klass is not None:
Expand Down Expand Up @@ -346,8 +352,11 @@ def make_task_spec(


def make_outputs_spec(
outputs: dict[str, Out], bases: ty.Sequence[type], spec_name: str
) -> type["OutputsSpec"]:
spec_type: type["OutSpec"],
outputs: dict[str, Out],
bases: ty.Sequence[type],
spec_name: str,
) -> type["OutSpec"]:
"""Create an outputs specification class and its outputs specification class from the
output fields provided to the decorator/function.
Expand All @@ -368,10 +377,14 @@ def make_outputs_spec(
klass : type
The class created using the attrs package
"""
from pydra.engine.specs import OutputsSpec
from pydra.engine.specs import OutSpec

if not any(issubclass(b, OutputsSpec) for b in bases):
outputs_bases = bases + (OutputsSpec,)
if not any(issubclass(b, spec_type) for b in bases):
if out_spec_bases := [b for b in bases if issubclass(b, OutSpec)]:
raise ValueError(
f"Cannot make {spec_type} output spec from {out_spec_bases} bases"
)
outputs_bases = bases + (spec_type,)
if reserved_names := [n for n in outputs if n in RESERVED_OUTPUT_NAMES]:
raise ValueError(
f"{reserved_names} are reserved and cannot be used for output field names"
Expand Down Expand Up @@ -549,7 +562,7 @@ def make_validator(field: Field, interface_name: str) -> ty.Callable[..., None]
def allowed_values_validator(_, attribute, value):
"""checking if the values is in allowed_values"""
allowed = attribute.metadata[PYDRA_ATTR_METADATA].allowed_values
if value is attrs.NOTHING or isinstance(value, LazyField):
if value is attrs.NOTHING or is_lazy(value):
pass
elif value not in allowed:
raise ValueError(
Expand Down
8 changes: 5 additions & 3 deletions pydra/design/python.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import inspect
import attrs
from pydra.engine.task import FunctionTask
from pydra.engine.specs import TaskSpec
from pydra.engine.specs import PythonSpec, PythonOutSpec
from .base import (
Arg,
Out,
Expand Down Expand Up @@ -87,7 +87,7 @@ def define(
bases: ty.Sequence[type] = (),
outputs_bases: ty.Sequence[type] = (),
auto_attribs: bool = True,
) -> TaskSpec:
) -> PythonSpec:
"""
Create an interface for a function or a class.
Expand All @@ -103,7 +103,7 @@ def define(
Whether to use auto_attribs mode when creating the class.
"""

def make(wrapped: ty.Callable | type) -> TaskSpec:
def make(wrapped: ty.Callable | type) -> PythonSpec:
if inspect.isclass(wrapped):
klass = wrapped
function = klass.function
Expand Down Expand Up @@ -139,6 +139,8 @@ def make(wrapped: ty.Callable | type) -> TaskSpec:
)

interface = make_task_spec(
PythonSpec,
PythonOutSpec,
FunctionTask,
parsed_inputs,
parsed_outputs,
Expand Down
33 changes: 25 additions & 8 deletions pydra/design/shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from fileformats.core import from_mime
from fileformats import generic
from fileformats.core.exceptions import FormatRecognitionError
from pydra.engine.specs import TaskSpec
from pydra.engine.specs import ShellSpec, ShellOutSpec
from .base import (
Arg,
Out,
Expand Down Expand Up @@ -177,9 +177,8 @@ class outarg(Out, arg):
inputs (entire inputs will be passed) or any input field name (a specific input
field will be sent).
path_template: str, optional
If provided, the field is treated also as an output field and it is added to
the output spec. The template can use other fields, e.g. {file1}. Used in order
to create an output specification.
The template used to specify where the output file will be written to can use
other fields, e.g. {file1}. Used in order to create an output specification.
"""

path_template: str | None = attrs.field(default=None)
Expand All @@ -202,7 +201,7 @@ def define(
outputs_bases: ty.Sequence[type] = (),
auto_attribs: bool = True,
name: str | None = None,
) -> TaskSpec:
) -> ShellSpec:
"""Create a task specification for a shell command. Can be used either as a decorator on
the "canonical" dataclass-form of a task specification or as a function that takes a
"shell-command template string" of the form
Expand Down Expand Up @@ -251,13 +250,13 @@ def define(
Returns
-------
TaskSpec
ShellSpec
The interface for the shell command
"""

def make(
wrapped: ty.Callable | type | None = None,
) -> TaskSpec:
) -> ShellSpec:

if inspect.isclass(wrapped):
klass = wrapped
Expand All @@ -272,6 +271,14 @@ def make(
f"Shell task class {wrapped} must have an `executable` "
"attribute that specifies the command to run"
) from None
if not isinstance(executable, str) and not (
isinstance(executable, ty.Sequence)
and all(isinstance(e, str) for e in executable)
):
raise ValueError(
"executable must be a string or a sequence of strings"
f", not {executable!r}"
)
class_name = klass.__name__
check_explicit_fields_are_none(klass, inputs, outputs)
parsed_inputs, parsed_outputs = extract_fields_from_class(
Expand Down Expand Up @@ -309,7 +316,15 @@ def make(
{o.name: o for o in parsed_outputs.values() if isinstance(o, arg)}
)
parsed_inputs["executable"] = arg(
name="executable", type=str, argstr="", position=0, default=executable
name="executable",
type=str | ty.Sequence[str],
argstr="",
position=0,
default=executable,
help_string=(
"the first part of the command, can be a string, "
"e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']"
),
)

# Set positions for the remaining inputs that don't have an explicit position
Expand All @@ -319,6 +334,8 @@ def make(
inpt.position = position_stack.pop(0)

interface = make_task_spec(
ShellSpec,
ShellOutSpec,
ShellCommandTask,
parsed_inputs,
parsed_outputs,
Expand Down
4 changes: 3 additions & 1 deletion pydra/design/tests/test_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,9 @@ def MyTestWorkflow(a: list[int], b: list[float]) -> list[float]:
wf = Workflow.construct(MyTestWorkflow(a=[1, 2, 3], b=[1.0, 10.0, 100.0]))
assert wf["Mul"].splitter == ["Mul.x", "Mul.y"]
assert wf["Mul"].combiner == ["Mul.x"]
assert wf.outputs.out == LazyOutField(node=wf["Sum"], field="out", type=list[float])
assert wf.outputs.out == LazyOutField(
node=wf["Sum"], field="out", type=list[float], type_checked=True
)


def test_workflow_split_combine2():
Expand Down
14 changes: 8 additions & 6 deletions pydra/design/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
check_explicit_fields_are_none,
extract_fields_from_class,
)
from pydra.engine.specs import TaskSpec
from pydra.engine.specs import TaskSpec, OutSpec, WorkflowSpec, WorkflowOutSpec


__all__ = ["define", "add", "this", "arg", "out"]
Expand Down Expand Up @@ -154,6 +154,8 @@ def make(wrapped: ty.Callable | type) -> TaskSpec:
parsed_inputs[inpt_name].lazy = True

interface = make_task_spec(
WorkflowSpec,
WorkflowOutSpec,
WorkflowTask,
parsed_inputs,
parsed_outputs,
Expand All @@ -172,9 +174,6 @@ def make(wrapped: ty.Callable | type) -> TaskSpec:
return make


OutputType = ty.TypeVar("OutputType")


def this() -> Workflow:
"""Get the workflow currently being constructed.
Expand All @@ -186,7 +185,10 @@ def this() -> Workflow:
return Workflow.under_construction


def add(task_spec: TaskSpec[OutputType], name: str = None) -> OutputType:
OutSpecType = ty.TypeVar("OutSpecType", bound=OutSpec)


def add(task_spec: TaskSpec[OutSpecType], name: str = None) -> OutSpecType:
"""Add a node to the workflow currently being constructed
Parameters
Expand All @@ -199,7 +201,7 @@ def add(task_spec: TaskSpec[OutputType], name: str = None) -> OutputType:
Returns
-------
OutputType
OutSpec
The outputs specification of the node
"""
return this().add(task_spec, name=name)
Loading

0 comments on commit 032fd4e

Please sign in to comment.