stdout/stderr outputs; styxdefs 0.4

.pre-commit-config.yaml

@@ -1,20 +1,20 @@
 fail_fast: false
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.8
+  rev: v0.8.0
   hooks:
   - id: ruff
     args:
     - --fix
   - id: ruff-format
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.7.1
+  rev: v1.13.0
   hooks:
   - id: mypy
     args:
     - --ignore-missing-imports
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.11.0
+  rev: v2.14.0
   hooks:
   - id: pretty-format-yaml
     args:
@@ -27,7 +27,7 @@ repos:
     - --indent=2
     - --no-sort
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.5.0
+  rev: v5.0.0
   hooks:
   - id: check-case-conflict
   - id: end-of-file-fixer

pyproject.toml

@@ -9,14 +9,14 @@ packages = [{include = "styx", from = "src"}]
 
 [tool.poetry.dependencies]
 python = "^3.11"
-styxdefs = "^0.3.0"
+styxdefs = "^0.4.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^8.2.0"
-mypy = "^1.10.0"
+pytest = "^8.2.2"
+mypy = "^1.10.1"
 pre-commit = "^4.0.1"
-pytest-cov = "^5.0.0"
-ruff = "^0.6.9"
+pytest-cov = "^6.0.0"
+ruff = "^0.8.0"
 
 [tool.poetry.group.docs.dependencies]
 pdoc = "^15.0.0"
@@ -49,8 +49,6 @@ target-version = "py311"
 [tool.ruff.lint]
 select = ["ANN", "D", "E", "F", "I"]
 ignore = [
-  "ANN101",  # self should not be annotated.
-  "ANN102",  # cls should not be annotated.
   "D100",  # Missing docstring in public module.
   "D101",  # Missing docstring in public class.
   "D102",  # Missing docstring in public method.

src/styx/backend/generic/gen/constraints.py

@@ -58,7 +58,7 @@ def _param_compile_constraint_checks(
         else:
             # Case: X <= len(list[]) <= Y
             buf.extend([
-                f"if {val_opt}not ({list_count_min} <= " f"len({py_symbol}) <= {list_count_max}): ",
+                f"if {val_opt}not ({list_count_min} <= len({py_symbol}) <= {list_count_max}): ",
                 *indent(
                     _generate_raise_value_err(
                         f"Length of '{py_symbol}'",
@@ -100,8 +100,7 @@ def _param_compile_constraint_checks(
         assert min_value <= max_value
         if param.list_:
             buf.extend([
-                f"if {val_opt}not ({min_value} {op_min} min({py_symbol}) "
-                f"and max({py_symbol}) {op_max} {max_value}): ",
+                f"if {val_opt}not ({min_value} {op_min} min({py_symbol}) and max({py_symbol}) {op_max} {max_value}): ",
                 *indent(
                     _generate_raise_value_err(
                         f"All elements of '{py_symbol}'",

src/styx/backend/generic/gen/interface.py

@@ -17,6 +17,8 @@ def _compile_struct(
     lookup: LookupParam,
     metadata_symbol: str,
     root_function: bool,
+    stdout_as_string_output: ir.StdOutErrAsStringOutput | None = None,
+    stderr_as_string_output: ir.StdOutErrAsStringOutput | None = None,
 ) -> None:
     has_outputs = root_function or struct_has_outputs(struct)
 
@@ -27,7 +29,7 @@ def _compile_struct(
         func_cargs_building = GenericFunc(
             name=lookup.py_type[struct.base.id_],
             return_type=outputs_type,
-            return_descr=f"NamedTuple of outputs " f"(described in `{outputs_type}`).",
+            return_descr=f"NamedTuple of outputs (described in `{outputs_type}`).",
             docstring_body=docs_to_docstring(struct.base.docs),
         )
         pyargs = func_cargs_building.args
@@ -54,7 +56,7 @@ def _compile_struct(
                 name="outputs",
                 docstring_body="Collect output file paths.",
                 return_type=outputs_type,
-                return_descr=f"NamedTuple of outputs " f"(described in `{outputs_type}`).",
+                return_descr=f"NamedTuple of outputs (described in `{outputs_type}`).",
                 args=[
                     GenericArg(name="self", type=None, default=None, docstring="The sub-command object."),
                     GenericArg(
@@ -104,6 +106,8 @@ def _compile_struct(
             struct=struct,
             interface_module=interface_module,
             lookup=lookup,
+            stdout_as_string_output=stdout_as_string_output,
+            stderr_as_string_output=stderr_as_string_output,
         )
 
     if root_function:
@@ -129,8 +133,22 @@ def _compile_struct(
             func=func_cargs_building,
             lookup=lookup,
             access_via_self=False,
+            stderr_as_string_output=stderr_as_string_output,
+            stdout_as_string_output=stdout_as_string_output,
         )
-        func_cargs_building.body.extend([*lang.execution_run("execution", "cargs"), lang.return_statement("ret")])
+        func_cargs_building.body.extend([
+            *lang.execution_run(
+                execution_symbol="execution",
+                cargs_symbol="cargs",
+                stdout_output_symbol=lookup.py_output_field_symbol[stdout_as_string_output.id_]
+                if stdout_as_string_output
+                else None,
+                stderr_output_symbol=lookup.py_output_field_symbol[stderr_as_string_output.id_]
+                if stderr_as_string_output
+                else None,
+            ),
+            lang.return_statement("ret"),
+        ])
         interface_module.funcs_and_classes.append(func_cargs_building)
     else:
         if has_outputs:
@@ -235,6 +253,8 @@ def _compile_outputs_class(
     struct: ir.Param[ir.Param.Struct],
     interface_module: GenericModule,
     lookup: LookupParam,
+    stdout_as_string_output: ir.StdOutErrAsStringOutput | None = None,
+    stderr_as_string_output: ir.StdOutErrAsStringOutput | None = None,
 ) -> None:
     outputs_class: GenericNamedTuple = GenericNamedTuple(
         name=lookup.py_output_type[struct.base.id_],
@@ -249,6 +269,18 @@ def _compile_outputs_class(
         )
     )
 
+    for stdout_stderr_output in (stdout_as_string_output, stderr_as_string_output):
+        if stdout_stderr_output is None:
+            continue
+        outputs_class.fields.append(
+            GenericArg(
+                name=lookup.py_output_field_symbol[stdout_stderr_output.id_],
+                type=lang.type_string_list(),
+                default=None,
+                docstring=stdout_stderr_output.docs.description,
+            )
+        )
+
     for output in struct.base.outputs:
         output_symbol = lookup.py_output_field_symbol[output.id_]
 
@@ -342,6 +374,8 @@ def _compile_outputs_building(
     func: GenericFunc,
     lookup: LookupParam,
     access_via_self: bool = False,
+    stdout_as_string_output: ir.StdOutErrAsStringOutput | None = None,
+    stderr_as_string_output: ir.StdOutErrAsStringOutput | None = None,
 ) -> None:
     """Generate the outputs building code."""
     members = {}
@@ -374,6 +408,13 @@ def _py_get_val(
             raise Exception(f"Unsupported input type for output path template of '{param.base.name}'.")
         assert False
 
+    for stdout_stderr_output in (stdout_as_string_output, stderr_as_string_output):
+        if stdout_stderr_output is None:
+            continue
+        output_symbol = lookup.py_output_field_symbol[stdout_stderr_output.id_]
+
+        members[output_symbol] = lang.empty_str_list()
+
     for output in struct.base.outputs:
         output_symbol = lookup.py_output_field_symbol[output.id_]
 
@@ -466,4 +507,6 @@ def compile_interface(
         lookup=lookup,
         metadata_symbol=metadata_symbol,
         root_function=True,
+        stdout_as_string_output=interface.stdout_as_string_output,
+        stderr_as_string_output=interface.stderr_as_string_output,
     )

src/styx/backend/generic/gen/lookup.py

@@ -19,6 +19,14 @@ def _collect_output_field_symbols(
         ) -> None:
             scope = Scope(parent=package_scope)
             scope.add_or_die("root")
+
+            for stdout_stderr_output in (interface.stdout_as_string_output, interface.stderr_as_string_output):
+                if stdout_stderr_output is None:
+                    continue
+                output_field_symbol = scope.add_or_dodge(lang.ensure_var_case(stdout_stderr_output.name))
+                assert stdout_stderr_output.id_ not in lookup_output_field_symbol
+                lookup_output_field_symbol[stdout_stderr_output.id_] = output_field_symbol
+
             for output in param.base.outputs:
                 output_field_symbol = scope.add_or_dodge(lang.ensure_var_case(output.name))
                 assert output.id_ not in lookup_output_field_symbol

src/styx/backend/generic/languageprovider.py

@@ -290,7 +290,13 @@ def execution_declare(self, execution_symbol: str, metadata_symbol: str) -> Line
         """Construct command line args list."""
         return NotImplemented
 
-    def execution_run(self, execution_symbol: str, cargs_symbol: str) -> LineBuffer:
+    def execution_run(
+        self,
+        execution_symbol: str,
+        cargs_symbol: str,
+        stdout_output_symbol: str | None,
+        stderr_output_symbol: str | None,
+    ) -> LineBuffer:
         """Start execution."""
         return NotImplemented
 
@@ -323,4 +329,4 @@ def struct_collect_outputs(self, struct: ir.Param[ir.Param.Struct], struct_symbo
     @classmethod
     def styxdefs_compat(cls) -> str:
         """Return what version of styxdefs generated wrappers will be compatible with."""
-        return "^0.3.0"
+        return "^0.4.0"

src/styx/backend/python/languageprovider.py

@@ -46,8 +46,16 @@ def execution_symbol(self) -> str:
     def execution_declare(self, execution_symbol: str, metadata_symbol: str) -> LineBuffer:
         return [f"{execution_symbol} = runner.start_execution({metadata_symbol})"]
 
-    def execution_run(self, execution_symbol: str, cargs_symbol: str) -> LineBuffer:
-        return [f"{execution_symbol}.run({cargs_symbol})"]
+    def execution_run(
+        self,
+        execution_symbol: str,
+        cargs_symbol: str,
+        stdout_output_symbol: str | None,
+        stderr_output_symbol: str | None,
+    ) -> LineBuffer:
+        so = "" if stdout_output_symbol is None else f", handle_stdout=lambda s: ret.{stdout_output_symbol}.append(s)"
+        se = "" if stderr_output_symbol is None else f", handle_stderr=lambda s: ret.{stderr_output_symbol}.append(s)"
+        return [f"{execution_symbol}.run({cargs_symbol}{so}{se})"]
 
     def legal_symbol(self, symbol: str) -> bool:
         return symbol.isidentifier()

src/styx/frontend/boutiques/core.py

@@ -176,9 +176,9 @@ def _arg_elem_from_bt_elem(
     match input_type.primitive:
         case InputTypePrimitive.String:
             choices = d.get("value-choices")
-            assert choices is None or all([
-                isinstance(o, str) for o in choices
-            ]), "value-choices must be all string for string input"
+            assert choices is None or all([isinstance(o, str) for o in choices]), (
+                "value-choices must be all string for string input"
+            )
 
             return ir.Param(
                 base=dparam,
@@ -193,9 +193,9 @@ def _arg_elem_from_bt_elem(
 
         case InputTypePrimitive.Integer:
             choices = d.get("value-choices")
-            assert choices is None or all([
-                isinstance(o, int) for o in choices
-            ]), "value-choices must be all int for integer input"
+            assert choices is None or all([isinstance(o, int) for o in choices]), (
+                "value-choices must be all int for integer input"
+            )
             assert constraints.value_min is None or isinstance(constraints.value_min, int)
             assert constraints.value_max is None or isinstance(constraints.value_max, int)
 
@@ -449,6 +449,15 @@ def _collect_inputs(bt: dict, id_counter: IdCounter) -> tuple[list[ir.Conditiona
     return groups, ir_id_lookup
 
 
+def _collect_stdout_stderr_output(bt: dict, id_counter: IdCounter) -> ir.StdOutErrAsStringOutput:
+    assert "id" in bt, "StdOut / StdErr Output needs id"
+    return ir.StdOutErrAsStringOutput(
+        id_=id_counter.next(),
+        name=bt["id"],
+        docs=ir.Documentation(title=bt.get("name"), description=bt.get("description")),
+    )
+
+
 def from_boutiques(
     tool: dict,
     package_name: str,
@@ -463,6 +472,13 @@ def from_boutiques(
 
     id_counter = IdCounter()
 
+    stdout_output: ir.StdOutErrAsStringOutput | None = None
+    stderr_output: ir.StdOutErrAsStringOutput | None = None
+    if "stdout-output" in tool:
+        stdout_output = _collect_stdout_stderr_output(tool["stdout-output"], id_counter)
+    if "stderr-output" in tool:
+        stderr_output = _collect_stdout_stderr_output(tool["stderr-output"], id_counter)
+
     dparam, dstruct = _struct_from_boutiques(tool, id_counter)
 
     return ir.Interface(
@@ -477,4 +493,6 @@ def from_boutiques(
             base=dparam,
             body=dstruct,
         ),
+        stderr_as_string_output=stderr_output,
+        stdout_as_string_output=stdout_output,
     )

src/styx/ir/core.py

@@ -169,6 +169,11 @@ class Struct:
         groups: list[ConditionalGroup] = dataclasses.field(default_factory=list)
         """List of conditional groups."""
 
+        group_join: str | None = None
+        """If not none, collapse all groups to a single string with this delimiter.
+        Args within groups should be joined without delimiter.
+        """
+
         docs: Documentation | None = None
         """Documentation for the struct."""
 
@@ -485,7 +490,10 @@ def iter_params(self) -> Generator[Param, Any, None]:
 
 @dataclass
 class ConditionalGroup:
-    """Represents a group of conditional command arguments."""
+    """Represents a group of command arguments.
+
+    Arguments will only be emitted if at least one parameter within is defined, or if there are no parameters.
+    """
 
     cargs: list[Carg] = dataclasses.field(default_factory=list)
     """List of command arguments."""
@@ -500,6 +508,18 @@ def iter_params(self) -> Generator[Param, Any, None]:
             yield from carg.iter_params()
 
 
+@dataclass
+class StdOutErrAsStringOutput:
+    id_: IdType
+    """Unique ID of the output. Unique including param IDs."""
+
+    name: str
+    """The name of the output."""
+
+    docs: Documentation = dataclasses.field(default_factory=Documentation)
+    """Documentation for the output."""
+
+
 @dataclass
 class Interface:
     """Represents an interface."""
@@ -512,3 +532,9 @@ class Interface:
 
     command: Param[Param.Struct]
     """The command structure for this interface."""
+
+    stdout_as_string_output: StdOutErrAsStringOutput | None = None
+    """Collect stdout as string output."""
+
+    stderr_as_string_output: StdOutErrAsStringOutput | None = None
+    """Collect stderr as string output."""