Skip to content

Commit

Permalink
cleaned up insertion of special stdout, stderr and return_code output…
Browse files Browse the repository at this point in the history 
…s form shell commands
  • Loading branch information
@tclose
tclose committed Dec 10, 2024
1 parent 328e0b1 commit cf7b331
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 16 deletions.
12 changes: 12 additions & 0 deletions pydra/design/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,18 @@ def make_outputs_spec(
raise ValueError(
f"{reserved_names} are reserved and cannot be used for output field names"
)
# Add in any fields in base classes that haven't already been converted into attrs
# fields (e.g. stdout, stderr and return_code)
for base in outputs_bases:
base_outputs = {
n: o
for n, o in base.__dict__.items()
if isinstance(o, Out) and n not in outputs
}
for name, field in base_outputs.items():
field.name = name
field.type = base.__annotations__.get(name, ty.Any)
outputs.update(base_outputs)
outputs_klass = type(
spec_name + "Outputs",
tuple(outputs_bases),
Expand Down
164 changes: 154 additions & 10 deletions pydra/design/tests/test_shell.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,13 @@
import cloudpickle as cp
from pydra.design import shell
from pydra.engine.helpers import list_fields
from pydra.engine.specs import ShellSpec, ShellOutputs
from pydra.engine.specs import (
ShellSpec,
ShellOutputs,
RETURN_CODE_HELP,
STDOUT_HELP,
STDERR_HELP,
)
from fileformats.generic import File, Directory, FsObject
from fileformats import text, image
from pydra.utils.typing import MultiInputObj
Expand Down Expand Up @@ -34,7 +40,24 @@ def test_interface_template():
shell.arg(name="in_path", type=FsObject, position=1),
output,
]
assert sorted_fields(SampleInterface.Outputs) == [output]
assert sorted_fields(SampleInterface.Outputs) == [
output,
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]
intf = SampleInterface(in_path=File.mock("in-path.txt"))
assert intf.executable == "cp"
SampleInterface(in_path=File.mock("in-path.txt"), out_path=Path("./out-path.txt"))
Expand Down Expand Up @@ -65,7 +88,24 @@ def test_interface_template_w_types_and_path_template_ext():
shell.arg(name="in_image", type=image.Png, position=1),
output,
]
assert sorted_fields(SampleInterface.Outputs) == [output]
assert sorted_fields(SampleInterface.Outputs) == [
output,
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]
SampleInterface(in_image=image.Png.mock())
SampleInterface(in_image=image.Png.mock(), out_image=Path("./new_image.png"))
SampleInterface.Outputs(out_image=image.Png.mock())
Expand Down Expand Up @@ -93,7 +133,22 @@ def test_interface_template_w_modify():
name="image",
type=image.Png,
callable=shell._InputPassThrough("image"),
)
),
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]
SampleInterface(image=image.Png.mock())
SampleInterface.Outputs(image=image.Png.mock())
Expand Down Expand Up @@ -153,7 +208,24 @@ def test_interface_template_more_complex():
position=6,
),
]
assert sorted_fields(SampleInterface.Outputs) == [output]
assert sorted_fields(SampleInterface.Outputs) == [
output,
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]
SampleInterface(in_fs_objects=[File.sample(), File.sample(seed=1)])
SampleInterface.Outputs(out_dir=Directory.sample())

Expand Down Expand Up @@ -234,7 +306,23 @@ def test_interface_template_with_overrides_and_optionals():
]
+ outargs
)
assert sorted_fields(SampleInterface.Outputs) == outargs
assert sorted_fields(SampleInterface.Outputs) == outargs + [
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]


def test_interface_template_with_defaults():
Expand Down Expand Up @@ -281,7 +369,24 @@ def test_interface_template_with_defaults():
position=6,
),
]
assert sorted_fields(SampleInterface.Outputs) == [output]
assert sorted_fields(SampleInterface.Outputs) == [
output,
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]
SampleInterface(in_fs_objects=[File.sample(), File.sample(seed=1)])
SampleInterface.Outputs(out_dir=Directory.sample())

Expand Down Expand Up @@ -333,7 +438,24 @@ def test_interface_template_with_type_overrides():
position=6,
),
]
assert sorted_fields(SampleInterface.Outputs) == [output]
assert sorted_fields(SampleInterface.Outputs) == [
output,
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]


@pytest.fixture(params=["static", "dynamic"])
Expand Down Expand Up @@ -582,7 +704,12 @@ class Outputs:
)

assert sorted([a.name for a in attrs.fields(A)]) == ["executable", "x", "y"]
assert [a.name for a in attrs.fields(A.Outputs)] == ["y"]
assert sorted(a.name for a in attrs.fields(A.Outputs)) == [
"return_code",
"stderr",
"stdout",
"y",
]
output = shell.outarg(
name="y",
type=File,
Expand All @@ -609,7 +736,24 @@ class Outputs:
),
output,
]
assert sorted_fields(A.Outputs) == [output]
assert sorted_fields(A.Outputs) == [
output,
shell.out(
name="return_code",
type=int,
help_string=RETURN_CODE_HELP,
),
shell.out(
name="stderr",
type=str,
help_string=STDERR_HELP,
),
shell.out(
name="stdout",
type=str,
help_string=STDOUT_HELP,
),
]


def test_shell_output_field_name_dynamic():
Expand Down
14 changes: 8 additions & 6 deletions pydra/engine/specs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,15 +335,17 @@ class WorkflowSpec(TaskSpec[WorkflowOutputsType]):
pass


RETURN_CODE_HELP = """The process' exit code."""
STDOUT_HELP = """The standard output stream produced by the command."""
STDERR_HELP = """The standard error stream produced by the command."""


class ShellOutputs(Outputs):
"""Output specification of a generic shell process."""

return_code: int = shell.out()
"""The process' exit code."""
stdout: str = shell.out()
"""The process' standard output."""
stderr: str = shell.out()
"""The process' standard input."""
return_code: int = shell.out(help_string=RETURN_CODE_HELP)
stdout: str = shell.out(help_string=STDOUT_HELP)
stderr: str = shell.out(help_string=STDERR_HELP)

@classmethod
def collect_outputs(
Expand Down

0 comments on commit cf7b331

Please sign in to comment.