From 5ab86cca5fc2070937cfa90b1a479d06878e68dd Mon Sep 17 00:00:00 2001 From: Marnik Bercx Date: Fri, 9 Apr 2021 16:05:52 +0200 Subject: [PATCH] DOCS: Add "How to extend workflows" section (#4562) Split the section on "How to run multi-step workflows" into one that focuses on running the workflows and one on "How to write and extend workflows". Add a subsection on how to extend workflows to the second. This subsection continues with the `MultiplyAddWorkChain` example and covers: * How to submit the `MultiplyAddWorkChain` within a parent work chain. * How to expose the inputs using the `expose_inputs` method and a proper namespace. * How to use the exposed inputs with the `exposed_inputs` method. * How to expose outputs and pass them to the outputs of the parent work chain. --- .../include/snippets/extend_workflows.py | 205 ++++++++++++++ docs/source/howto/index.rst | 3 +- docs/source/howto/plugin_codes.rst | 2 +- docs/source/howto/run_workflows.rst | 163 +++++++++++ docs/source/howto/workchains_restart.rst | 4 +- .../{workflows.rst => write_workflows.rst} | 252 +++++++++--------- docs/source/intro/tutorial.rst | 2 +- docs/source/topics/processes/usage.rst | 2 +- docs/source/topics/workflows/index.rst | 2 +- docs/source/topics/workflows/usage.rst | 2 +- 10 files changed, 502 insertions(+), 135 deletions(-) create mode 100644 docs/source/howto/include/snippets/extend_workflows.py create mode 100644 docs/source/howto/run_workflows.rst rename docs/source/howto/{workflows.rst => write_workflows.rst} (51%) diff --git a/docs/source/howto/include/snippets/extend_workflows.py b/docs/source/howto/include/snippets/extend_workflows.py new file mode 100644 index 0000000000..5cf4439afd --- /dev/null +++ b/docs/source/howto/include/snippets/extend_workflows.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +# pylint: disable=no-member +# start-marker for docs +"""Code snippets for the "How to extend workflows" section.""" +from aiida.orm import Code, Int, Bool +from aiida.engine import calcfunction, WorkChain, ToContext +from aiida.plugins.factories import CalculationFactory + +ArithmeticAddCalculation = CalculationFactory('arithmetic.add') + + +@calcfunction +def multiply(x, y): + return x * y + + +@calcfunction +def is_even(number): + """Check if a number is even.""" + return Bool(number % 2 == 0) + + +class MultiplyAddWorkChain(WorkChain): + """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes.""" + + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.input('x', valid_type=Int) + spec.input('y', valid_type=Int) + spec.input('z', valid_type=Int) + spec.input('code', valid_type=Code) + spec.outline( + cls.multiply, + cls.add, + cls.validate_result, + cls.result, + ) + spec.output('result', valid_type=Int) + spec.exit_code(400, 'ERROR_NEGATIVE_NUMBER', message='The result is a negative number.') + + def multiply(self): + """Multiply two integers.""" + self.ctx.product = multiply(self.inputs.x, self.inputs.y) + + def add(self): + """Add two numbers using the `ArithmeticAddCalculation` calculation job plugin.""" + inputs = {'x': self.ctx.product, 'y': self.inputs.z, 'code': self.inputs.code} + future = self.submit(ArithmeticAddCalculation, **inputs) + + return ToContext(addition=future) + + def validate_result(self): + """Make sure the result is not negative.""" + result = self.ctx.addition.outputs.sum + + if result.value < 0: + return self.exit_codes.ERROR_NEGATIVE_NUMBER + + def result(self): + """Add the result to the outputs.""" + self.out('result', self.ctx.addition.outputs.sum) + + +class BadMultiplyAddIsEvenWorkChain(WorkChain): + """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes.""" + + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.input('x', valid_type=Int) + spec.input('y', valid_type=Int) + spec.input('z', valid_type=Int) + spec.input('code', valid_type=Code) + spec.outline( + cls.multiply, + cls.add, + cls.validate_result, + cls.is_even, + ) + spec.output('is_even', valid_type=Bool) + spec.exit_code(400, 'ERROR_NEGATIVE_NUMBER', message='The result is a negative number.') + + def multiply(self): + """Multiply two integers.""" + self.ctx.product = multiply(self.inputs.x, self.inputs.y) + + def add(self): + """Add two numbers using the `ArithmeticAddCalculation` calculation job plugin.""" + inputs = {'x': self.ctx.product, 'y': self.inputs.z, 'code': self.inputs.code} + future = self.submit(ArithmeticAddCalculation, **inputs) + + return ToContext(addition=future) + + def validate_result(self): + """Make sure the result is not negative.""" + result = self.ctx.addition.outputs.sum + + if result.value < 0: + return self.exit_codes.ERROR_NEGATIVE_NUMBER + + def is_even(self): + """Check if the result is even.""" + result_is_even = is_even(self.ctx.addition.outputs.sum) + + self.out('is_even', result_is_even) + + +class BetterMultiplyAddIsEvenWorkChain(WorkChain): + """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes.""" + + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.input('x', valid_type=Int) + spec.input('y', valid_type=Int) + spec.input('z', valid_type=Int) + spec.input('code', valid_type=Code) + spec.outline( + cls.multiply_add, + cls.is_even, + ) + spec.output('is_even', valid_type=Bool) + + def multiply_add(self): + """Multiply two integers and add a third.""" + inputs = {'x': self.inputs.x, 'y': self.inputs.y, 'z': self.inputs.z, 'code': self.inputs.code} + future = self.submit(MultiplyAddWorkChain, **inputs) + + return ToContext(multi_addition=future) + + def is_even(self): + """Check if the result is even.""" + result_is_even = is_even(self.ctx.multi_addition.outputs.result) + + self.out('is_even', result_is_even) + + +class MultiplyAddIsEvenWorkChain(WorkChain): + """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes.""" + + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.expose_inputs(MultiplyAddWorkChain, namespace='multiply_add') + spec.outline( + cls.multiply_add, + cls.is_even, + ) + spec.output('is_even', valid_type=Bool) + + def multiply_add(self): + """Multiply two integers and add a third.""" + future = self.submit( + MultiplyAddWorkChain, **self.exposed_inputs(MultiplyAddWorkChain, 'multiply_add') + ) + return ToContext(multi_addition=future) + + def is_even(self): + """Check if the result is even.""" + result_is_even = is_even(self.ctx.multi_addition.outputs.result) + + self.out('is_even', result_is_even) + +class ResultMultiplyAddIsEvenWorkChain(WorkChain): + """WorkChain to multiply two numbers and add a third, for testing and demonstration purposes.""" + + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.expose_inputs(MultiplyAddWorkChain, namespace='multiply_add') + spec.outline( + cls.multiply_add, + cls.is_even, + ) + spec.expose_outputs(MultiplyAddWorkChain) + spec.output('is_even', valid_type=Bool) + + def multiply_add(self): + """Multiply two integers and add a third.""" + future = self.submit( + MultiplyAddWorkChain, **self.exposed_inputs(MultiplyAddWorkChain, 'multiply_add') + ) + + return ToContext(multi_addition=future) + + def is_even(self): + """Check if the result is even.""" + result_is_even = is_even(self.ctx.multi_addition.outputs.result) + + self.out_many(self.exposed_outputs(self.ctx.multi_addition, MultiplyAddWorkChain)) + self.out('is_even', result_is_even) diff --git a/docs/source/howto/index.rst b/docs/source/howto/index.rst index 070e2ba600..82801fca62 100644 --- a/docs/source/howto/index.rst +++ b/docs/source/howto/index.rst @@ -8,9 +8,10 @@ How-To Guides interact plugins_install run_codes + run_workflows ssh plugin_codes - workflows + write_workflows workchains_restart data exploring diff --git a/docs/source/howto/plugin_codes.rst b/docs/source/howto/plugin_codes.rst index 225e51e7ec..976c65e660 100644 --- a/docs/source/howto/plugin_codes.rst +++ b/docs/source/howto/plugin_codes.rst @@ -223,7 +223,7 @@ Handling parsing errors So far, we have not spent much attention on dealing with potential errors that can arise when running external codes. However, there are lots of ways in which codes can fail to execute nominally. -A |Parser| can play an important role in detecting and communicating such errors, where :ref:`workflows ` can then decide how to proceed, e.g., by modifying input parameters and resubmitting the calculation. +A |Parser| can play an important role in detecting and communicating such errors, where :ref:`workflows ` can then decide how to proceed, e.g., by modifying input parameters and resubmitting the calculation. Parsers communicate errors through :ref:`exit codes`, which are defined in the |spec| of the |CalcJob| they parse. The :py:class:`~aiida.calculations.arithmetic.add.ArithmeticAddCalculation` example, defines the following exit codes: diff --git a/docs/source/howto/run_workflows.rst b/docs/source/howto/run_workflows.rst new file mode 100644 index 0000000000..07969a6047 --- /dev/null +++ b/docs/source/howto/run_workflows.rst @@ -0,0 +1,163 @@ +.. _how-to:run-workflows: + +******************************* +How to run multi-step workflows +******************************* + +Launching a predefined workflow +=============================== + +The first step to launching a predefined workflow is loading the work function or work chain class that defines the workflow you want to run. +The recommended method for loading a workflow is using the ``WorkflowFactory``, for example: + +.. code-block:: python + + from aiida.plugins import WorkflowFactory + add_and_multiply = WorkflowFactory('arithmetic.add_multiply') + MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') + +This is essentially the same as importing the workflow from its respective module, but using the ``WorkflowFactory`` has the advantage that the so called *entry point* (e.g. ``'arithmetic.multiply_add'``) will not change when the packages or plugins are reorganised. +This means your code is less likely to break when updating AiiDA or the plugin that supplies the workflow. + +The list of installed plugins can be easily accessed via the verdi CLI: + +.. code-block:: console + + $ verdi plugin list + +To see the list of workflow entry points, simply use: + +.. code-block:: console + + $ verdi plugin list aiida.workflows + +By further specifying the entry point of the workflow, you can see its description, inputs, outputs and exit codes: + +.. code-block:: console + + $ verdi plugin list aiida.workflows arithmetic.multiply_add + +Work functions +-------------- + +Running a work function is as simple as calling a typical Python function: simply call it with the required input arguments: + +.. code-block:: python + + from aiida.plugins import WorkflowFactory, DataFactory + add_and_multiply = WorkflowFactory('arithmetic.add_multiply') + Int = DataFactory('int') + + result = add_and_multiply(Int(2), Int(3), Int(5)) + +Here, the ``add_and_multiply`` work function returns the output ``Int`` node and we assign it to the variable ``result``. +Note that the input arguments of a work function must be an instance of ``Data`` node, or any of its subclasses. +Just calling the ``add_and_multiply`` function with regular integers will result in a ``ValueError``, as these cannot be stored in the provenance graph. + +.. note:: + + Although the example above shows the most straightforward way to run the ``add_and_multiply`` work function, there are several other ways of running processes that can return more than just the result. + For example, the ``run_get_node`` function from the AiiDA engine returns both the result of the workflow and the work function node. + See the :ref:`corresponding topics section for more details `. + +Work chains +----------- + +To launch a work chain, you can either use the ``run`` or ``submit`` functions. +For either function, you need to provide the class of the work chain as the first argument, followed by the inputs as keyword arguments. +When "running the work chain" (using the ``run`` function), it will be executed in the same system process as the interpreter in which it is launched: + +.. code-block:: python + + from aiida.engine import run + from aiida.plugins import WorkflowFactory, DataFactory + Int = DataFactory('int') + MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') + + add_code = load_code(label='add') + + results = run(MultiplyAddWorkChain, x=Int(2), y=Int(3), z=Int(5), code=add_code) + +Alternatively, you can first construct a dictionary of the inputs, and pass it to the ``run`` function by taking advantage of `Python's automatic keyword expansion `_: + +.. code-block:: python + + inputs = {'x': Int(1), 'y': Int(2), 'z': Int(3), 'code': add_code} + results = run(MultiplyAddWorkChain, **inputs) + +This is particularly useful in case you have a workflow with a lot of inputs. +In both cases, running the ``MultiplyAddWorkChain`` workflow returns the **results** of the workflow, i.e. a dictionary of the nodes that are produced as outputs, where the keys of the dictionary correspond to the labels of each respective output. + +.. note:: + + Similar to other processes, there are multiple functions for launching a work chain. + See the section on :ref:`launching processes for more details`. + +Since *running* a workflow will block the interpreter, you will have to wait until the workflow is finished before you get back control. +Moreover, you won't be able to turn your computer or even your terminal off until the workflow has fully terminated, and it is difficult to run multiple workflows in parallel. +So, it is advisable to *submit* more complex or longer work chains to the daemon: + +.. code-block:: python + + from aiida.engine import submit + from aiida.plugins import WorkflowFactory, DataFactory + Int = DataFactory('int') + MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') + + add_code = load_code(label='add') + inputs = {'x': Int(1), 'y': Int(2), 'z': Int(3), 'code': add_code} + + workchain_node = submit(MultiplyAddWorkChain, **inputs) + +Note that when using ``submit`` the work chain is not run in the local interpreter but is sent off to the daemon and you get back control instantly. +This allows you to submit multiple work chains at the same time and the daemon will start working on them in parallel. +Once the ``submit`` call returns, you will not get the result as with ``run``, but you will get the **node** that represents the work chain. +Submitting a work chain instead of directly running it not only makes it easier to execute multiple work chains in parallel, but also ensures that the progress of a workchain is not lost when you restart your computer. + +.. note:: + + As of AiiDA v1.5.0, it is possible to submit both work *chains* and work *functions* to the daemon. Older versions only allow the submission of work *chains*, whereas work *functions* cannot be submitted to the daemon, and hence can only be *run*. + +If you are unfamiliar with the inputs of a particular ``WorkChain``, a convenient tool for setting up the work chain is the :ref:`process builder`. +This can be obtained by using the ``get_builder()`` method, which is implemented for every ``CalcJob`` and ``WorkChain``: + +.. code-block:: ipython + + In [1]: from aiida.plugins import WorkflowFactory, DataFactory + ...: Int = DataFactory('int') + ...: MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') + ...: builder = MultiplyAddWorkChain.get_builder() + +To explore the inputs of the work chain, you can use tab autocompletion by typing ``builder.`` and then hitting ``TAB``. +If you want to get more details on a specific input, you can simply add a ``?`` and press enter: + +.. code-block:: ipython + + In [2]: builder.x? + Type: property + String form: + Docstring: {"name": "x", "required": "True", "valid_type": "", "non_db": "False"} + +Here you can see that the ``x`` input is required, needs to be of the ``Int`` type and is stored in the database (``"non_db": "False"``). + +Using the builder, the inputs of the ``WorkChain`` can be provided one by one: + +.. code-block:: ipython + + In [3]: builder.code = load_code(label='add') + ...: builder.x = Int(2) + ...: builder.y = Int(3) + ...: builder.z = Int(5) + +Once the *required* inputs of the workflow have been provided to the builder, you can either run the work chain or submit it to the daemon: + +.. code-block:: ipython + + In [4]: from aiida.engine import submit + ...: workchain_node = submit(builder) + +.. note:: + + For more detail on the process builder, see the :ref:`corresponding topics section`. + +Now that you know how to run a pre-defined workflow, you may want to start :ref:`writing your own`. diff --git a/docs/source/howto/workchains_restart.rst b/docs/source/howto/workchains_restart.rst index 12c3d7e9d9..3f5db0bf29 100644 --- a/docs/source/howto/workchains_restart.rst +++ b/docs/source/howto/workchains_restart.rst @@ -1,4 +1,4 @@ -.. _how-to:restart_workchain: +.. _how-to:restart-workchain: ************************************** How to write error-resistant workflows @@ -8,7 +8,7 @@ How to write error-resistant workflows This how-to introduces the :py:class:`~aiida.engine.processes.workchains.restart.BaseRestartWorkChain`, and how it can be sub-classed to handle known failure modes of processes and calculations. -In the :ref:`multi-step workflows how-to ` we discussed how to write a simple multi-step workflow using work chains. +In the :ref:`how-to on writing workflows ` we discussed how to write a simple multi-step workflow using work chains. However, there is one thing that we did not consider there: What if a calculation step fails? diff --git a/docs/source/howto/workflows.rst b/docs/source/howto/write_workflows.rst similarity index 51% rename from docs/source/howto/workflows.rst rename to docs/source/howto/write_workflows.rst index 844fc7b2c1..ebf7a62137 100644 --- a/docs/source/howto/workflows.rst +++ b/docs/source/howto/write_workflows.rst @@ -1,10 +1,8 @@ -.. _how-to:workflows: +.. _how-to:write-workflows: -******************************* -How to run multi-step workflows -******************************* - -.. _how-to:workflows:write: +********************************* +How to write and extend workflows +********************************* Writing workflows ================= @@ -139,171 +137,171 @@ The second argument is the result of the work chain, extracted from the ``Int`` For a more complete discussion on workflows and their usage, please read :ref:`the corresponding topics section`. -.. _how-to:workflows:run: - -Launching a predefined workflow -=============================== - -The first step to launching a predefined workflow is loading the work function or work chain class that defines the workflow you want to run. -The recommended method for loading a workflow is using the ``WorkflowFactory``, for example: - -.. code-block:: python - - from aiida.plugins import WorkflowFactory - add_and_multiply = WorkflowFactory('arithmetic.add_multiply') - MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') - -This is essentially the same as importing the workflow from its respective module, but using the ``WorkflowFactory`` has the advantage that the so called *entry point* (e.g. ``'arithmetic.multiply_add'``) will not change when the packages or plugins are reorganised. -This means your code is less likely to break when updating AiiDA or the plugin that supplies the workflow. - -The list of installed plugins can be easily accessed via the verdi CLI: - -.. code-block:: console - - $ verdi plugin list +.. _how-to:write-workflows:extend: -To see the list of workflow entry points, simply use: +Extending workflows +=================== -.. code-block:: console +When designing workflows, there are many cases where you want to reuse an existing process. +This section explains how to extend workflows by wrapping them around other processes or linking them together. - $ verdi plugin list aiida.workflows +As an example, let's say you want to extend the ``MultiplyAddWorkChain`` by adding another step of analysis that checks whether the result is an even number or not. +This final step can be written as a simple ``calcfunction``: -By further specifying the entry point of the workflow, you can see its description, inputs, outputs and exit codes: +.. literalinclude:: include/snippets/extend_workflows.py + :language: python + :pyobject: is_even -.. code-block:: console +We could simply write a new workflow based off ``MultiplyAddWorkChain`` that includes an extra step in the outline which runs the ``is_even`` calculation function. +However, this would lead to a lot of code duplication, and longer workflows consisting of multiple work chains would become very cumbersome to deal with (see the dropdown panel below). - $ verdi plugin list aiida.workflows arithmetic.multiply_add +.. dropdown:: ``BadMultiplyAddIsEvenWorkChain`` -Work functions --------------- + .. literalinclude:: include/snippets/extend_workflows.py + :language: python + :pyobject: BadMultiplyAddIsEvenWorkChain -Running a work function is as simple as calling a typical Python function: simply call it with the required input arguments: + .. note:: -.. code-block:: python + We've removed the ``result`` step from the outline, as well as the ``result`` output. + For this work chain, we're assuming that for now we are only interested in whether or not the result is even. - from aiida.plugins import WorkflowFactory, DataFactory - add_and_multiply = WorkflowFactory('arithmetic.add_multiply') - Int = DataFactory('int') +We can avoid some code duplication by simply submitting the ``MultiplyAddWorkChain`` within one of the steps of a new work chain which would then call ``is_even`` in a second step: - result = add_and_multiply(Int(2), Int(3), Int(5)) +.. literalinclude:: include/snippets/extend_workflows.py + :language: python + :pyobject: BetterMultiplyAddIsEvenWorkChain -Here, the ``add_and_multiply`` work function returns the output ``Int`` node and we assign it to the variable ``result``. -Note that the input arguments of a work function must be an instance of ``Data`` node, or any of its subclasses. -Just calling the ``add_and_multiply`` function with regular integers will result in a ``ValueError``, as these cannot be stored in the provenance graph. +This already simplifies the extended work chain, and avoids duplicating the steps of the ``MultiplyAddWorkChain`` in the outline. +However, we still had to copy all of the input definitions of the ``MultiplyAddWorkChain``, and manually extract them from the inputs before passing them to the ``self.submit`` method. +Fortunately, there is a better way of *exposing* the inputs and outputs of subprocesses of the work chain. -.. note:: +Exposing inputs and outputs +--------------------------- - Although the example above shows the most straightforward way to run the ``add_and_multiply`` work function, there are several other ways of running processes that can return more than just the result. - For example, the ``run_get_node`` function from the AiiDA engine returns both the result of the workflow and the work function node. - See the :ref:`corresponding topics section for more details `. +In many cases it is convenient for work chains to expose the inputs of the subprocesses it wraps so users can specify these inputs directly, as well as exposing some of the outputs produced as one of the results of the parent work chain. +For the simple example presented in the previous section, simply copy-pasting the input and output port definitions of the subprocess ``MultiplyAddWorkChain`` was not too troublesome. +However, this quickly becomes tedious and error-prone once you start to wrap processes with quite a few more inputs. -Work chains ------------ - -To launch a work chain, you can either use the ``run`` or ``submit`` functions. -For either function, you need to provide the class of the work chain as the first argument, followed by the inputs as keyword arguments. -Using the ``run`` function, or "running", a work chain means it is executed in the same system process as the interpreter in which it is launched: +To prevent the copy-pasting of input and output specifications, the :class:`~aiida.engine.processes.process_spec.ProcessSpec` class provides the :meth:`~plumpy.ProcessSpec.expose_inputs` and :meth:`~plumpy.ProcessSpec.expose_outputs` methods. +Calling :meth:`~plumpy.ProcessSpec.expose_inputs` for a particular ``Process`` class, will automatically copy the inputs of the class into the inputs namespace of the process specification: .. code-block:: python - from aiida.engine import run - from aiida.plugins import WorkflowFactory, DataFactory - Int = DataFactory('int') - MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') - - add_code = load_code(label='add') - - results = run(MultiplyAddWorkChain, x=Int(2), y=Int(3), z=Int(5), code=add_code) - -Alternatively, you can first construct a dictionary of the inputs, and pass it to the ``run`` function by taking advantage of `Python's automatic keyword expansion `_: + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.expose_inputs(MultiplyAddWorkChain) # Expose the inputs instead of copying their definition + spec.outline( + cls.multiply_add, + cls.is_even, + ) + spec.output('is_even', valid_type=Bool) + +Be aware that any inputs that already exist in the namespace will be overridden. +To prevent this, the method accepts the ``namespace`` argument, which will cause the inputs to be copied into that namespace instead of the top-level namespace. +This is especially useful for exposing inputs since *all* processes have the ``metadata`` input. +If you expose the inputs without a namespace, the ``metadata`` input port of the exposed class will override the one of the host, which is often not desirable. +Let's copy the inputs of the ``MultiplyAddWorkChain`` into the ``multiply_add`` namespace: + +.. literalinclude:: include/snippets/extend_workflows.py + :language: python + :pyobject: MultiplyAddIsEvenWorkChain.define + :dedent: 4 -.. code-block:: python +That takes care of exposing the port specification of the wrapped process class in a very efficient way. +To easily retrieve the inputs that have been passed to the process, one can use the :meth:`~aiida.engine.processes.process.Process.exposed_inputs` method. +Note the past tense of the method name. +The method takes a process class and an optional namespace as arguments, and will return the inputs that have been passed into that namespace when it was launched. +This utility now allows us to simplify the ``multiply_add`` step in the outline: - inputs = {'x': Int(1), 'y': Int(2), 'z': Int(3), 'code': add_code} - results = run(MultiplyAddWorkChain, **inputs) +.. literalinclude:: include/snippets/extend_workflows.py + :language: python + :pyobject: MultiplyAddIsEvenWorkChain.multiply_add + :dedent: 4 -This is particularly useful in case you have a workflow with a lot of inputs. -In both cases, running the ``MultiplyAddWorkChain`` workflow returns the **results** of the workflow, i.e. a dictionary of the nodes that are produced as outputs, where the keys of the dictionary correspond to the labels of each respective output. +This way we don't have to manually fish out all the individual inputs from the ``self.inputs`` but have to just call this single method, saving time and lines of code. +The final ``MultiplyAddIsEvenWorkChain`` can be found in the dropdown panel below. -.. note:: +.. dropdown:: ``MultiplyAddIsEvenWorkChain`` - Similar to other processes, there are multiple functions for launching a work chain. - See the section on :ref:`launching processes for more details`. + .. literalinclude:: include/snippets/extend_workflows.py + :language: python + :pyobject: MultiplyAddIsEvenWorkChain -Since *running* a workflow will block the interpreter, you will have to wait until the workflow is finished before you get back control. -Moreover, you won't be able to turn your computer or even your terminal off until the workflow has fully terminated, and it is difficult to run multiple workflows in parallel. -So, it is advisable to *submit* more complex or longer work chains to the daemon: +When submitting or running the work chain using namespaced inputs (``multiply_add`` in the example above), it is important to use the namespace when providing the inputs: .. code-block:: python - from aiida.engine import submit - from aiida.plugins import WorkflowFactory, DataFactory - Int = DataFactory('int') - MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') - add_code = load_code(label='add') - inputs = {'x': Int(1), 'y': Int(2), 'z': Int(3), 'code': add_code} + inputs = { + 'multiply_add': {'x': Int(1), 'y': Int(2), 'z': Int(3), 'code': add_code} + } workchain_node = submit(MultiplyAddWorkChain, **inputs) -Note that when using ``submit`` the work chain is not run in the local interpreter but is sent off to the daemon, and you get back control instantly. -This allows you to submit multiple work chains at the same time and the daemon will start working on them in parallel. -Once the ``submit`` call returns, you will not get the result as with ``run``, but you will get the **node** representing the work chain. -Submitting a work chain instead of directly running it not only makes it easier to execute multiple work chains in parallel, but also ensures that the progress of a workchain is not lost when you restart your computer. - -.. important:: - - In contrast to work chains, work *functions* cannot be submitted to the daemon, and hence can only be *run*. - -If you are unfamiliar with the inputs of a particular ``WorkChain``, a convenient tool for setting up the work chain is the :ref:`process builder`. -This can be obtained by using the ``get_builder()`` method, which is implemented for every ``CalcJob`` and ``WorkChain``: +After running the ``MultiplyAddIsEvenWorkChain``, you can see a hierarchical overview of the processes called by the work chain using the ``verdi process status`` command: -.. code-block:: ipython - - In [1]: from aiida.plugins import WorkflowFactory, DataFactory - ...: Int = DataFactory('int') - ...: MultiplyAddWorkChain = WorkflowFactory('arithmetic.multiply_add') - ...: builder = MultiplyAddWorkChain.get_builder() - -To explore the inputs of the work chain, you can use tab autocompletion by typing ``builder.`` and then hitting ``TAB``. -If you want to get more details on a specific input, you can simply add a ``?`` and press enter: - -.. code-block:: ipython - - In [2]: builder.x? - Type: property - String form: - Docstring: {"name": "x", "required": "True", "valid_type": "", "non_db": "False"} +.. code-block:: console -Here you can see that the ``x`` input is required, needs to be of the ``Int`` type and is stored in the database (``"non_db": "False"``). + $ verdi process status 164 + MultiplyAddIsEvenWorkChain<164> Finished [0] [1:is_even] + ├── MultiplyAddWorkChain<165> Finished [0] [3:result] + │ ├── multiply<166> Finished [0] + │ └── ArithmeticAddCalculation<168> Finished [0] + └── is_even<172> Finished [0] -Using the builder, the inputs of the ``WorkChain`` can be provided one by one: +Note that this command also recursively shows the processes called by the subprocesses of the ``MultiplyAddIsEvenWorkChain`` work chain. -.. code-block:: ipython +As mentioned earlier, you can also expose the outputs of the ``MultiplyAddWorkChain`` using the :meth:`~plumpy.ProcessSpec.expose_outputs` method. +Say we want to add the ``result`` of the ``MultiplyAddWorkChain`` as one of the outputs of the extended work chain: - In [3]: builder.code = load_code(label='add') - ...: builder.x = Int(2) - ...: builder.y = Int(3) - ...: builder.z = Int(5) +.. code-block:: python -Once the *required* inputs of the workflow have been provided to the builder, you can either run the work chain or submit it to the daemon: + @classmethod + def define(cls, spec): + """Specify inputs and outputs.""" + super().define(spec) + spec.expose_inputs(MultiplyAddWorkChain, namespace='multiply_add') + spec.outline( + cls.multiply_add, + cls.is_even, + ) + spec.expose_outputs(MultiplyAddWorkChain) + spec.output('is_even', valid_type=Bool) + +Since there is not one output port that is shared by all process classes, it is less critical to use the ``namespace`` argument when exposing outputs. +However, take care not to override the outputs of the parent work chain in case they do have outputs with the same port name. +We still need to pass the ``result`` of the ``MultiplyAddWorkChain`` to the outputs of the parent work chain. +For example, we could do this in the ``is_even`` step by using the :meth:`~aiida.engine.processes.process.Process.out` method: -.. code-block:: ipython +.. code-block:: python - In [4]: from aiida.engine import submit - ...: workchain_node = submit(builder) + def is_even(self): + """Check if the result is even.""" + result = self.ctx.multi_addition.outputs.result + result_is_even = is_even(result) -.. note:: + self.out('result', result) + self.out('is_even', result_is_even) - For more detail on the process builder, see the :ref:`corresponding topics section`. +This works fine if we want to pass a single output to the parent work chain, but once again becomes tedious and error-prone when passing multiple outputs. +Instead we can use the :meth:`~aiida.engine.processes.process.Process.exposed_outputs` method in combination with the :meth:`~aiida.engine.processes.process.Process.out_many` method: +.. code-block:: python -.. todo:: + def is_even(self): + """Check if the result is even.""" + result_is_even = is_even(self.ctx.multi_addition.outputs.result) - .. _how-to:workflows:extend: + self.out_many(self.exposed_outputs(self.ctx.multi_addition, MultiplyAddWorkChain)) + self.out('is_even', result_is_even) - title: Extending workflows +The :meth:`~aiida.engine.processes.process.Process.exposed_outputs` method returns a dictionary of the exposed outputs of the ``MultiplyAddWorkChain``, extracted from the workchain node stored in the ``multi_addition`` key of the context. +The :meth:`~aiida.engine.processes.process.Process.out_many` method takes this dictionary and assigns its values to the output ports with names equal to the corresponding keys. - `#3993`_ +.. important:: -.. _#3993: https://github.com/aiidateam/aiida-core/issues/3993 + Besides avoiding code duplication and errors, using the methods for exposing inputs and outputs also has the advantage that our parent work chain doesn't have to be adjusted in case the inputs or outputs of the child work chain change. + This makes the code much easier to maintain. diff --git a/docs/source/intro/tutorial.rst b/docs/source/intro/tutorial.rst index 8fffa94945..c72676c7da 100644 --- a/docs/source/intro/tutorial.rst +++ b/docs/source/intro/tutorial.rst @@ -614,7 +614,7 @@ We have also compiled useful how-to guides that are especially relevant for the :container: Designing a workflow - After reading the :ref:`Basic Tutorial `, you may want to learn about how to encode the logic of a typical scientific workflow in the :ref:`multi-step workflows how-to `. + After reading the :ref:`Basic Tutorial `, you may want to learn about how to encode the logic of a typical scientific workflow in the :ref:`writing workflows how-to `. Reusable data types If you have a certain input or output data type, which you use often, then you may wish to turn it into its own :ref:`data plugin `. diff --git a/docs/source/topics/processes/usage.rst b/docs/source/topics/processes/usage.rst index e9653be16c..94200702c5 100644 --- a/docs/source/topics/processes/usage.rst +++ b/docs/source/topics/processes/usage.rst @@ -261,7 +261,7 @@ Additionally, the exit codes make it very easy to query for failed processes wit - :ref:`CalcJob parsers` - :ref:`Workchain exit code specification` - :ref:`External code plugins` - - :ref:`Restart workchains` + - :ref:`Restart workchains` .. _topics:processes:usage:exit_code_conventions: diff --git a/docs/source/topics/workflows/index.rst b/docs/source/topics/workflows/index.rst index 3d7373d8ad..f89b40d33f 100644 --- a/docs/source/topics/workflows/index.rst +++ b/docs/source/topics/workflows/index.rst @@ -5,7 +5,7 @@ Workflows ********* This topic section provides detailed information on the concept of workflows in AiiDA and an extensive guide on how to work with them. -An introductory guide to working with workflows can be found in :ref:`"How to run multi-step workflows"`. +An introductory guide to working with workflows can be found in :ref:`"How to run multi-step workflows"`. .. toctree:: :maxdepth: 2 diff --git a/docs/source/topics/workflows/usage.rst b/docs/source/topics/workflows/usage.rst index 4de24a9f7c..8f7160fe90 100644 --- a/docs/source/topics/workflows/usage.rst +++ b/docs/source/topics/workflows/usage.rst @@ -589,7 +589,7 @@ Again, the ``namespace`` and ``agglomerate`` options can be used to select which .. seealso:: - For further practical examples of creating workflows, see the :ref:`how to run multi-step workflows` and :ref:`how to write error resistant workflows ` sections. + For further practical examples of creating workflows, see the :ref:`how to write workflows` and :ref:`how to write error resistant workflows ` sections. .. rubric:: Footnotes