dev: improve python-run-tests execution

Improves the `python-run-tests` command to create Python-3.8 based
virtual environments to use the same version as container images.

Updates `pip` before installing each component in order to speed up the
dependency resolution for certain cluster components.

Allows execution of selected pytests only by passing over
`PYTEST_ADDOPTS` environment variable.

Switches to using `run-tests.sh` to execute pytests in individual
components which takes care of the DB setup at the source.

Allows excluding certain Python components such as `-c CLUSTER
--exclude-components r-j-controller` on macOS.

Example:

```
$ PYTEST_ADDOPTS=tests/test_version.py reana-dev python-unit-tests -c ALL -k
```

Closes #755.

CHANGES.rst

@@ -1,6 +1,14 @@
 Changes
 =======
 
+Version 0.9.2 (UNRELEASED)
+--------------------------
+
+- Developers:
+    - Changes `python-run-tests` command to allow execution of selected pytests only by passing over `PYTEST_ADDOPTS` environment variable.
+    - Changes `python-run-tests` command to allow excluding certain Python components.
+    - Fixes `python-run-tests` command to create Python-3.8 based virtual environments to use the same version as container images.
+
 Version 0.9.1 (2023-09-27)
 --------------------------
 

reana/config.py

@@ -293,5 +293,8 @@
 CODECOV_REANAHUB_URL = "https://codecov.io/gh/reanahub"
 """REANA Hub organisation Codecov URL."""
 
+PYTHON_EXECUTABLE_NAME = "python3.8"
+"""Python executable name with the same version as cluster components."""
+
 PYTHON_DOCKER_IMAGE = "docker.io/library/python:3.8"
-"""Python docker image with same version as cluster components."""
+"""Python docker image with the same version as cluster components."""

reana/reana_dev/python.py

@@ -12,7 +12,11 @@
 
 import click
 
-from reana.config import COMPONENTS_USING_SHARED_MODULE_DB, REPO_LIST_CLUSTER
+from reana.config import (
+    COMPONENTS_USING_SHARED_MODULE_DB,
+    REPO_LIST_CLUSTER,
+    PYTHON_EXECUTABLE_NAME,
+)
 from reana.reana_dev.utils import (
     display_message,
     get_srcdir,
@@ -76,13 +80,20 @@ def python_install_eggs():
     default=["ALL"],
     help="Which components? [shortname|name|.|CLUSTER|ALL]",
 )
+@click.option(
+    "--exclude-components",
+    default="",
+    help="Which components to exclude? [c1,c2,c3]",
+)
 @click.option(
     "--keep-virtual-environment",
     "-k",
     is_flag=True,
     help="Whether to keep or not virtual environment after tests are finished",
 )
-def python_unit_tests(component: str, keep_virtual_environment: bool):  # noqa: D301
+def python_unit_tests(
+    component: str, exclude_components: str, keep_virtual_environment: bool
+):  # noqa: D301
     """Run Python unit tests in independent environments.
 
     For each component, create a dedicated throw-away virtual environment,
@@ -107,11 +118,15 @@ def python_unit_tests(component: str, keep_virtual_environment: bool):  # noqa:
                          * (7) special value 'ALL' that will expand to include
                                all REANA repositories.
     :type component: str
+    :param exclude_components: List of components to exclude from command.
+    :type exclude_components: str
     :param keep_virtual_environment: flag, whether to keep or not virtual environment
                                     after tests are finished
     :type keep_virtual_environment: bool
     """
-    components = select_components(component)
+    if exclude_components:
+        exclude_components = exclude_components.split(",")
+    components = select_components(component, exclude_components)
     for component in components:
         if component == "reana-job-controller" and platform.system() == "Darwin":
             msg = (
@@ -128,31 +143,49 @@ def python_unit_tests(component: str, keep_virtual_environment: bool):  # noqa:
                     "-e POSTGRES_PASSWORD=mysecretpassword -d docker.io/library/postgres:12.13"
                 )
 
+            env_pytestarg = ""
+            if os.getenv("PYTEST_ADDOPTS", ""):
+                env_pytestarg = 'PYTEST_ADDOPTS="{}"'.format(
+                    os.getenv("PYTEST_ADDOPTS", "")
+                )
+
             for cmd in [
-                "virtualenv ~/.virtualenvs/_{}".format(component),
+                "virtualenv ~/.virtualenvs/_{} -p {}".format(
+                    component, PYTHON_EXECUTABLE_NAME
+                ),
                 "{} && which python".format(cmd_activate_venv),
+                "{} && pip install pip --upgrade".format(cmd_activate_venv),
                 "{} && cd ../pytest-reana && "
                 " pip install . --upgrade".format(cmd_activate_venv),
                 "{} && cd ../reana-commons && "
                 " pip install . --upgrade".format(cmd_activate_venv),
                 "{} && cd ../reana-db && "
                 " pip install . --upgrade".format(cmd_activate_venv),
                 "git clean -d -ff -x",
+                # Fix installation of r-w-e-snakemake test dependencies for macOS/brew
+                '{} && GRAPHVIZ_DIR="$(brew --prefix graphviz)" pip install pygraphviz==1.7 --global-option=build_ext --global-option="-I$GRAPHVIZ_DIR/include" --global-option="-L$GRAPHVIZ_DIR/lib"'.format(
+                    cmd_activate_venv
+                )
+                if component == "reana-workflow-engine-snakemake"
+                and platform.system() == "Darwin"
+                else "",
+                # Fix installation of r-commons test dependencies
+                '{} && pip install ".[cwl,snakemake,yadage]" --upgrade'.format(
+                    cmd_activate_venv
+                )
+                if component == "reana-commons"
+                else "",
+                # Now we can call installing regular test dependencies
                 '{} && pip install ".[tests]" --upgrade'.format(cmd_activate_venv),
-                "{} && {} python setup.py test".format(
-                    cmd_activate_venv,
-                    "REANA_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://postgres:mysecretpassword@localhost/postgres"
-                    if does_component_need_db(component)
-                    else "",
+                "{} && {} ./run-tests.sh --check-pytest".format(
+                    cmd_activate_venv, env_pytestarg
                 ),
             ]:
                 run_command(cmd, component)
 
             if not keep_virtual_environment:
                 run_command(f"rm -rf ~/.virtualenvs/_{component}")
 
-            if does_component_need_db(component):
-                run_command(f"docker stop postgres__{component}")
         else:
             msg = (
                 "Ignoring this component that does not contain"