new atomic variable: AtomicJsonReader (#341)

* new atomic variable: `AtomicJsonReader`

get testdata from a json-file with a list of objects.

move `@templatingfilter` from `grizzly.testdata.utils` to `grizzly.testdata.filters` and add some good-to-have filters.

added documentation regarding templating filters.

fixed bug where `ask for variable` steps, added to all the scenarios causes problems if they are not used in all scenarios, also add them as a orphan template.

* fixed tests

* new step to create configuration in background section in feature-file

rename och inverted `only_grizzly` to `try_template`.

docs/content/framework/usage/variables/templating/filters.md

@@ -0,0 +1,4 @@
+---
+title: Filters
+---
+@pydoc grizzly.testdata.filters

docs/mkdocs.yaml

@@ -9,7 +9,9 @@ nav:
       - Variables:
         - framework/usage/variables/environment-configuration.md
         - Testdata:
-        - framework/usage/variables/templating.md
+        - Templating:
+          - framework/usage/variables/templating/index.md
+          - framework/usage/variables/templating/filters.md
       - Load Users:
       - Steps:
         - Background:

example/features/environment.py

@@ -17,7 +17,7 @@
 )
 from grizzly.context import GrizzlyContext
 from grizzly.exceptions import StopUser
-from grizzly.testdata.utils import templatingfilter
+from grizzly.testdata.filters import templatingfilter
 
 
 @templatingfilter

grizzly/behave.py

@@ -18,6 +18,7 @@
 from .exceptions import FeatureError, StepError
 from .locust import on_worker
 from .locust import run as locustrun
+from .testdata import filters
 from .testdata.variables import destroy_variables
 from .types import RequestType
 from .types.behave import Context, Feature, Scenario, Status, Step
@@ -26,6 +27,10 @@
 
 logger = logging.getLogger(__name__)
 
+__all__ = [
+    'filters',
+]
+
 try:
     import pymqi
 except ModuleNotFoundError:

grizzly/steps/_helpers.py

@@ -104,7 +104,7 @@ def add_request_task(
     content_type: Optional[TransformerContentType] = None
 
     if endpoint is not None and ('$env::' in endpoint or '$conf::' in endpoint):
-        endpoint = cast(str, resolve_variable(grizzly.scenario, endpoint, guess_datatype=False, only_grizzly=True))
+        endpoint = cast(str, resolve_variable(grizzly.scenario, endpoint, guess_datatype=False, try_template=False))
 
     table = context.table if context.table is not None else [None]
 

grizzly/steps/background/setup.py

@@ -157,3 +157,29 @@ def my_custom_callback(env: Environment, msg: Message) -> None:
     }, f'{module_name}.{callback_name} does not have grizzly.types.MessageCallback method signature: {method_signature}'
 
     grizzly.setup.locust.messages.register(message_direction, message_type, callback)
+
+
+@given('value for configuration "{name}" is "{value}"')
+def step_setup_configuration_value(context: Context, name: str, value: str) -> None:
+    """Step to set configuration variables not present in specified environment file.
+
+    The configuration value can then be used in the following steps. If the specified `name` already exists, it will be
+    overwritten.
+
+    Example:
+    ```gherkin
+    Given value for configuration "default.host" is "example.com"
+    ...
+    Then log message "default.host=$conf::default.host$"
+    ```
+
+    Args:
+        name (str): dot separated name/path of configuration value
+        value (str): configuration value, any `$..$` variables are resolved, but `{{ .. }}` templates are kept
+
+    """
+    grizzly = cast(GrizzlyContext, context.grizzly)
+
+    resolved_value = resolve_variable(grizzly.scenario, value, try_template=False)
+
+    grizzly.state.configuration.update({name: resolved_value})

grizzly/steps/setup.py

@@ -43,16 +43,18 @@ def step_setup_variable_value_ask(context: Context, name: str) -> None:
 
     value = environ.get(f'TESTDATA_VARIABLE_{name}', None)
     assert value is not None, f'variable "{name}" does not have a value'
-    assert name not in grizzly.scenario.variables, f'variable "{name}" has already been set'
 
     try:
         if not context.step.in_background:
+            assert name not in grizzly.scenario.variables, f'variable "{name}" has already been set'
             resolved_value = resolve_variable(grizzly.scenario, value, guess_datatype=True)
             grizzly.scenario.variables.update({name: resolved_value})
         else:
             for scenario in grizzly.scenarios:
+                assert name not in scenario.variables, f'variable "{name}" has already been set in scenario {scenario.name}'
                 resolved_value = resolve_variable(scenario, value, guess_datatype=True)
                 scenario.variables.update({name: resolved_value})
+                scenario.orphan_templates.append(f'{{{{ {name }}}}}')
     except ValueError as e:
         raise AssertionError(e) from e
 
@@ -125,7 +127,7 @@ def step_setup_variable_value(context: Context, name: str, value: str) -> None:
             # data type will be guessed when setting the variable
             persisted_initial_value = grizzly.scenario.variables.persistent.get(name, None)
             if persisted_initial_value is None:
-                resolved_value = resolve_variable(grizzly.scenario, value, guess_datatype=False)
+                resolved_value = resolve_variable(grizzly.scenario, value, guess_datatype=False, try_file=False)
                 if isinstance(value, str) and has_template(value):
                     grizzly.scenario.orphan_templates.append(value)
             else:

grizzly/tasks/clients/__init__.py

@@ -93,7 +93,7 @@ def __init__(  # noqa: PLR0915
         self._scenario = copy(self.__scenario__)
         self._scenario._tasks = self.__scenario__._tasks
 
-        endpoint = cast(str, resolve_variable(self._scenario, endpoint, only_grizzly=True))
+        endpoint = cast(str, resolve_variable(self._scenario, endpoint, try_template=False))
 
         try:
             parsed = urlparse(endpoint)

grizzly/testdata/filters.py

@@ -0,0 +1,77 @@
+"""Grizzly native tempalting filters that can be used to manipulate variable values where they are used."""
+from __future__ import annotations
+
+import json
+from collections import namedtuple
+from typing import Any, Callable, NamedTuple, Optional, Union
+
+from jinja2.filters import FILTERS
+
+
+class templatingfilter:
+    name: str
+    func: Callable
+
+    def __init__(self, func: Callable) -> None:
+        self.func = func
+        name = func.__name__
+        existing_filter = FILTERS.get(name, None)
+
+        if existing_filter is None:
+            FILTERS[name] = func
+        elif existing_filter is not func:
+            message = f'{name} is already registered as a filter'
+            raise AssertionError(message)
+        else:
+            # code executed twice, so adding the same filter again
+            pass
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        return self.func(*args, **kwargs)
+
+
+@templatingfilter
+def fromtestdata(value: NamedTuple) -> dict[str, Any]:
+    """Convert testdata object to a dictionary.
+
+    Nested testdata is a `namedtuple` object, e.g. `AtomicCsvReader.test`, where column values are accessed with
+    `AtomicCsvReader.test.header1`. If anything should be done for the whole row/item it must be converted to a
+    dictionary.
+
+    Example:
+    ```gherkin
+    Given value of variable "AtomicCsvReader.test" is "test.csv"
+
+    Then log message "test={{ AtomicCsvReader.test | fromtestdata | fromjson }}"
+    ```
+
+    Args:
+        value (NamedTuple): testdata objekt
+
+    """
+    testdata = dict(sorted(value._asdict().items()))
+    for k, v in testdata.items():
+        if type(v) is namedtuple:  # noqa: PYI024
+            testdata.update({k: fromtestdata(v)})
+
+    return testdata
+
+
+@templatingfilter
+def fromjson(value: Optional[Union[list[Any], dict[str, Any], str, int, float]]) -> str:
+    """Convert python object to JSON string.
+
+    Convert any (valid) python object to a JSON string.
+
+    Example:
+    ```gherkin
+    Given value of variable "AtomicCsvReader.test" is "test.csv"
+
+    Then log message "test={{ AtomicCsvReader.test | fromtestdata | fromjson }}"
+    ```
+
+    Args:
+    value (JsonSerializable): value to convert to JSON string
+
+    """
+    return json.dumps(value)

grizzly/testdata/utils.py

@@ -6,9 +6,8 @@
 from collections import namedtuple
 from os import environ
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional, cast
+from typing import TYPE_CHECKING, Any, Optional, cast
 
-from jinja2.filters import FILTERS
 from jinja2.meta import find_undeclared_variables
 
 from grizzly.testdata.ast import get_template_variables
@@ -188,7 +187,7 @@ def read_file(value: str) -> str:
 
 
 def resolve_variable(
-        scenario: GrizzlyContextScenario, value: str, *, guess_datatype: bool = True, only_grizzly: bool = False,
+        scenario: GrizzlyContextScenario, value: str, *, guess_datatype: bool = True, try_template: bool = True, try_file: bool = True,
 ) -> GrizzlyVariableType:
     """Resolve a value to its actual value, since it can be a jinja2 template or any dollar reference. Return type can be actual type of the value."""
     if len(value) < 1:
@@ -199,10 +198,10 @@ def resolve_variable(
         quote_char = value[0]
         value = value[1:-1]
 
-    if is_file(value):
+    if try_file and is_file(value):
         value = read_file(value)
 
-    if has_template(value) and not only_grizzly:
+    if try_template and has_template(value):
         value = resolve_template(scenario, value)
 
     if '$conf' in value or '$env' in value:
@@ -218,18 +217,3 @@ def resolve_variable(
     return resolved_variable
 
 
-class templatingfilter:
-    name: str
-
-    def __init__(self, func: Callable) -> None:
-        name = func.__name__
-        existing_filter = FILTERS.get(name, None)
-
-        if existing_filter is None:
-            FILTERS[name] = func
-        elif existing_filter is not func:
-            message = f'{name} is already registered as a filter'
-            raise AssertionError(message)
-        else:
-            # code executed twice, so adding the same filter again
-            pass

grizzly/testdata/variables/__init__.py

@@ -191,6 +191,7 @@ def destroy_variables() -> None:
 from .date import AtomicDate
 from .directory_contents import AtomicDirectoryContents
 from .integer_incrementer import AtomicIntegerIncrementer
+from .json_reader import AtomicJsonReader
 from .random_integer import AtomicRandomInteger
 from .random_string import AtomicRandomString
 
@@ -201,6 +202,7 @@ def destroy_variables() -> None:
     'AtomicDirectoryContents',
     'AtomicCsvReader',
     'AtomicCsvWriter',
+    'AtomicJsonReader',
     'AtomicRandomString',
     'destroy_variables',
 ]