Merge branch 'main' into main

Signed-off-by: Merel Theisen <[email protected]>

RELEASE.md

@@ -4,19 +4,24 @@
 * Implemented `KedroDataCatalog.to_config()` method that converts the catalog instance into a configuration format suitable for serialization.
 * Improve OmegaConfigLoader performance.
 * Replaced `trufflehog` with `detect-secrets` for detecting secrets within a code base.
+* Added support for `%load_ext kedro`.
 
 ## Bug fixes and other changes
 * Added validation to ensure dataset versions consistency across catalog.
 * Fixed a bug in project creation when using a custom starter template offline.
 * Added `node` import to the pipeline template.
 * Update error message when executing kedro run without pipeline.
 * Safeguard hooks when user incorrectly registers a hook class in settings.py.
+* Fixed parsing paths with query and fragment.
+* Remove lowercase transformation in regex validation.
 * Moved `kedro-catalog` JSON schema to `kedro-datasets`.
 
 ## Breaking changes to the API
 ## Documentation changes
+
 ## Community contributions
 Many thanks to the following Kedroids for contributing PRs to this release:
+* [Hendrik Scherner](https://github.com/SchernHe)
 * [Chris Schopp](https://github.com/chrisschopp)
 
 # Release 0.19.10

docs/source/nodes_and_pipelines/run_a_pipeline.md

@@ -233,99 +233,6 @@ Out[11]: {'v': 0.666666666666667}
 </details>
 
 
-
-## Output to a file
-
-We can also use IO to save outputs to a file. In this example, we define a custom `LambdaDataset` that would serialise the output to a file locally:
-
-```{warning}
-`LambdaDataset` has been deprecated and will be removed in Kedro `0.20.0`.
-```
-
-<details>
-<summary><b>Click to expand</b></summary>
-
-
-```python
-def save(value):
-    with open("./data/07_model_output/variance.pickle", "wb") as f:
-        pickle.dump(value, f)
-
-
-def load():
-    with open("./data/07_model_output/variance.pickle", "rb") as f:
-        return pickle.load(f)
-
-
-pickler = LambdaDataset(load=load, save=save)
-io.add("v", pickler)
-```
-</details>
-
-It is important to make sure that the data catalog variable name `v` matches the name `v` in the pipeline definition.
-
-Next we can confirm that this `LambdaDataset` behaves correctly:
-
-<details>
-<summary><b>Click to expand</b></summary>
-
-```python
-io.save("v", 5)
-```
-
-```python
-io.load("v")
-```
-
-`Ouput`:
-
-```Console
-Out[12]: 5
-```
-</details>
-
-Finally, let's run the pipeline again now and serialise the output:
-
-<details>
-<summary><b>Click to expand</b></summary>
-
-
-```python
-SequentialRunner().run(pipeline, catalog=io)
-```
-
-`Ouput`:
-
-```console
-Out[13]: {}
-```
-</details>
-
-The output has been persisted to a local file so we don't see it directly, but it can be retrieved from the catalog:
-
-<details>
-<summary><b>Click to expand</b></summary>
-
-
-```python
-io.load("v")
-```
-
-`Ouput`:
-
-```console
-Out[14]: 0.666666666666667
-```
-
-```python
-try:
-    os.remove("./data/07_model_output/variance.pickle")
-except FileNotFoundError:
-    pass
-```
-</details>
-
-
 ## Configure `kedro run` arguments
 
 The [Kedro CLI documentation](../development/commands_reference.md#run-the-project) lists the available CLI options for `kedro run`. You can alternatively supply a configuration file that contains the arguments to `kedro run`.

kedro/__init__.py

@@ -3,6 +3,13 @@
 configuration and pipeline assembly.
 """
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from IPython.core.interactiveshell import InteractiveShell
+
 import sys
 import warnings
 
@@ -29,3 +36,9 @@ class KedroPythonVersionWarning(UserWarning):
 or set the PYTHONWARNINGS environment variable accordingly.""",
         KedroPythonVersionWarning,
     )
+
+
+def load_ipython_extension(ipython: InteractiveShell) -> None:
+    import kedro.ipython
+
+    kedro.ipython.load_ipython_extension(ipython)

kedro/framework/cli/starters.py

@@ -1005,8 +1005,7 @@ def __str__(self) -> str:
 
     def validate(self, user_input: str) -> None:
         """Validate a given prompt value against the regex validator"""
-
-        if self.regexp and not re.match(self.regexp, user_input.lower()):
+        if self.regexp and not re.match(self.regexp, user_input):
             message = f"'{user_input}' is an invalid value for {(self.title).lower()}."
             click.secho(message, fg="red", err=True)
             click.secho(self.error_message, fg="red", err=True)

kedro/io/core.py

@@ -901,6 +901,11 @@ def _parse_filepath(filepath: str) -> dict[str, str]:
         if windows_path:
             path = ":".join(windows_path.groups())
 
+    if parsed_path.query:
+        path = f"{path}?{parsed_path.query}"
+    if parsed_path.fragment:
+        path = f"{path}#{parsed_path.fragment}"
+
     options = {"protocol": protocol, "path": path}
 
     if parsed_path.netloc and protocol in CLOUD_PROTOCOLS:

kedro/ipython/__init__.py

@@ -19,6 +19,8 @@
 if TYPE_CHECKING:
     from collections import OrderedDict
 
+    from IPython.core.interactiveshell import InteractiveShell
+
 from IPython.core.getipython import get_ipython
 from IPython.core.magic import needs_local_scope, register_line_magic
 from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
@@ -50,16 +52,16 @@
 RICH_INSTALLED = True if importlib.util.find_spec("rich") is not None else False
 
 
-def load_ipython_extension(ipython: Any) -> None:
+def load_ipython_extension(ipython: InteractiveShell) -> None:
     """
     Main entry point when %load_ext kedro.ipython is executed, either manually or
     automatically through `kedro ipython` or `kedro jupyter lab/notebook`.
     IPython will look for this function specifically.
     See https://ipython.readthedocs.io/en/stable/config/extensions/index.html
     """
-    ipython.register_magic_function(magic_reload_kedro, magic_name="reload_kedro")
+    ipython.register_magic_function(func=magic_reload_kedro, magic_name="reload_kedro")  # type: ignore[call-arg]
     logger.info("Registered line magic '%reload_kedro'")
-    ipython.register_magic_function(magic_load_node, magic_name="load_node")
+    ipython.register_magic_function(func=magic_load_node, magic_name="load_node")  # type: ignore[call-arg]
     logger.info("Registered line magic '%load_node'")
 
     if _find_kedro_project(Path.cwd()) is None:

kedro/templates/project/prompts.yml

@@ -26,7 +26,7 @@ tools:
     7) Kedro-Viz: Kedro's native visualisation tool
 
     Which tools would you like to include in your project? [1-7/1,3/all/none]:
-  regex_validator: "^(all|none|(( )*\\d+( *- *\\d+)?( *, *\\d+( *- *\\d+)?)*( )*)?)$"
+  regex_validator: "(?i)^(all|none|(( )*\\d+( *- *\\d+)?( *, *\\d+( *- *\\d+)?)*( )*)?)$"
   error_message: |
     Invalid input. Please select valid options for project tools using comma-separated values, ranges, or 'all/none'.
 

pyproject.toml

@@ -83,7 +83,7 @@ test = [
 docs = [
     "ipykernel>=5.3, <7.0",
     "Jinja2<3.2.0",
-    "kedro-sphinx-theme==2024.10.2",
+    "kedro-sphinx-theme==2024.10.3",
     "sphinx-notfound-page!=1.0.3",  # Required by kedro-sphinx-theme. 1.0.3 results in `AttributeError: 'tuple' object has no attribute 'default'`.
 ]
 jupyter = [
@@ -178,7 +178,8 @@ layers = [
 ]
 ignore_imports = [
     "kedro.runner.task -> kedro.framework.project",
-    "kedro.framework.hooks.specs -> kedro.framework.context"
+    "kedro.framework.hooks.specs -> kedro.framework.context",
+    "kedro -> kedro.ipython"
 ]
 
 [[tool.importlinter.contracts]]
@@ -188,6 +189,9 @@ modules = [
     "kedro.pipeline",
     "kedro.io"
 ]
+ignore_imports = [
+    "kedro -> kedro.ipython"
+]
 
 [[tool.importlinter.contracts]]
 name = "Config cannot import Runner et al"

tests/framework/cli/test_starters.py

@@ -422,19 +422,27 @@ def test_empty_prompts(self, fake_kedro_cli):
         _assert_template_ok(result)
         _clean_up_project(Path("./new-kedro-project"))
 
-    def test_custom_prompt_valid_input(self, fake_kedro_cli):
+    @pytest.mark.parametrize(
+        "regex, valid_value",
+        [
+            ("^\\w+(-*\\w+)*$", "my-value"),
+            ("^[A-Z_]+", "MY-VALUE"),
+            ("^\\d+$", "123"),
+        ],
+    )
+    def test_custom_prompt_valid_input(self, fake_kedro_cli, regex, valid_value):
         shutil.copytree(TEMPLATE_PATH, "template")
         _write_yaml(
             Path("template") / "prompts.yml",
             {
                 "project_name": {"title": "Project Name"},
                 "custom_value": {
                     "title": "Custom Value",
-                    "regex_validator": "^\\w+(-*\\w+)*$",
+                    "regex_validator": regex,
                 },
             },
         )
-        custom_input = "\n".join(["my-project", "My Project"])
+        custom_input = "\n".join([valid_value, "My Project"])
         result = CliRunner().invoke(
             fake_kedro_cli,
             ["new", "--starter", "template"],
@@ -446,6 +454,7 @@ def test_custom_prompt_valid_input(self, fake_kedro_cli):
             repo_name="my-project",
             python_package="my_project",
         )
+
         _clean_up_project(Path("./my-project"))
 
     def test_custom_prompt_for_essential_variable(self, fake_kedro_cli):

tests/framework/session/test_session_extension_hooks.py

@@ -241,7 +241,7 @@ def test_before_and_after_node_run_hooks_sequential_runner(
     def test_on_node_error_hook_parallel_runner(self, mock_session, logs_listener):
         with pytest.raises(ValueError, match="broken"):
             mock_session.run(
-                runner=ParallelRunner(max_workers=2), node_names=["node1", "node2"]
+                runner=ParallelRunner(max_workers=1), node_names=["node1", "node2"]
             )
 
         on_node_error_records = [

tests/io/test_core.py

@@ -271,6 +271,24 @@ def test_get_filepath_str(self):
             ("file:///C:\\Projects\\file.txt", ("file", "C:\\Projects\\file.txt")),
             ("https://example.com/file.txt", ("https", "example.com/file.txt")),
             ("http://example.com/file.txt", ("http", "example.com/file.txt")),
+            (
+                "https://example.com/search?query=books&category=fiction#reviews",
+                ("https", "example.com/search?query=books&category=fiction#reviews"),
+            ),
+            (
+                "https://example.com/search#reviews",
+                ("https", "example.com/search#reviews"),
+            ),
+            (
+                "http://example.com/search?query=books&category=fiction",
+                ("http", "example.com/search?query=books&category=fiction"),
+            ),
+            (
+                "s3://some/example?query=query#filename",
+                ("s3", "some/example?query=query#filename"),
+            ),
+            ("s3://some/example#filename", ("s3", "some/example#filename")),
+            ("s3://some/example?query=query", ("s3", "some/example?query=query")),
         ],
     )
     def test_get_protocol_and_path(self, filepath, expected_result):

tests/ipython/test_ipython.py

@@ -222,6 +222,17 @@ def test_line_magic_with_invalid_arguments(self, mocker, ipython):
         ):
             ipython.magic("reload_kedro --invalid_arg=dummy")
 
+    def test_ipython_kedro_extension_alias(self, mocker, ipython):
+        mock_ipython_extension = mocker.patch(
+            "kedro.ipython.load_ipython_extension", autospec=True
+        )
+        # Ensure that `kedro` is not loaded initially
+        assert "kedro" not in ipython.extension_manager.loaded
+        ipython.magic("load_ext kedro")
+        mock_ipython_extension.assert_called_once_with(ipython)
+        # Ensure that `kedro` extension has been loaded
+        assert "kedro" in ipython.extension_manager.loaded
+
 
 class TestProjectPathResolution:
     def test_only_path_specified(self):