From d5a8b83fbe598f1b011252594d2afbb12d7b4a33 Mon Sep 17 00:00:00 2001 From: rashidakanchwala <37628668+rashidakanchwala@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:50:40 +0100 Subject: [PATCH] Refactor backend integration with Kedro by replacing bootstrap_project with configure_project (#1796) * Try remove bootstrap_project() Signed-off-by: Merel Theisen * Try set the package name for spawn process Signed-off-by: Merel Theisen * move configure project before kedro integration Signed-off-by: ravi-kumar-pilla * include package name in jupyter and dev run Signed-off-by: ravi-kumar-pilla * fix tests and lint Signed-off-by: ravi-kumar-pilla * update release note Signed-off-by: ravi-kumar-pilla * revert permission change Signed-off-by: ravi-kumar-pilla * fix PR comments Signed-off-by: ravi-kumar-pilla * working draft with parent project search Signed-off-by: ravi-kumar-pilla * update error message to be consistent with other viz commands Signed-off-by: ravi-kumar-pilla * reverting project not found bug fix Signed-off-by: ravi-kumar-pilla * update release note Signed-off-by: ravi-kumar-pilla --------- Signed-off-by: Merel Theisen Signed-off-by: ravi-kumar-pilla Co-authored-by: Merel Theisen Co-authored-by: ravi-kumar-pilla --- RELEASE.md | 2 ++ .../integrations/kedro/data_loader.py | 17 ++++++++----- package/kedro_viz/launchers/cli.py | 15 ++++++++++-- package/kedro_viz/launchers/jupyter.py | 3 +++ package/kedro_viz/server.py | 24 ++++++++++++++----- package/tests/test_launchers/test_cli.py | 19 +++++++++++---- package/tests/test_launchers/test_jupyter.py | 3 +++ 7 files changed, 65 insertions(+), 18 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index b81494519d..e9dc534e79 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,7 +10,9 @@ Please follow the established format: ## Major features and improvements ## Bug fixes and other changes + - Upgrade the gitpod workspace-full to a newer version which includes both Node 18 and Python 3.11.5. (#1862) +- Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) # Release 9.0.0 diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index f3506d74fb..4662951781 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -3,7 +3,7 @@ load data from projects created in a range of Kedro versions. """ -# pylint: disable=import-outside-toplevel, protected-access +# pylint: disable=protected-access import json import logging @@ -11,8 +11,10 @@ from typing import Any, Dict, Optional, Tuple from kedro import __version__ +from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession from kedro.framework.session.store import BaseSessionStore +from kedro.framework.startup import bootstrap_project from kedro.io import DataCatalog from kedro.pipeline import Pipeline @@ -69,14 +71,16 @@ def load_data( project_path: Path, env: Optional[str] = None, include_hooks: bool = False, + package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, ) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: """Load data from a Kedro project. Args: - project_path: the path whether the Kedro project is located. + project_path: the path where the Kedro project is located. env: the Kedro environment to load the data. If not provided. it will use Kedro default, which is local. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project @@ -85,10 +89,11 @@ def load_data( A tuple containing the data catalog and the pipeline dictionary and the session store. """ - from kedro.framework.project import pipelines - from kedro.framework.startup import bootstrap_project - - bootstrap_project(project_path) + if package_name: + configure_project(package_name) + else: + # bootstrap project when viz is run in dev mode + bootstrap_project(project_path) with KedroSession.create( project_path=project_path, diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index a97d01e60c..87fa5b2676 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -9,6 +9,7 @@ from click_default_group import DefaultGroup from kedro.framework.cli.project import PARAMS_ARG_HELP from kedro.framework.cli.utils import KedroCliError, _split_params +from kedro.framework.project import PACKAGE_NAME from packaging.version import parse from watchgod import RegExpWatcher, run_process @@ -153,6 +154,7 @@ def run( "env": env, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, } if autoreload: @@ -268,6 +270,7 @@ def create_shareableviz_process( endpoint, bucket_name, include_hooks, + PACKAGE_NAME, process_completed, exception_queue, ), @@ -338,11 +341,19 @@ def create_shareableviz_process( def load_and_deploy_viz( - platform, endpoint, bucket_name, include_hooks, process_completed, exception_queue + platform, + endpoint, + bucket_name, + include_hooks, + package_name, + process_completed, + exception_queue, ): """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: - load_and_populate_data(Path.cwd(), include_hooks=include_hooks) + load_and_populate_data( + Path.cwd(), include_hooks=include_hooks, package_name=package_name + ) # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) diff --git a/package/kedro_viz/launchers/jupyter.py b/package/kedro_viz/launchers/jupyter.py index d44493f424..f51f6ce7eb 100644 --- a/package/kedro_viz/launchers/jupyter.py +++ b/package/kedro_viz/launchers/jupyter.py @@ -1,6 +1,7 @@ """`kedro_viz.launchers.jupyter` provides line_magic to launch the viz server from a jupyter notebook. """ + # pragma: no cover import logging import multiprocessing @@ -12,6 +13,7 @@ import IPython from IPython.display import HTML, display +from kedro.framework.project import PACKAGE_NAME from watchgod import RegExpWatcher, run_process from kedro_viz.launchers.utils import _check_viz_up, _wait_for @@ -140,6 +142,7 @@ def run_viz( # pylint: disable=too-many-locals "env": env, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, "project_path": project_path, } diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 2c5b205576..eb31f7a9c9 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -53,14 +53,19 @@ def load_and_populate_data( path: Path, env: Optional[str] = None, include_hooks: bool = False, - extra_params: Optional[Dict[str, Any]] = None, + package_name: Optional[str] = None, pipeline_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, ): """Loads underlying Kedro project data and populates Kedro Viz Repositories""" # Loads data from underlying Kedro Project catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data( - path, env, include_hooks, extra_params + path, + env, + include_hooks, + package_name, + extra_params, ) pipelines = ( @@ -83,6 +88,7 @@ def run_server( project_path: Optional[str] = None, autoreload: bool = False, include_hooks: bool = False, + package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file @@ -101,15 +107,24 @@ def run_server( project_path: the optional path of the Kedro project that contains the pipelines to visualise. If not supplied, the current working directory will be used. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. """ + path = Path(project_path) if project_path else Path.cwd() if load_file is None: - load_and_populate_data(path, env, include_hooks, extra_params, pipeline_name) + load_and_populate_data( + path, + env, + include_hooks, + package_name, + pipeline_name, + extra_params, + ) if save_file: save_api_responses_to_fs(save_file, fsspec.filesystem("file")) @@ -127,8 +142,6 @@ def run_server( if __name__ == "__main__": # pragma: no cover import argparse - from kedro.framework.startup import bootstrap_project - parser = argparse.ArgumentParser(description="Launch a development viz server") parser.add_argument("project_path", help="Path to a Kedro project") parser.add_argument( @@ -140,7 +153,6 @@ def run_server( args = parser.parse_args() project_path = (Path.cwd() / args.project_path).absolute() - bootstrap_project(project_path) run_process_kwargs = { "path": project_path, diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 83baeb917f..df53b70ca5 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -87,6 +87,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -101,6 +102,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -120,6 +122,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -150,6 +153,7 @@ def mock_project_path(mocker): "env": "local", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {"extra_param": "param"}, }, ), @@ -164,6 +168,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": {}, }, ), @@ -284,6 +289,7 @@ def test_kedro_viz_command_with_autoreload( "autoreload": True, "project_path": mock_project_path, "include_hooks": False, + "package_name": None, "extra_params": {}, }, "watcher_cls": RegExpWatcher, @@ -579,6 +585,7 @@ def test_create_shareableviz_process( endpoint, bucket_name, include_hooks, + None, mock_process_completed.return_value, mock_exception_queue.return_value, ), @@ -614,22 +621,24 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, endpoint, bucket_name, include_hooks", + "platform, endpoint, bucket_name, include_hooks, package_name", [ ( "azure", "https://example-bucket.web.core.windows.net", "example-bucket", False, + "demo_project", ), ( "aws", "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", True, + "demo_project", ), - ("gcp", "http://34.120.87.227/", "example-bucket", False), - ("local", None, None, True), + ("gcp", "http://34.120.87.227/", "example-bucket", False, "demo_project"), + ("local", None, None, True, "demo_project"), ], ) def test_load_and_deploy_viz_success( @@ -637,6 +646,7 @@ def test_load_and_deploy_viz_success( endpoint, bucket_name, include_hooks, + package_name, mock_DeployerFactory, mock_load_and_populate_data, mock_process_completed, @@ -651,12 +661,13 @@ def test_load_and_deploy_viz_success( endpoint, bucket_name, include_hooks, + package_name, mock_process_completed, mock_exception_queue, ) mock_load_and_populate_data.assert_called_once_with( - mock_project_path, include_hooks=include_hooks + mock_project_path, include_hooks=include_hooks, package_name=package_name ) mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name diff --git a/package/tests/test_launchers/test_jupyter.py b/package/tests/test_launchers/test_jupyter.py index af41f9b884..dd489778ca 100644 --- a/package/tests/test_launchers/test_jupyter.py +++ b/package/tests/test_launchers/test_jupyter.py @@ -35,6 +35,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) @@ -59,6 +60,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": "", }, ) @@ -105,6 +107,7 @@ def test_run_viz_on_databricks(self, mocker, patched_check_viz_up, monkeypatch): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, )