diff --git a/.github/actions/install_kedro_and_python_dependencies/action.yml b/.github/actions/install_kedro_and_python_dependencies/action.yml new file mode 100644 index 0000000000..3874ccdc20 --- /dev/null +++ b/.github/actions/install_kedro_and_python_dependencies/action.yml @@ -0,0 +1,15 @@ +name: Install Kedro and other Python Dependencies +description: Installs Kedro from the main branch and other Python dependencies, then prints the Python version and installed packages. +runs: + using: composite + steps: + - name: Install Python dependencies + run: |- + pip install git+https://github.com/kedro-org/kedro@main + pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U + shell: bash + - name: Echo package versions + run: |- + python -V + pip freeze + shell: bash \ No newline at end of file diff --git a/.github/actions/install_node_dependencies/action.yml b/.github/actions/install_node_dependencies/action.yml new file mode 100644 index 0000000000..b36a41e83a --- /dev/null +++ b/.github/actions/install_node_dependencies/action.yml @@ -0,0 +1,37 @@ +name: Setup Node.js and Install Dependencies +description: Sets up a specific Node.js version, caches Node modules, and installs Node dependencies. + +inputs: + node-version: + description: 'Node.js version' + required: false + default: '18.20.0' + + package-path: + description: 'Path to package.json file' + required: false + default: '.' + +runs: + using: composite + steps: + - name: Setup Node.js + uses: actions/setup-node@v4.0.0 + with: + node-version: ${{ inputs.node-version }} + + - name: Get NPM Cache Directory + id: npm-cache-dir + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Node.js packages + uses: actions/cache@v2 + with: + path: "${{ steps.npm-cache-dir.outputs.dir }}" + key: "${{ runner.os }}-node-${{ hashFiles(format('{0}/package-lock.json', inputs.package-path)) }}" + restore-keys: "${{ runner.os }}-node-" + + - name: Install Node Dependencies + run: npm install + shell: bash diff --git a/.github/actions/setup_tests/action.yml b/.github/actions/setup_tests/action.yml new file mode 100644 index 0000000000..4de29e785a --- /dev/null +++ b/.github/actions/setup_tests/action.yml @@ -0,0 +1,46 @@ +name: Setup Tests +description: Sets up the testing environment by setting up Python and Node.js, caching Python packages, installing Kedro and other Python dependencies, and building the React application. + +inputs: + os: + description: 'Operating system' + required: false + default: 'ubuntu-latest' + python-version: + description: 'Python version' + required: false + default: '3.9' + +runs: + using: "composite" + steps: + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + shell: bash \ No newline at end of file diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..4c981d6c8b --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,44 @@ +name: Run all checks on Kedro-Viz +# Runs end-to-end tests, unit tests, linting and JavaScript +# linting & tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + workflow_dispatch: + schedule: + # Run every day at 1:00 AM(UTC time) + - cron: 0 1 * * * +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + javascript_lint_and_tests: + uses: ./.github/workflows/javascript-lint-and-tests.yml diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..70a8691854 --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +name: Build backend +# Runs end-to-end tests, unit tests, and linting on the backend code +# for different operating systems and Python versions. + +on: + push: + paths: + - 'package/**' + - '.github/**' + pull_request: + paths: + - 'package/**' + - '.github/**' + workflow_dispatch: +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..5eee3d98cd --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,39 @@ +name: Run e2e tests on Kedro-Viz +# Runs end-to-end tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + e2e_tests: + runs-on: ${{ inputs.os }} + + # below condition checks if the operating system is Ubuntu, or + # if the operating system is Windows and the branch is main/demo + if: > + inputs.os == 'ubuntu-latest' || + ( + ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/demo' + ) && + inputs.os == 'windows-latest' + ) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} + + - name: Run all end to end tests + run: make e2e-tests \ No newline at end of file diff --git a/.github/workflows/javascript-lint-and-tests.yml b/.github/workflows/javascript-lint-and-tests.yml new file mode 100644 index 0000000000..0cff820c9b --- /dev/null +++ b/.github/workflows/javascript-lint-and-tests.yml @@ -0,0 +1,61 @@ +name: Run javascript linters and tests on Kedro-Viz +# Runs JavaScript linting, unit tests, and end-to-end tests on +# Kedro-Viz for ubuntu-latest operating systems and Python 3.9. + +on: + push: + paths-ignore: + - 'package/**' + pull_request: + paths-ignore: + - 'package/**' + workflow_dispatch: + workflow_call: + +jobs: + javascript_lint_and_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python "3.9" + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Cache python packages for Linux + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ubuntu-latest-python-3.9 + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Setup Cypress requirements + run: |- + sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list + sudo apt-get update + sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + + - name: Test lib transpilation + run: npm run lib + + - name: Test JS library imports + run: |- + npm run lib-test:setup + cd tools/test-lib/react-app + npm run test:ci + + - name: Run Eslint + run: npm run lint + + - name: Run JavaScript tests + run: npm run test:ci + + - name: Run Javascript end to end tests + run: npm run cy:ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..02489d36dd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Run linters on Kedro-Viz +# Runs secret scan, security scan, GraphQL schema check, +# and Python formatters and linters on Kedro-Viz for +# different operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Run secret scan + run: make secret-scan + + - name: Run security scan + run: make security-scan + + - name: Verify GraphQL schema is up to date + run: make schema-check + + - name: Run Python formatters and linters + run: make format-check lint-check diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml new file mode 100644 index 0000000000..abc5c013fe --- /dev/null +++ b/.github/workflows/merge-gatekeeper.yml @@ -0,0 +1,26 @@ +name: Merge Gatekeeper + +on: + pull_request: + branches: + - main + +jobs: + merge-gatekeeper: + runs-on: ubuntu-latest + # Restrict permissions of the GITHUB_TOKEN. + # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs + permissions: + checks: read + statuses: read + steps: + - name: Run Merge Gatekeeper + # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs: + # https://github.com/upsidr/merge-gatekeeper/tags + # https://github.com/upsidr/merge-gatekeeper/branches + uses: upsidr/merge-gatekeeper@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + timeout: 3600 + interval: 30 + ignored: 'ci/circleci: win_unit_tests-3.9,ci/circleci: win_unit_tests-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: unit_tests-3.9,ci/circleci: win_e2e_tests-3.10,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_e2e_tests-3.8,ci/circleci: e2e_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: lint-3.9,ci/circleci: lint-3.10,ci/circleci: javascript_lint_and_tests,ci/circleci: check-updated-files,ci/circleci: all_circleci_checks_succeeded' \ No newline at end of file diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..fe970683c3 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,39 @@ +name: Run unit-tests on Kedro-Viz +# Runs unit tests on Kedro-Viz across different +# OS and Python versions after environment setup. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit_tests: + runs-on: ${{ inputs.os }} + + # below condition checks if the operating system is Ubuntu, or + # if the operating system is Windows and the branch is main/demo + if: > + inputs.os == 'ubuntu-latest' || + ( + ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/demo' + ) && + inputs.os == 'windows-latest' + ) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} + + - name: Run Python tests + run: make pytest \ No newline at end of file diff --git a/RELEASE.md b/RELEASE.md index b81494519d..e9dc534e79 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,7 +10,9 @@ Please follow the established format: ## Major features and improvements ## Bug fixes and other changes + - Upgrade the gitpod workspace-full to a newer version which includes both Node 18 and Python 3.11.5. (#1862) +- Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) # Release 9.0.0 diff --git a/cypress/tests/ui/toolbar/global-toolbar.cy.js b/cypress/tests/ui/toolbar/global-toolbar.cy.js index 95f07b127d..a0bcc80e91 100644 --- a/cypress/tests/ui/toolbar/global-toolbar.cy.js +++ b/cypress/tests/ui/toolbar/global-toolbar.cy.js @@ -146,38 +146,24 @@ describe('Global Toolbar', () => { }); }); - it('verifies that users can expand all modular pipelines on first load. #TC-7', () => { + it('verifies that users can expand all modular pipelines directly from the toolbar. #TC-7', () => { const modularPipelineChildNodeText = 'Create Derived Features'; - // Alias - cy.get('[data-test="pipeline-toggle-input-expandAllPipelines"]').as( - 'isExpandAllPipelinesCheckBox' - ); + // Alias for better readability + cy.get('[data-test="expand-all-pipelines-toggle"]').as('expandAllPipelinesToggle'); // Assert before action - cy.get('@isExpandAllPipelinesCheckBox').should('not.be.checked'); - cy.get('.pipeline-node__text').should( - 'not.contain', - modularPipelineChildNodeText - ); - cy.get('[role="treeitem"]') - .should('have.attr', 'aria-expanded') - .should('eq', 'false'); + cy.get('@expandAllPipelinesToggle').should('not.be.checked'); + cy.get('.pipeline-node__text').should('not.contain', modularPipelineChildNodeText); + cy.get('[role="treeitem"]').should('have.attr', 'aria-expanded', 'false'); - // Action - cy.get('@isExpandAllPipelinesCheckBox').check({ force: true }); - cy.get('[data-test="Apply changes and close in Settings Modal"]').click({ - force: true, - }); + // Action - toggling the expand all pipelines directly from the toolbar + cy.get('@expandAllPipelinesToggle').click(); // Assert after action - cy.get('[role="treeitem"]', { timeout: 5000 }) - .should('have.attr', 'aria-expanded') - .should('eq', 'true'); - cy.get('.pipeline-node__text').should( - 'contain', - modularPipelineChildNodeText - ); + cy.get('[role="treeitem"]') + .should('have.attr', 'aria-expanded', 'true'); + cy.get('.pipeline-node__text').should('contain', modularPipelineChildNodeText); }); }); }); diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index f3506d74fb..4662951781 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -3,7 +3,7 @@ load data from projects created in a range of Kedro versions. """ -# pylint: disable=import-outside-toplevel, protected-access +# pylint: disable=protected-access import json import logging @@ -11,8 +11,10 @@ from typing import Any, Dict, Optional, Tuple from kedro import __version__ +from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession from kedro.framework.session.store import BaseSessionStore +from kedro.framework.startup import bootstrap_project from kedro.io import DataCatalog from kedro.pipeline import Pipeline @@ -69,14 +71,16 @@ def load_data( project_path: Path, env: Optional[str] = None, include_hooks: bool = False, + package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, ) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: """Load data from a Kedro project. Args: - project_path: the path whether the Kedro project is located. + project_path: the path where the Kedro project is located. env: the Kedro environment to load the data. If not provided. it will use Kedro default, which is local. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project @@ -85,10 +89,11 @@ def load_data( A tuple containing the data catalog and the pipeline dictionary and the session store. """ - from kedro.framework.project import pipelines - from kedro.framework.startup import bootstrap_project - - bootstrap_project(project_path) + if package_name: + configure_project(package_name) + else: + # bootstrap project when viz is run in dev mode + bootstrap_project(project_path) with KedroSession.create( project_path=project_path, diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index a97d01e60c..87fa5b2676 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -9,6 +9,7 @@ from click_default_group import DefaultGroup from kedro.framework.cli.project import PARAMS_ARG_HELP from kedro.framework.cli.utils import KedroCliError, _split_params +from kedro.framework.project import PACKAGE_NAME from packaging.version import parse from watchgod import RegExpWatcher, run_process @@ -153,6 +154,7 @@ def run( "env": env, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, } if autoreload: @@ -268,6 +270,7 @@ def create_shareableviz_process( endpoint, bucket_name, include_hooks, + PACKAGE_NAME, process_completed, exception_queue, ), @@ -338,11 +341,19 @@ def create_shareableviz_process( def load_and_deploy_viz( - platform, endpoint, bucket_name, include_hooks, process_completed, exception_queue + platform, + endpoint, + bucket_name, + include_hooks, + package_name, + process_completed, + exception_queue, ): """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: - load_and_populate_data(Path.cwd(), include_hooks=include_hooks) + load_and_populate_data( + Path.cwd(), include_hooks=include_hooks, package_name=package_name + ) # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) diff --git a/package/kedro_viz/launchers/jupyter.py b/package/kedro_viz/launchers/jupyter.py index d44493f424..f51f6ce7eb 100644 --- a/package/kedro_viz/launchers/jupyter.py +++ b/package/kedro_viz/launchers/jupyter.py @@ -1,6 +1,7 @@ """`kedro_viz.launchers.jupyter` provides line_magic to launch the viz server from a jupyter notebook. """ + # pragma: no cover import logging import multiprocessing @@ -12,6 +13,7 @@ import IPython from IPython.display import HTML, display +from kedro.framework.project import PACKAGE_NAME from watchgod import RegExpWatcher, run_process from kedro_viz.launchers.utils import _check_viz_up, _wait_for @@ -140,6 +142,7 @@ def run_viz( # pylint: disable=too-many-locals "env": env, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, "project_path": project_path, } diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 2c5b205576..eb31f7a9c9 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -53,14 +53,19 @@ def load_and_populate_data( path: Path, env: Optional[str] = None, include_hooks: bool = False, - extra_params: Optional[Dict[str, Any]] = None, + package_name: Optional[str] = None, pipeline_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, ): """Loads underlying Kedro project data and populates Kedro Viz Repositories""" # Loads data from underlying Kedro Project catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data( - path, env, include_hooks, extra_params + path, + env, + include_hooks, + package_name, + extra_params, ) pipelines = ( @@ -83,6 +88,7 @@ def run_server( project_path: Optional[str] = None, autoreload: bool = False, include_hooks: bool = False, + package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file @@ -101,15 +107,24 @@ def run_server( project_path: the optional path of the Kedro project that contains the pipelines to visualise. If not supplied, the current working directory will be used. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. """ + path = Path(project_path) if project_path else Path.cwd() if load_file is None: - load_and_populate_data(path, env, include_hooks, extra_params, pipeline_name) + load_and_populate_data( + path, + env, + include_hooks, + package_name, + pipeline_name, + extra_params, + ) if save_file: save_api_responses_to_fs(save_file, fsspec.filesystem("file")) @@ -127,8 +142,6 @@ def run_server( if __name__ == "__main__": # pragma: no cover import argparse - from kedro.framework.startup import bootstrap_project - parser = argparse.ArgumentParser(description="Launch a development viz server") parser.add_argument("project_path", help="Path to a Kedro project") parser.add_argument( @@ -140,7 +153,6 @@ def run_server( args = parser.parse_args() project_path = (Path.cwd() / args.project_path).absolute() - bootstrap_project(project_path) run_process_kwargs = { "path": project_path, diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 83baeb917f..df53b70ca5 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -87,6 +87,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -101,6 +102,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -120,6 +122,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -150,6 +153,7 @@ def mock_project_path(mocker): "env": "local", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {"extra_param": "param"}, }, ), @@ -164,6 +168,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": {}, }, ), @@ -284,6 +289,7 @@ def test_kedro_viz_command_with_autoreload( "autoreload": True, "project_path": mock_project_path, "include_hooks": False, + "package_name": None, "extra_params": {}, }, "watcher_cls": RegExpWatcher, @@ -579,6 +585,7 @@ def test_create_shareableviz_process( endpoint, bucket_name, include_hooks, + None, mock_process_completed.return_value, mock_exception_queue.return_value, ), @@ -614,22 +621,24 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, endpoint, bucket_name, include_hooks", + "platform, endpoint, bucket_name, include_hooks, package_name", [ ( "azure", "https://example-bucket.web.core.windows.net", "example-bucket", False, + "demo_project", ), ( "aws", "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", True, + "demo_project", ), - ("gcp", "http://34.120.87.227/", "example-bucket", False), - ("local", None, None, True), + ("gcp", "http://34.120.87.227/", "example-bucket", False, "demo_project"), + ("local", None, None, True, "demo_project"), ], ) def test_load_and_deploy_viz_success( @@ -637,6 +646,7 @@ def test_load_and_deploy_viz_success( endpoint, bucket_name, include_hooks, + package_name, mock_DeployerFactory, mock_load_and_populate_data, mock_process_completed, @@ -651,12 +661,13 @@ def test_load_and_deploy_viz_success( endpoint, bucket_name, include_hooks, + package_name, mock_process_completed, mock_exception_queue, ) mock_load_and_populate_data.assert_called_once_with( - mock_project_path, include_hooks=include_hooks + mock_project_path, include_hooks=include_hooks, package_name=package_name ) mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name diff --git a/package/tests/test_launchers/test_jupyter.py b/package/tests/test_launchers/test_jupyter.py index af41f9b884..dd489778ca 100644 --- a/package/tests/test_launchers/test_jupyter.py +++ b/package/tests/test_launchers/test_jupyter.py @@ -35,6 +35,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) @@ -59,6 +60,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": "", }, ) @@ -105,6 +107,7 @@ def test_run_viz_on_databricks(self, mocker, patched_check_viz_up, monkeypatch): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) diff --git a/src/components/app/app.js b/src/components/app/app.js index 6948da6558..ba70cfb8af 100644 --- a/src/components/app/app.js +++ b/src/components/app/app.js @@ -97,6 +97,7 @@ App.propTypes = { labelBtn: PropTypes.bool, layerBtn: PropTypes.bool, exportBtn: PropTypes.bool, + pipelineBtn: PropTypes.bool, sidebar: PropTypes.bool, }), /** diff --git a/src/components/experiment-tracking/run-details-modal/run-details-modal.scss b/src/components/experiment-tracking/run-details-modal/run-details-modal.scss index ed6b438fbc..c43b8b4dae 100644 --- a/src/components/experiment-tracking/run-details-modal/run-details-modal.scss +++ b/src/components/experiment-tracking/run-details-modal/run-details-modal.scss @@ -18,3 +18,15 @@ text-align: right; width: 100%; } + +.version-reminder-and-run-details-button-wrapper { + align-items: baseline; + display: flex; + justify-content: space-between; + width: 100%; + margin-top: 50px; + + .button:first-of-type { + margin-right: 20px; + } +} diff --git a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js index 2d7932fc57..844fc7ec90 100644 --- a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js +++ b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.js @@ -5,13 +5,18 @@ import { toggleLayers, toggleSidebar, toggleTextLabels, + changeFlag, } from '../../actions'; +import { loadInitialPipelineData } from '../../actions/pipelines'; import IconButton from '../ui/icon-button'; import LabelIcon from '../icons/label'; import ExportIcon from '../icons/export'; import LayersIcon from '../icons/layers'; import PrimaryToolbar from '../primary-toolbar'; import { getVisibleLayerIDs } from '../../selectors/disabled'; +import ExpandPipelinesIcon from '../icons/expand-pipelines'; +import CollapsePipelinesIcon from '../icons/collapse-pipelines'; +import { useGeneratePathname } from '../../utils/hooks/use-generate-pathname'; /** * Main controls for filtering the chart data @@ -28,47 +33,76 @@ export const FlowchartPrimaryToolbar = ({ textLabels, visible, visibleLayers, -}) => ( - <> - - onToggleTextLabels(!textLabels)} - visible={visible.labelBtn} - /> - onToggleLayers(!visibleLayers)} - visible={visible.layerBtn} - /> - onToggleExportModal(true)} - visible={visible.exportBtn} - /> - - -); + expandedPipelines, + onToggleExpandAllPipelines, +}) => { + const { toSetQueryParam } = useGeneratePathname(); + + const handleToggleExpandAllPipelines = () => { + const isExpanded = !expandedPipelines; + onToggleExpandAllPipelines(isExpanded); + toSetQueryParam('expandAllPipelines', isExpanded.toString()); + }; + + return ( + <> + + onToggleTextLabels(!textLabels)} + visible={visible.labelBtn} + /> + onToggleLayers(!visibleLayers)} + visible={visible.layerBtn} + /> + + onToggleExportModal(true)} + visible={visible.exportBtn} + /> + + + ); +}; export const mapStateToProps = (state) => ({ disableLayerBtn: !state.layer.ids.length, @@ -76,6 +110,7 @@ export const mapStateToProps = (state) => ({ textLabels: state.textLabels, visible: state.visible, visibleLayers: Boolean(getVisibleLayerIDs(state).length), + expandedPipelines: state.flags.expandAllPipelines, }); export const mapDispatchToProps = (dispatch) => ({ @@ -91,6 +126,10 @@ export const mapDispatchToProps = (dispatch) => ({ onToggleTextLabels: (value) => { dispatch(toggleTextLabels(Boolean(value))); }, + onToggleExpandAllPipelines: (isExpanded) => { + dispatch(changeFlag('expandAllPipelines', isExpanded)); + dispatch(loadInitialPipelineData()); + }, }); export default connect( diff --git a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js index f9c332ca2e..732389f33c 100644 --- a/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js +++ b/src/components/flowchart-primary-toolbar/flowchart-primary-toolbar.test.js @@ -6,10 +6,16 @@ import ConnectedFlowchartPrimaryToolbar, { } from './flowchart-primary-toolbar'; import { mockState, setup } from '../../utils/state.mock'; +jest.mock('../../utils/hooks/use-generate-pathname', () => ({ + useGeneratePathname: () => ({ + toSetQueryParam: jest.fn(), + }), +})); + describe('PrimaryToolbar', () => { it('renders without crashing', () => { const wrapper = setup.mount(); - expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(4); + expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(5); }); it('hides all buttons (except menu button) when visible prop is false for each of them', () => { @@ -17,6 +23,7 @@ describe('PrimaryToolbar', () => { labelBtn: false, layerBtn: false, exportBtn: false, + pipelineBtn: false, }; const wrapper = setup.mount(, { visible, @@ -31,7 +38,7 @@ describe('PrimaryToolbar', () => { const wrapper = setup.mount(, { visible, }); - expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(3); + expect(wrapper.find('.pipeline-icon-toolbar__button').length).toBe(4); }); const functionCalls = [ @@ -39,6 +46,7 @@ describe('PrimaryToolbar', () => { ['.pipeline-menu-button--labels', 'onToggleTextLabels'], ['.pipeline-menu-button--export', 'onToggleExportModal'], ['.pipeline-menu-button--layers', 'onToggleLayers'], + ['.pipeline-menu-button--pipeline', 'onToggleExpandAllPipelines'], ]; test.each(functionCalls)( @@ -70,6 +78,7 @@ describe('PrimaryToolbar', () => { settingsModal: expect.any(Boolean), labelBtn: expect.any(Boolean), layerBtn: expect.any(Boolean), + pipelineBtn: expect.any(Boolean), sidebar: expect.any(Boolean), }), visibleLayers: expect.any(Boolean), @@ -113,5 +122,15 @@ describe('PrimaryToolbar', () => { type: 'TOGGLE_TEXT_LABELS', }); }); + + it('onToggleExpandAllPipelines', () => { + const dispatch = jest.fn(); + mapDispatchToProps(dispatch).onToggleExpandAllPipelines(true); + expect(dispatch.mock.calls[0][0]).toEqual({ + name: 'expandAllPipelines', + type: 'CHANGE_FLAG', + value: true, + }); + }); }); }); diff --git a/src/components/global-toolbar/global-toolbar.scss b/src/components/global-toolbar/global-toolbar.scss index 5a0cb30971..b117e90442 100644 --- a/src/components/global-toolbar/global-toolbar.scss +++ b/src/components/global-toolbar/global-toolbar.scss @@ -46,6 +46,10 @@ } } +.pipeline-menu-button--pipeline svg { + opacity: 0.7; +} + .pipeline-global-routes-toolbar a.active .pipeline-menu-button--link { background-color: var(--color-global-toolbar-active-btn); border-right: 1px solid var(--color-border-line); diff --git a/src/components/global-toolbar/global-toolbar.test.js b/src/components/global-toolbar/global-toolbar.test.js index f591299f08..74e9012844 100644 --- a/src/components/global-toolbar/global-toolbar.test.js +++ b/src/components/global-toolbar/global-toolbar.test.js @@ -56,6 +56,7 @@ describe('GlobalToolbar', () => { miniMapBtn: true, modularPipelineFocusMode: null, metadataModal: false, + pipelineBtn: true, settingsModal: false, shareableUrlModal: false, sidebar: true, diff --git a/src/components/icons/collapse-pipelines.js b/src/components/icons/collapse-pipelines.js new file mode 100644 index 0000000000..e23687c878 --- /dev/null +++ b/src/components/icons/collapse-pipelines.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const CollapsePipelinesIcon = ({ className }) => ( + + + +); + +export default CollapsePipelinesIcon; diff --git a/src/components/icons/expand-pipelines.js b/src/components/icons/expand-pipelines.js new file mode 100644 index 0000000000..d280fa2638 --- /dev/null +++ b/src/components/icons/expand-pipelines.js @@ -0,0 +1,9 @@ +import React from 'react'; + +const ExpandPipelinesIcon = ({ className }) => ( + + + +); + +export default ExpandPipelinesIcon; diff --git a/src/components/settings-modal/settings-modal.js b/src/components/settings-modal/settings-modal.js index c3f081f8e3..a4847d04e7 100644 --- a/src/components/settings-modal/settings-modal.js +++ b/src/components/settings-modal/settings-modal.js @@ -8,14 +8,9 @@ import { } from '../../actions'; import { getFlagsState } from '../../utils/flags'; import SettingsModalRow from './settings-modal-row'; -import { - settings as settingsConfig, - localStorageName, - params, -} from '../../config'; +import { settings as settingsConfig, localStorageName } from '../../config'; import { saveLocalStorage } from '../../store/helpers'; import { localStorageKeyFeatureHintsStep } from '../../components/feature-hints/feature-hints'; -import { useGeneratePathname } from '../../utils/hooks/use-generate-pathname'; import Button from '../ui/button'; import Modal from '../ui/modal'; @@ -47,8 +42,6 @@ const SettingsModal = ({ useState(showFeatureHints); const [toggleFlags, setToggleFlags] = useState(flags); - const { toSetQueryParam } = useGeneratePathname(); - useEffect(() => { setShowFeatureHintsValue(showFeatureHints); }, [showFeatureHints]); @@ -66,9 +59,6 @@ const SettingsModal = ({ const updatedFlags = Object.entries(toggleFlags); updatedFlags.map((each) => { const [name, value] = each; - if (name === params.expandAll) { - toSetQueryParam(params.expandAll, value); - } return onToggleFlag(name, value); }); @@ -95,7 +85,6 @@ const SettingsModal = ({ onToggleIsPrettyName, showSettingsModal, toggleFlags, - toSetQueryParam, ]); const resetStateCloseModal = () => { @@ -115,7 +104,6 @@ const SettingsModal = ({ >
-
General
Name
State
@@ -149,9 +137,6 @@ const SettingsModal = ({ } }} /> -
-
-
Experiments
{flagData.map(({ name, value, description }) => ( ))} +
+
{isRunningLocally() ? ( isOutdated ? (
@@ -190,33 +177,34 @@ const SettingsModal = ({
) ) : null} -
-
- - +
+ + +
diff --git a/src/components/ui/toggle/toggle.scss b/src/components/ui/toggle/toggle.scss index cbb1af584e..c7c900600a 100644 --- a/src/components/ui/toggle/toggle.scss +++ b/src/components/ui/toggle/toggle.scss @@ -8,8 +8,8 @@ } .kui-theme--dark { - --color-toggle-on: #{colors.$blue-300}; - --color-toggle-on-bar: #{colors.$ocean-400}; + --color-toggle-on: #{colors.$blue-0}; + --color-toggle-on-bar: #{colors.$blue-0}; --color-toggle-off: #{colors.$white-0}; --color-toggle-off-bar: #{colors.$black-400}; } @@ -75,6 +75,7 @@ .pipeline-toggle-label--checked::before { background-color: var(--color-toggle-on-bar); + opacity: 0.3; } .pipeline-toggle-label--checked::after { diff --git a/src/config.js b/src/config.js index 0dca77dcd1..9de91e6263 100644 --- a/src/config.js +++ b/src/config.js @@ -63,12 +63,6 @@ export const flags = { default: true, icon: '🐳', }, - expandAllPipelines: { - name: 'Expand all modular pipelines', - description: 'Expand all modular pipelines on first load', - default: false, - icon: '🔛', - }, }; export const settings = { diff --git a/src/store/initial-state.js b/src/store/initial-state.js index 5279bdd49b..8bc03021ea 100644 --- a/src/store/initial-state.js +++ b/src/store/initial-state.js @@ -40,6 +40,7 @@ export const createInitialState = () => ({ miniMap: true, miniMapBtn: true, modularPipelineFocusMode: null, + pipelineBtn: true, settingsModal: false, shareableUrlModal: false, sidebar: window.innerWidth > sidebarWidth.breakpoint, diff --git a/src/store/initial-state.test.js b/src/store/initial-state.test.js index 240bfabd54..56b3458e93 100644 --- a/src/store/initial-state.test.js +++ b/src/store/initial-state.test.js @@ -123,6 +123,7 @@ describe('getInitialState', () => { exportBtn: true, labelBtn: true, layerBtn: true, + pipelineBtn: true, }, }); }); diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt index da31b5fafa..470cd17fce 100644 --- a/trufflehog-ignore.txt +++ b/trufflehog-ignore.txt @@ -7,4 +7,5 @@ README.md LAYOUT_ENGINE.md src/utils/random-utils.js src/components/update-reminder/update-reminder-content.js -src/components/feature-hints/feature-hints-content.js \ No newline at end of file +src/components/feature-hints/feature-hints-content.js +cypress/fixtures/graphql/ \ No newline at end of file