Skip to content

Commit

Permalink
Make e2e tests pull main branch of kedro-starters when using `too…
Browse files Browse the repository at this point in the history 
…ls` (#3518)

* attempt to fix mismatch version of kedro-starters and kedro for unit tests

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* fix for e2e

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* revert unit test changes

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* lint

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* test with env variables

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* make cookiecutter_args fixture on classes

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* Update e2e-tests.yml

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* Update cli_steps.py

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* simplify e2e test for tools

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* set example_pipeline to "n"

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* undo version update for testing

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* fix merge

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* lint

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* update check_created_project_structure_from_tools for shortnames

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* lint

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* changes based on review

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

* revert exit code change

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>

---------

Signed-off-by: Sajid Alam <sajid_alam@mckinsey.com>
Signed-off-by: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com>
@SajidAlamQB
SajidAlamQB authored Jan 18, 2024
1 parent d6a78f4 commit 3d14cea
Showing 3 changed files with 26 additions and 64 deletions.
22 changes: 12 additions & 10 deletions features/steps/cli_steps.py
View file
Original file line number Diff line number Diff line change
@@ -186,7 +186,7 @@ def create_config_file_without_starter(context):
def create_config_file_with_tools(context, tools):
"""Behave step to create a temporary config file
(given the existing temp directory) and store it in the context.
It takes a custom tools list and sets example prompt to `y`.
It takes a custom tools list and sets example prompt to `n`.
"""

context.config_file = context.temp_dir / "config.yml"
@@ -195,7 +195,7 @@ def create_config_file_with_tools(context, tools):
context.package_name = context.project_name.replace("-", "_")
config = {
"tools": tools,
"example_pipeline": "y",
"example_pipeline": "n",
"project_name": context.project_name,
"repo_name": context.project_name,
"output_dir": str(context.temp_dir),
@@ -512,28 +512,30 @@ def is_created(name):
assert is_created(path), f"{path} does not exist"

tools_list = (
tools.split(",") if tools != "all" else ["1", "2", "3", "4", "5", "6", "7"]
tools.split(",")
if tools != "all"
else ["lint", "test", "log", "docs", "data", "pyspark", "viz"]
)

if "1" in tools_list: # lint tool
if "lint" in tools_list: # lint tool
pass # No files are added

if "2" in tools_list: # test tool
if "test" in tools_list: # test tool
assert is_created("tests"), "tests directory does not exist"

if "3" in tools_list: # log tool
if "log" in tools_list: # log tool
assert is_created("conf/logging.yml"), "logging configuration does not exist"

if "4" in tools_list: # docs tool
if "docs" in tools_list: # docs tool
assert is_created("docs"), "docs directory does not exist"

if "5" in tools_list: # data tool
if "data" in tools_list: # data tool
assert is_created("data"), "data directory does not exist"

if "6" in tools_list: # PySpark tool
if "pyspark" in tools_list: # PySpark tool
assert is_created("conf/base/spark.yml"), "spark.yml does not exist"

if "7" in tools_list: # viz tool
if "viz" in tools_list: # viz tool
expected_reporting_path = Path(
f"src/{context.package_name}/pipelines/reporting"
)
30 changes: 0 additions & 30 deletions features/tools.feature
View file
Original file line number Diff line number Diff line change
@@ -4,38 +4,8 @@ Feature: New Kedro project with tools
Given I have prepared a config file with tools "none"
When I run a non-interactive kedro new without starter
Then the expected tool directories and files should be created with "none"
Given I have installed the project dependencies
When I execute the kedro command "run"
Then I should get a successful exit code

Scenario: Create a new Kedro project with all tools except 'viz' and 'pyspark'
Given I have prepared a config file with tools "lint, test, log, docs, data"
When I run a non-interactive kedro new without starter
Then the expected tool directories and files should be created with "lint, test, log, docs, data"
Given I have installed the project dependencies
When I execute the kedro command "run"
Then I should get a successful exit code

Scenario: Create a new Kedro project with all tools
Given I have prepared a config file with tools "all"
When I run a non-interactive kedro new without starter
Then the expected tool directories and files should be created with "all"
Given I have installed the project dependencies
When I execute the kedro command "run"
Then I should get a successful exit code

Scenario: Create a new Kedro project with only 'pyspark' tool
Given I have prepared a config file with tools "pyspark"
When I run a non-interactive kedro new without starter
Then the expected tool directories and files should be created with "pyspark"
Given I have installed the project dependencies
When I execute the kedro command "run"
Then I should get a successful exit code

Scenario: Create a new Kedro project with only 'viz' tool
Given I have prepared a config file with tools "viz"
When I run a non-interactive kedro new without starter
Then the expected tool directories and files should be created with "viz"
Given I have installed the project dependencies
When I execute the kedro command "run"
Then I should get a successful exit code
38 changes: 14 additions & 24 deletions tests/framework/cli/test_starters.py
View file
Original file line number Diff line number Diff line change
@@ -45,6 +45,14 @@ def mock_cookiecutter(mocker):
return mocker.patch("cookiecutter.main.cookiecutter")


@pytest.fixture
def patch_cookiecutter_args(mocker):
mocker.patch(
"kedro.framework.cli.starters._make_cookiecutter_args_and_fetch_template",
side_effect=mock_make_cookiecutter_args_and_fetch_template,
)


def mock_make_cookiecutter_args_and_fetch_template(*args, **kwargs):
cookiecutter_args, starter_path = _make_cookiecutter_args_and_fetch_template(
*args, **kwargs
@@ -998,7 +1006,7 @@ def test_starter_flag_with_example_flag(self, fake_kedro_cli):
)


@pytest.mark.usefixtures("chdir_to_tmp")
@pytest.mark.usefixtures("chdir_to_tmp", "patch_cookiecutter_args")
class TestToolsAndExampleFromUserPrompts:
@pytest.mark.parametrize(
"tools",
@@ -1026,13 +1034,7 @@ class TestToolsAndExampleFromUserPrompts:
],
)
@pytest.mark.parametrize("example_pipeline", ["Yes", "No"])
def test_valid_tools_and_example(
self, fake_kedro_cli, tools, example_pipeline, mocker
):
mocker.patch(
"kedro.framework.cli.starters._make_cookiecutter_args_and_fetch_template",
side_effect=mock_make_cookiecutter_args_and_fetch_template,
)
def test_valid_tools_and_example(self, fake_kedro_cli, tools, example_pipeline):
result = CliRunner().invoke(
fake_kedro_cli,
["new"],
@@ -1137,7 +1139,7 @@ def test_invalid_example(self, fake_kedro_cli, bad_input):
)


@pytest.mark.usefixtures("chdir_to_tmp")
@pytest.mark.usefixtures("chdir_to_tmp", "patch_cookiecutter_args")
class TestToolsAndExampleFromConfigFile:
@pytest.mark.parametrize(
"tools",
@@ -1167,15 +1169,8 @@ class TestToolsAndExampleFromConfigFile:
],
)
@pytest.mark.parametrize("example_pipeline", ["Yes", "No"])
def test_valid_tools_and_example(
self, fake_kedro_cli, tools, example_pipeline, mocker
):
def test_valid_tools_and_example(self, fake_kedro_cli, tools, example_pipeline):
"""Test project created from config."""
mocker.patch(
"kedro.framework.cli.starters._make_cookiecutter_args_and_fetch_template",
side_effect=mock_make_cookiecutter_args_and_fetch_template,
)

config = {
"tools": tools,
"project_name": "New Kedro Project",
@@ -1320,7 +1315,7 @@ def test_invalid_example(self, fake_kedro_cli, bad_input):
)


@pytest.mark.usefixtures("chdir_to_tmp")
@pytest.mark.usefixtures("chdir_to_tmp", "patch_cookiecutter_args")
class TestToolsAndExampleFromCLI:
@pytest.mark.parametrize(
"tools",
@@ -1350,12 +1345,7 @@ class TestToolsAndExampleFromCLI:
],
)
@pytest.mark.parametrize("example_pipeline", ["Yes", "No"])
def test_valid_tools_flag(self, fake_kedro_cli, tools, example_pipeline, mocker):
mocker.patch(
"kedro.framework.cli.starters._make_cookiecutter_args_and_fetch_template",
side_effect=mock_make_cookiecutter_args_and_fetch_template,
)

def test_valid_tools_flag(self, fake_kedro_cli, tools, example_pipeline):
result = CliRunner().invoke(
fake_kedro_cli,
["new", "--tools", tools, "--example", example_pipeline],

0 comments on commit 3d14cea

Please sign in to comment.