diff --git a/.github/actions/failed-artifacts-and-slack-notifications/action.yml b/.github/actions/failed-artifacts-and-slack-notifications/action.yml index a0b1ce6de..716a8d686 100644 --- a/.github/actions/failed-artifacts-and-slack-notifications/action.yml +++ b/.github/actions/failed-artifacts-and-slack-notifications/action.yml @@ -7,8 +7,8 @@ runs: - name: Copy failed screenshots shell: bash run: | - mkdir /home/runner/work/vizro/vizro/vizro-core/failed_screenshots/ - cd /home/runner/work/vizro/vizro/vizro-core/ + mkdir ${{ env.PROJECT_PATH }}failed_screenshots/ + cd ${{ env.PROJECT_PATH }} cp *.png failed_screenshots - name: Archive production artifacts @@ -16,7 +16,7 @@ runs: with: name: Failed screenshots path: | - /home/runner/work/vizro/vizro/vizro-core/failed_screenshots/*.png + ${{ env.PROJECT_PATH }}failed_screenshots/*.png - name: Send custom JSON data to Slack id: slack diff --git a/.github/workflows/test-e2e-component-library-vizro-core.yml b/.github/workflows/test-e2e-component-library-vizro-core.yml index 85c08a0e6..bc162f0f3 100644 --- a/.github/workflows/test-e2e-component-library-vizro-core.yml +++ b/.github/workflows/test-e2e-component-library-vizro-core.yml @@ -43,3 +43,4 @@ jobs: env: TESTS_NAME: Vizro e2e component library tests SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + PROJECT_PATH: /home/runner/work/vizro/vizro/vizro-core/ diff --git a/.github/workflows/test-score-vizro-ai.yml b/.github/workflows/test-e2e-dashboard-vizro-ai.yml similarity index 81% rename from .github/workflows/test-score-vizro-ai.yml rename to .github/workflows/test-e2e-dashboard-vizro-ai.yml index 813ecba8c..2298294eb 100644 --- a/.github/workflows/test-score-vizro-ai.yml +++ b/.github/workflows/test-e2e-dashboard-vizro-ai.yml @@ -1,4 +1,4 @@ -name: Score tests for VizroAI +name: e2e dashboard tests for VizroAI defaults: run: @@ -12,9 +12,9 @@ env: FORCE_COLOR: 1 jobs: - test-score-vizro-ai-fork: + test-e2e-dashboard-vizro-ai-fork: if: ${{ github.event.pull_request.head.repo.fork }} - name: test-score-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} + name: test-e2e-dashboard-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -38,9 +38,9 @@ jobs: - name: Passed fork step run: echo "Success!" - test-score-vizro-ai: + test-e2e-dashboard-vizro-ai: if: ${{ ! github.event.pull_request.head.repo.fork }} - name: test-score-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} + name: test-e2e-dashboard-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -72,8 +72,8 @@ jobs: - name: Show dependency tree run: hatch run ${{ matrix.config.hatch-env }}:pip tree - - name: Run vizro-ai score tests with PyPI vizro - run: hatch run ${{ matrix.config.hatch-env }}:test-score + - name: Run vizro-ai e2e dashboard tests with PyPI vizro + run: hatch run ${{ matrix.config.hatch-env }}:test-e2e-dashboard env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} @@ -81,10 +81,10 @@ jobs: BRANCH: ${{ github.head_ref }} PYTHON_VERSION: ${{ matrix.config.python-version }} - - name: Run vizro-ai score tests with local vizro + - name: Run vizro-ai e2e dashboard tests with local vizro run: | hatch run ${{ matrix.config.hatch-env }}:pip install ../vizro-core - hatch run ${{ matrix.config.hatch-env }}:test-score + hatch run ${{ matrix.config.hatch-env }}:test-e2e-dashboard env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} @@ -99,7 +99,7 @@ jobs: with: payload: | { - "text": "Vizro-ai ${{ matrix.config.hatch-env }} score tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + "text": "Vizro-ai ${{ matrix.config.hatch-env }} e2e dashboard tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} @@ -111,10 +111,10 @@ jobs: with: name: Report-${{ matrix.config.python-version }}-${{ matrix.config.label }} path: | - /home/runner/work/vizro/vizro/vizro-ai/tests/score/reports/report*.csv + /home/runner/work/vizro/vizro/vizro-ai/tests/e2e/reports/report*.csv - test-score-vizro-ai-report: - needs: test-score-vizro-ai + test-e2e-dashboard-vizro-ai-report: + needs: test-e2e-dashboard-vizro-ai runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-integration-vizro-ai.yml b/.github/workflows/test-e2e-plot-vizro-ai.yml similarity index 85% rename from .github/workflows/test-integration-vizro-ai.yml rename to .github/workflows/test-e2e-plot-vizro-ai.yml index 95eb14531..5b3e37e16 100644 --- a/.github/workflows/test-integration-vizro-ai.yml +++ b/.github/workflows/test-e2e-plot-vizro-ai.yml @@ -1,4 +1,4 @@ -name: Integration tests for VizroAI +name: e2e plot tests for VizroAI defaults: run: @@ -20,9 +20,9 @@ env: FORCE_COLOR: 1 jobs: - test-integration-vizro-ai-fork: + test-e2e-plot-vizro-ai-fork: if: ${{ github.event.pull_request.head.repo.fork }} - name: test-integration-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} + name: test-e2e-plot-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} runs-on: ${{ matrix.config.os }} strategy: fail-fast: false @@ -69,9 +69,9 @@ jobs: - name: Passed fork step run: echo "Success!" - test-integration-vizro-ai: + test-e2e-plot-vizro-ai: if: ${{ ! github.event.pull_request.head.repo.fork }} - name: test-integration-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} + name: test-e2e-plot-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} runs-on: ${{ matrix.config.os }} strategy: fail-fast: false @@ -126,17 +126,17 @@ jobs: - name: Show dependency tree run: hatch run ${{ matrix.config.hatch-env }}:pip tree - - name: Run vizro-ai integration tests with PyPI vizro - run: hatch run ${{ matrix.config.hatch-env }}:test-integration + - name: Run vizro-ai e2e plot tests with PyPI vizro + run: hatch run ${{ matrix.config.hatch-env }}:test-e2e-plot env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} VIZRO_TYPE: pypi - - name: Run vizro-ai integration tests with local vizro + - name: Run vizro-ai e2e plot tests with local vizro run: | hatch run ${{ matrix.config.hatch-env }}:pip install ../vizro-core - hatch run ${{ matrix.config.hatch-env }}:test-integration + hatch run ${{ matrix.config.hatch-env }}:test-e2e-plot env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} @@ -149,7 +149,7 @@ jobs: with: payload: | { - "text": "Vizro-ai ${{ matrix.config.hatch-env }} integration tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + "text": "Vizro-ai ${{ matrix.config.hatch-env }} e2e plot tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/test-vizro-ai-ui.yml b/.github/workflows/test-vizro-ai-ui.yml new file mode 100644 index 000000000..a727d8365 --- /dev/null +++ b/.github/workflows/test-vizro-ai-ui.yml @@ -0,0 +1,68 @@ +name: tests for VizroAI UI + +defaults: + run: + working-directory: vizro-ai + +on: + push: + branches: [main] + pull_request: + branches: + - main + paths: + - "vizro-ai/**" + - "!vizro-ai/docs/**" + +env: + PYTHONUNBUFFERED: 1 + FORCE_COLOR: 1 + PYTHON_VERSION: "3.12" + +jobs: + test-vizro-ai-ui-fork: + if: ${{ github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Passed fork step + run: echo "Success!" + + test-vizro-ai-ui: + if: ${{ ! github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Hatch + run: pip install hatch + + - name: Show dependency tree + run: hatch run pip tree + + - name: Run VizroAI UI tests + run: hatch run test-vizro-ai-ui + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} + + - name: Send custom JSON data to Slack + id: slack + uses: slackapi/slack-github-action@v1.26.0 + if: failure() + with: + payload: | + { + "text": "VizroAI UI tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/vizro-qa-tests-trigger.yml b/.github/workflows/vizro-qa-tests-trigger.yml index 01e539abf..61a1b4efa 100644 --- a/.github/workflows/vizro-qa-tests-trigger.yml +++ b/.github/workflows/vizro-qa-tests-trigger.yml @@ -20,7 +20,6 @@ jobs: matrix: include: - label: integration tests - - label: vizro-ai ui tests steps: - name: Passed fork step run: echo "Success!" @@ -34,7 +33,6 @@ jobs: matrix: include: - label: integration tests - - label: vizro-ai ui tests steps: - uses: actions/checkout@v4 - name: Tests trigger @@ -44,8 +42,6 @@ jobs: if [ "${{ matrix.label }}" == "integration tests" ]; then export INPUT_WORKFLOW_FILE_NAME=${{ secrets.VIZRO_QA_INTEGRATION_TESTS_WORKFLOW }} - elif [ "${{ matrix.label }}" == "vizro-ai ui tests" ]; then - export INPUT_WORKFLOW_FILE_NAME=${{ secrets.VIZRO_QA_VIZRO_AI_UI_TESTS_WORKFLOW }} fi export INPUT_GITHUB_TOKEN=${{ secrets.VIZRO_SVC_PAT }} export INPUT_REF=main # because we should send existent branch to dispatch workflow diff --git a/vizro-ai/changelog.d/20241118_131003_alexey_snigir_vizro_ai_ui_tests.md b/vizro-ai/changelog.d/20241118_131003_alexey_snigir_vizro_ai_ui_tests.md new file mode 100644 index 000000000..f1f65e73c --- /dev/null +++ b/vizro-ai/changelog.d/20241118_131003_alexey_snigir_vizro_ai_ui_tests.md @@ -0,0 +1,48 @@ + + + + + + + + + diff --git a/vizro-ai/examples/dashboard_ui/app.py b/vizro-ai/examples/dashboard_ui/app.py index aa4f27a2b..025c18c5f 100644 --- a/vizro-ai/examples/dashboard_ui/app.py +++ b/vizro-ai/examples/dashboard_ui/app.py @@ -277,15 +277,15 @@ def update_model_dropdown(value): return available_models, default_model -app = Vizro().build(dashboard) -app.dash.layout.children.append( - dbc.NavLink( - ["Made with ", html.Img(src=get_asset_url("logo.svg"), id="banner", alt="Vizro logo"), "vizro"], - href="https://github.com/mckinsey/vizro", - target="_blank", - className="anchor-container", - ) -) -server = app.dash.server if __name__ == "__main__": + app = Vizro().build(dashboard) + app.dash.layout.children.append( + dbc.NavLink( + ["Made with ", html.Img(src=get_asset_url("logo.svg"), id="banner", alt="Vizro logo"), "vizro"], + href="https://github.com/mckinsey/vizro", + target="_blank", + className="anchor-container", + ) + ) + server = app.dash.server app.run() diff --git a/vizro-ai/hatch.toml b/vizro-ai/hatch.toml index 51465325c..08b4ff590 100644 --- a/vizro-ai/hatch.toml +++ b/vizro-ai/hatch.toml @@ -50,14 +50,15 @@ prep-release = [ ] pypath = "hatch run python -c 'import sys; print(sys.executable)'" test = "pytest tests {args}" -test-integration = "pytest -vs --reruns 1 tests/integration --headless {args}" -test-score = "pytest -vs --reruns 1 tests/score --headless {args}" +test-e2e-dashboard = "pytest -vs tests/e2e/test_dashboard.py --headless {args}" +test-e2e-plot = "pytest -vs --reruns 1 tests/e2e/test_plot.py --headless {args}" test-unit = "pytest tests/unit {args}" test-unit-coverage = [ "coverage run -m pytest tests/unit {args}", "- coverage combine", "coverage report" ] +test-vizro-ai-ui = "pytest -vs tests/vizro_ai_ui/test_vizro_ai_ui.py --headless" [envs.docs] dependencies = [ diff --git a/vizro-ai/pyproject.toml b/vizro-ai/pyproject.toml index 791d2ab72..d623ecccb 100644 --- a/vizro-ai/pyproject.toml +++ b/vizro-ai/pyproject.toml @@ -66,8 +66,13 @@ filterwarnings = [ # Ignore LLMchian deprecation warning: "ignore:.*The class `LLMChain` was deprecated in LangChain 0.1.17", # Ignore warning for Pydantic v1 API and Python 3.13: - "ignore:Failing to pass a value to the 'type_params' parameter of 'typing.ForwardRef._evaluate' is deprecated:DeprecationWarning" + "ignore:Failing to pass a value to the 'type_params' parameter of 'typing.ForwardRef._evaluate' is deprecated:DeprecationWarning", + # Ignore deprecation warning until this is solved: https://github.com/plotly/dash/issues/2590: + "ignore:HTTPResponse.getheader():DeprecationWarning", + # Happens during dash_duo teardown in vizro_ai_ui tests. Not affecting functionality: + "ignore:Exception in thread" ] +pythonpath = ["../tools/tests"] [tool.ruff] extend = "../pyproject.toml" diff --git a/vizro-ai/tests/integration/conftest.py b/vizro-ai/tests/e2e/conftest.py similarity index 100% rename from vizro-ai/tests/integration/conftest.py rename to vizro-ai/tests/e2e/conftest.py diff --git a/vizro-ai/tests/score/pytest.ini b/vizro-ai/tests/e2e/pytest.ini similarity index 100% rename from vizro-ai/tests/score/pytest.ini rename to vizro-ai/tests/e2e/pytest.ini diff --git a/vizro-ai/tests/score/test_dashboard.py b/vizro-ai/tests/e2e/test_dashboard.py similarity index 99% rename from vizro-ai/tests/score/test_dashboard.py rename to vizro-ai/tests/e2e/test_dashboard.py index 53d2e9033..53786f451 100644 --- a/vizro-ai/tests/score/test_dashboard.py +++ b/vizro-ai/tests/e2e/test_dashboard.py @@ -55,7 +55,7 @@ def logic( # noqa: PLR0912, PLR0915 config: json config of the expected dashboard """ - report_dir = "tests/score/reports" + report_dir = "tests/e2e/reports" os.makedirs(report_dir, exist_ok=True) app = Vizro().build(dashboard).dash diff --git a/vizro-ai/tests/integration/test_example.py b/vizro-ai/tests/e2e/test_plot.py similarity index 100% rename from vizro-ai/tests/integration/test_example.py rename to vizro-ai/tests/e2e/test_plot.py diff --git a/vizro-ai/tests/vizro_ai_ui/conftest.py b/vizro-ai/tests/vizro_ai_ui/conftest.py new file mode 100644 index 000000000..e623a6601 --- /dev/null +++ b/vizro-ai/tests/vizro_ai_ui/conftest.py @@ -0,0 +1,12 @@ +import pytest +from vizro import Vizro + + +@pytest.fixture(autouse=True) +def reset_managers(): + # this ensures that the managers are reset before and after each test + # the reset BEFORE all tests is important because at pytest test collection, fixtures are evaluated and hence + # the model_manager may be populated with models from other tests + Vizro._reset() + yield + Vizro._reset() diff --git a/vizro-ai/tests/vizro_ai_ui/fake_data_generator.py b/vizro-ai/tests/vizro_ai_ui/fake_data_generator.py new file mode 100644 index 000000000..cdf6adda1 --- /dev/null +++ b/vizro-ai/tests/vizro_ai_ui/fake_data_generator.py @@ -0,0 +1,46 @@ +import numpy as np +import pandas as pd + +# List of music genres +music_genres = [ + "Pop", + "Rock", + "Hip-Hop", + "Jazz", + "Classical", + "Electronic", + "Reggae", + "Country", + "Blues", + "R&B", + "Metal", + "Folk", +] + +# List of countries +countries = [ + "United States", + "Canada", + "United Kingdom", + "Germany", + "France", + "Japan", + "Australia", + "Brazil", + "India", + "South Korea", + "Italy", + "Spain", +] + + +# Function to generate fake music genre popularity dataset +def create_genre_popularity_by_country(first_year=1980, last_year=2023): + return pd.DataFrame( + { + "Year": (np.arange(first_year, last_year)), + "Countries": np.random.choice(countries, size=last_year - first_year), + "Genre": np.random.choice(music_genres, size=last_year - first_year), + "Popularity Score": np.random.choice(np.arange(0, 100), size=last_year - first_year), + } + ) diff --git a/vizro-ai/tests/vizro_ai_ui/test_vizro_ai_ui.py b/vizro-ai/tests/vizro_ai_ui/test_vizro_ai_ui.py new file mode 100644 index 000000000..8065842b5 --- /dev/null +++ b/vizro-ai/tests/vizro_ai_ui/test_vizro_ai_ui.py @@ -0,0 +1,76 @@ +import os +import runpy +from pathlib import Path + +import chromedriver_autoinstaller +import pytest +from fake_data_generator import create_genre_popularity_by_country +from vizro import Vizro + + +# Taken from https://github.com/pytest-dev/pytest/issues/363#issuecomment-1335631998. +@pytest.fixture(scope="module") +def monkeypatch_session(): + with pytest.MonkeyPatch.context() as mp: + yield mp + + +@pytest.fixture(scope="module", autouse=True) +def setup_integration_test_environment(monkeypatch_session): + # Dash debug mode seems to interfere with the tests, so we disable it here. Note "false" as a string is correct. + monkeypatch_session.setenv("DASH_DEBUG", "false") + # We only need to install chromedriver outside CI. + if not os.getenv("CI"): + chromedriver_autoinstaller.install() + + +@pytest.fixture +def dashboard(monkeypatch): + example_directory = Path(__file__).parents[2] / "examples/dashboard_ui" + monkeypatch.chdir(example_directory) + monkeypatch.syspath_prepend(example_directory) + return runpy.run_path("app.py")["dashboard"] + # Both run_path and run_module contaminate sys.modules, so we need to undo this in order to avoid interference + # between tests. However, if you do this then importlib.import_module seems to cause the problem due to mysterious + # reasons. The current system should work well so long as there's no sub-packages with clashing names in the + # examples. + + +@pytest.fixture +def fake_data(): + popularity_dataset = create_genre_popularity_by_country() + popularity_dataset.to_csv("genre_popularity_by_country.csv", index=False) + yield os.path.abspath("genre_popularity_by_country.csv") + + +def test_chart_ui(dash_duo, dashboard, fake_data): + app = Vizro(assets_folder=Path(__file__).parents[2] / "examples/dashboard_ui/assets").build(dashboard).dash + dash_duo.start_server(app) + + # fill in values + api_key = dash_duo.wait_for_element("#settings-api-key") + api_base = dash_duo.wait_for_element("#settings-api-base") + api_key.send_keys(os.environ["OPENAI_API_KEY"]) + api_base.send_keys(os.environ["OPENAI_API_BASE"]) + + # close panel + dash_duo.multiple_click(".btn-close", 1) + + # upload file + file_input = dash_duo.wait_for_element('input[type="file"]') + file_input.send_keys(fake_data) + dash_duo.multiple_click("#data-upload", 1) + + # enter prompt + prompt = dash_duo.wait_for_element("#text-area") + prompt.send_keys("Create bar graph by genre") + + # click run VizroAI + dash_duo.multiple_click("#trigger-button", 1) + + # check result + dash_duo.wait_for_element(".language-python") + dash_duo.wait_for_contains_text("span[class='hljs-keyword']", "import") + + # check console logs for errors + assert dash_duo.get_logs() == []