diff --git a/.github/actions/install_kedro_and_python_dependencies/action.yml b/.github/actions/install_kedro_and_python_dependencies/action.yml new file mode 100644 index 0000000000..3874ccdc20 --- /dev/null +++ b/.github/actions/install_kedro_and_python_dependencies/action.yml @@ -0,0 +1,15 @@ +name: Install Kedro and other Python Dependencies +description: Installs Kedro from the main branch and other Python dependencies, then prints the Python version and installed packages. +runs: + using: composite + steps: + - name: Install Python dependencies + run: |- + pip install git+https://github.com/kedro-org/kedro@main + pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U + shell: bash + - name: Echo package versions + run: |- + python -V + pip freeze + shell: bash \ No newline at end of file diff --git a/.github/actions/install_node_dependencies/action.yml b/.github/actions/install_node_dependencies/action.yml new file mode 100644 index 0000000000..b36a41e83a --- /dev/null +++ b/.github/actions/install_node_dependencies/action.yml @@ -0,0 +1,37 @@ +name: Setup Node.js and Install Dependencies +description: Sets up a specific Node.js version, caches Node modules, and installs Node dependencies. + +inputs: + node-version: + description: 'Node.js version' + required: false + default: '18.20.0' + + package-path: + description: 'Path to package.json file' + required: false + default: '.' + +runs: + using: composite + steps: + - name: Setup Node.js + uses: actions/setup-node@v4.0.0 + with: + node-version: ${{ inputs.node-version }} + + - name: Get NPM Cache Directory + id: npm-cache-dir + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Node.js packages + uses: actions/cache@v2 + with: + path: "${{ steps.npm-cache-dir.outputs.dir }}" + key: "${{ runner.os }}-node-${{ hashFiles(format('{0}/package-lock.json', inputs.package-path)) }}" + restore-keys: "${{ runner.os }}-node-" + + - name: Install Node Dependencies + run: npm install + shell: bash diff --git a/.github/actions/setup_tests/action.yml b/.github/actions/setup_tests/action.yml new file mode 100644 index 0000000000..4de29e785a --- /dev/null +++ b/.github/actions/setup_tests/action.yml @@ -0,0 +1,46 @@ +name: Setup Tests +description: Sets up the testing environment by setting up Python and Node.js, caching Python packages, installing Kedro and other Python dependencies, and building the React application. + +inputs: + os: + description: 'Operating system' + required: false + default: 'ubuntu-latest' + python-version: + description: 'Python version' + required: false + default: '3.9' + +runs: + using: "composite" + steps: + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + shell: bash \ No newline at end of file diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..4c981d6c8b --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,44 @@ +name: Run all checks on Kedro-Viz +# Runs end-to-end tests, unit tests, linting and JavaScript +# linting & tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + workflow_dispatch: + schedule: + # Run every day at 1:00 AM(UTC time) + - cron: 0 1 * * * +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + javascript_lint_and_tests: + uses: ./.github/workflows/javascript-lint-and-tests.yml diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..70a8691854 --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +name: Build backend +# Runs end-to-end tests, unit tests, and linting on the backend code +# for different operating systems and Python versions. + +on: + push: + paths: + - 'package/**' + - '.github/**' + pull_request: + paths: + - 'package/**' + - '.github/**' + workflow_dispatch: +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..5eee3d98cd --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,39 @@ +name: Run e2e tests on Kedro-Viz +# Runs end-to-end tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + e2e_tests: + runs-on: ${{ inputs.os }} + + # below condition checks if the operating system is Ubuntu, or + # if the operating system is Windows and the branch is main/demo + if: > + inputs.os == 'ubuntu-latest' || + ( + ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/demo' + ) && + inputs.os == 'windows-latest' + ) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} + + - name: Run all end to end tests + run: make e2e-tests \ No newline at end of file diff --git a/.github/workflows/javascript-lint-and-tests.yml b/.github/workflows/javascript-lint-and-tests.yml new file mode 100644 index 0000000000..0cff820c9b --- /dev/null +++ b/.github/workflows/javascript-lint-and-tests.yml @@ -0,0 +1,61 @@ +name: Run javascript linters and tests on Kedro-Viz +# Runs JavaScript linting, unit tests, and end-to-end tests on +# Kedro-Viz for ubuntu-latest operating systems and Python 3.9. + +on: + push: + paths-ignore: + - 'package/**' + pull_request: + paths-ignore: + - 'package/**' + workflow_dispatch: + workflow_call: + +jobs: + javascript_lint_and_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python "3.9" + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Cache python packages for Linux + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ubuntu-latest-python-3.9 + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Setup Cypress requirements + run: |- + sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list + sudo apt-get update + sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + + - name: Test lib transpilation + run: npm run lib + + - name: Test JS library imports + run: |- + npm run lib-test:setup + cd tools/test-lib/react-app + npm run test:ci + + - name: Run Eslint + run: npm run lint + + - name: Run JavaScript tests + run: npm run test:ci + + - name: Run Javascript end to end tests + run: npm run cy:ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..02489d36dd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Run linters on Kedro-Viz +# Runs secret scan, security scan, GraphQL schema check, +# and Python formatters and linters on Kedro-Viz for +# different operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_kedro_and_python_dependencies" + + - name: Run secret scan + run: make secret-scan + + - name: Run security scan + run: make security-scan + + - name: Verify GraphQL schema is up to date + run: make schema-check + + - name: Run Python formatters and linters + run: make format-check lint-check diff --git a/.github/workflows/merge-gatekeeper.yml b/.github/workflows/merge-gatekeeper.yml new file mode 100644 index 0000000000..abc5c013fe --- /dev/null +++ b/.github/workflows/merge-gatekeeper.yml @@ -0,0 +1,26 @@ +name: Merge Gatekeeper + +on: + pull_request: + branches: + - main + +jobs: + merge-gatekeeper: + runs-on: ubuntu-latest + # Restrict permissions of the GITHUB_TOKEN. + # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs + permissions: + checks: read + statuses: read + steps: + - name: Run Merge Gatekeeper + # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs: + # https://github.com/upsidr/merge-gatekeeper/tags + # https://github.com/upsidr/merge-gatekeeper/branches + uses: upsidr/merge-gatekeeper@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + timeout: 3600 + interval: 30 + ignored: 'ci/circleci: win_unit_tests-3.9,ci/circleci: win_unit_tests-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: unit_tests-3.9,ci/circleci: win_e2e_tests-3.10,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_e2e_tests-3.8,ci/circleci: e2e_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: lint-3.9,ci/circleci: lint-3.10,ci/circleci: javascript_lint_and_tests,ci/circleci: check-updated-files,ci/circleci: all_circleci_checks_succeeded' \ No newline at end of file diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..fe970683c3 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,39 @@ +name: Run unit-tests on Kedro-Viz +# Runs unit tests on Kedro-Viz across different +# OS and Python versions after environment setup. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit_tests: + runs-on: ${{ inputs.os }} + + # below condition checks if the operating system is Ubuntu, or + # if the operating system is Windows and the branch is main/demo + if: > + inputs.os == 'ubuntu-latest' || + ( + ( + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/demo' + ) && + inputs.os == 'windows-latest' + ) + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Tests + uses: "./.github/actions/setup_tests" + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python-version }} + + - name: Run Python tests + run: make pytest \ No newline at end of file diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt index da31b5fafa..470cd17fce 100644 --- a/trufflehog-ignore.txt +++ b/trufflehog-ignore.txt @@ -7,4 +7,5 @@ README.md LAYOUT_ENGINE.md src/utils/random-utils.js src/components/update-reminder/update-reminder-content.js -src/components/feature-hints/feature-hints-content.js \ No newline at end of file +src/components/feature-hints/feature-hints-content.js +cypress/fixtures/graphql/ \ No newline at end of file