diff --git a/.github/actions/install_node_dependencies/action.yml b/.github/actions/install_node_dependencies/action.yml new file mode 100644 index 0000000000..2d37f4302d --- /dev/null +++ b/.github/actions/install_node_dependencies/action.yml @@ -0,0 +1,22 @@ +name: Setup Node.js and Install Dependencies +description: Sets up a specific Node.js version, caches Node modules, and installs Node dependencies. + +runs: + using: composite + steps: + - uses: actions/setup-node@v4.0.0 + with: + node-version: 16.13.2 + + - id: npm-cache-dir + run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT + shell: bash + + - uses: actions/cache@v4 + with: + path: "${{ steps.npm-cache-dir.outputs.dir }}" + key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}" + restore-keys: "${{ runner.os }}-node-" + + - run: npm ci + shell: bash diff --git a/.github/actions/install_python_dependencies/action.yml b/.github/actions/install_python_dependencies/action.yml new file mode 100644 index 0000000000..3874ccdc20 --- /dev/null +++ b/.github/actions/install_python_dependencies/action.yml @@ -0,0 +1,15 @@ +name: Install Kedro and other Python Dependencies +description: Installs Kedro from the main branch and other Python dependencies, then prints the Python version and installed packages. +runs: + using: composite + steps: + - name: Install Python dependencies + run: |- + pip install git+https://github.com/kedro-org/kedro@main + pip install -r package/test_requirements.txt -r demo-project/src/docker_requirements.txt -U + shell: bash + - name: Echo package versions + run: |- + python -V + pip freeze + shell: bash \ No newline at end of file diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml new file mode 100644 index 0000000000..b47ed8f3a2 --- /dev/null +++ b/.github/workflows/all-checks.yml @@ -0,0 +1,51 @@ +name: Run all checks on Kedro-Viz +# Runs end-to-end tests, unit tests, linting and JavaScript +# linting & tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + workflow_dispatch: + schedule: + # Run every day at 1:00 AM(UTC time) + - cron: 0 1 * * * +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + javascript_lint_and_tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9" ] + uses: ./.github/workflows/javascript-lint-and-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml new file mode 100644 index 0000000000..70a8691854 --- /dev/null +++ b/.github/workflows/build-backend.yml @@ -0,0 +1,44 @@ +name: Build backend +# Runs end-to-end tests, unit tests, and linting on the backend code +# for different operating systems and Python versions. + +on: + push: + paths: + - 'package/**' + - '.github/**' + pull_request: + paths: + - 'package/**' + - '.github/**' + workflow_dispatch: +jobs: + e2e_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/e2e-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit_tests: + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/lint.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml new file mode 100644 index 0000000000..172e3c91b8 --- /dev/null +++ b/.github/workflows/build-frontend.yml @@ -0,0 +1,21 @@ +name: Build frontend +# Runs JavaScript linting & tests on the frontend code for Ubuntu OS and Python 3.9. + +on: + push: + paths-ignore: + - 'package/**' + pull_request: + paths-ignore: + - 'package/**' + workflow_dispatch: +jobs: + javascript_lint_and_tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.9" ] + uses: ./.github/workflows/javascript-lint-and-tests.yml + with: + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} \ No newline at end of file diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000000..0cb344f68c --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,52 @@ +name: Run e2e tests on Kedro-Viz +# Runs end-to-end tests on Kedro-Viz for different +# operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + e2e_tests: + runs-on: ${{ inputs.os }} + # Only run on main and demo branches for Windows + if: inputs.os == 'ubuntu-latest' || ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/demo') && inputs.os == 'windows-latest') + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + + - name: Run Python tests + run: make e2e-tests \ No newline at end of file diff --git a/.github/workflows/javascript-lint-and-tests.yml b/.github/workflows/javascript-lint-and-tests.yml new file mode 100644 index 0000000000..70245689e5 --- /dev/null +++ b/.github/workflows/javascript-lint-and-tests.yml @@ -0,0 +1,59 @@ +name: Run javascript linters and tests on Kedro-Viz +# Runs JavaScript linting, unit tests, and end-to-end tests on +# Kedro-Viz for different operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + javascript_lint_and_tests: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Setup Cypress requirements + run: |- + sudo sed -i 's/archive.ubuntu.com/us-east-1.ec2.archive.ubuntu.com/g' /etc/apt/sources.list + sudo apt-get update + sudo apt-get install libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + + - name: Test lib transpilation + run: npm run lib + + - name: Test JS library imports + run: |- + npm run lib-test:setup + cd tools/test-lib/react-app + npm run test:ci + + - name: Run Eslint + run: npm run lint + + - name: Run JavaScript tests + run: npm run test:ci + + - name: Run Javascript end to end tests + run: npm run cy:ci diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..3bfe309faa --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,45 @@ +name: Run linters on Kedro-Viz +# Runs secret scan, security scan, GraphQL schema check, +# and Python formatters and linters on Kedro-Viz for +# different operating systems and Python versions. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + lint: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_python_dependencies" + + - name: Run secret scan + run: make secret-scan + + - name: Run security scan + run: make security-scan + + - name: Verify GraphQL schema is up to date + run: make schema-check + + - name: Run Python formatters and linters + run: make format-check lint-check diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..96a7aadc1a --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,52 @@ +name: Run unit-tests on Kedro-Viz +# Runs unit tests on Kedro-Viz across different +# OS and Python versions after environment setup. + +on: + workflow_call: + inputs: + os: + type: string + python-version: + type: string +jobs: + unit_tests: + runs-on: ${{ inputs.os }} + # Only run on main and demo branches for Windows + if: inputs.os == 'ubuntu-latest' || ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/demo') && inputs.os == 'windows-latest') + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v5 + with: + python-version: ${{inputs.python-version}} + + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v4 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.os}}-python-${{inputs.python-version}} + + - name: Install Kedro and other Python Dependencies + uses: "./.github/actions/install_python_dependencies" + + - name: Setup Node.js and Install Dependencies + uses: "./.github/actions/install_node_dependencies" + + - name: Build React application + run: |- + node -v + make build + + - name: Run Python tests + run: make pytest \ No newline at end of file