From 5f052796f0b3371c3954436fb1cd3569ae0fb037 Mon Sep 17 00:00:00 2001 From: avifenesh Date: Sun, 21 Jul 2024 16:01:05 +0000 Subject: [PATCH 1/2] limit pr run to ubuntu latest X first and last version supported, added schedule and label runs Signed-off-by: avifenesh --- .github/json_matrices/engine-matrix.json | 18 ++- .github/workflows/java.yml | 98 +++++++++---- .github/workflows/node.yml | 160 +++++++++++++++------ .github/workflows/python.yml | 169 +++++++++++++++++++---- 4 files changed, 342 insertions(+), 103 deletions(-) diff --git a/.github/json_matrices/engine-matrix.json b/.github/json_matrices/engine-matrix.json index f20f0c955e..4621fe597b 100644 --- a/.github/json_matrices/engine-matrix.json +++ b/.github/json_matrices/engine-matrix.json @@ -1,6 +1,16 @@ [ - { - "type": "valkey", - "version": "7.2.5" - } + { + "type": "valkey", + "version": "7.2.5", + "run": "always" + }, + { + "type": "valkey", + "version": "redis-7.0.15" + }, + { + "type": "valkey", + "version": "redis-6.2.14", + "run": "always" + } ] diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 5106aec4ce..c2a181fd7d 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -26,46 +26,90 @@ on: - .github/json_matrices/build-matrix.json workflow_dispatch: + pull_request_target: + types: [labeled] + paths: + - glide-core/src/** + - submodules/** + - java/** + - .github/workflows/java.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/test-benchmark/action.yml + - .github/workflows/lint-rust/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json + schedule: + - cron: "0 0 * * *" + concurrency: group: java-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: - load-engine-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 + check-label: + runs-on: ubuntu-latest + # If the trigger is a label we want to check if it is a CoreChanges label, + # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved + environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + steps: + - name: Check for CoreChanges label + if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + run: | + echo "This workflow only runs for the CoreChanges label on pull requests" + exit 1 - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + load-engine-matrix: + runs-on: ubuntu-latest + # We want to run this job only if the previous job outputs should_run=true, + # Which means that if the trigger was a label we run this job just if the label was CoreChanges + needs: check-label + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + + create-host-matrix: + runs-on: ubuntu-latest + needs: check-label + outputs: + matrix: ${{ steps.create-host-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create host matrix + id: create-host-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then + echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"}]}' >> $GITHUB_OUTPUT + else + echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"},{"OS":"macos","RUNNER":"macos-latest","TARGET":"aarch64-apple-darwin"}]}' >> $GITHUB_OUTPUT + fi build-and-test-java-client: - needs: load-engine-matrix + needs: [load-engine-matrix, create-host-matrix] timeout-minutes: 35 strategy: # Run all jobs fail-fast: false matrix: java: - # - 11 + - 11 - 17 engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } + host: ${{ fromJson(needs.create-host-matrix.outputs.matrix).include }} runs-on: ${{ matrix.host.RUNNER }} @@ -122,15 +166,16 @@ jobs: java/client/build/reports/spotbugs/** build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' + if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event.label.name == 'CoreChanges') strategy: # Run all jobs fail-fast: false matrix: java: - # - 11 + - 11 - 17 runs-on: ubuntu-latest + needs: check-label container: amazonlinux:latest timeout-minutes: 35 steps: @@ -186,6 +231,7 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest + needs: check-label steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 32db45e5c5..cf8693878b 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -30,6 +30,23 @@ on: - .github/json_matrices/build-matrix.json workflow_dispatch: + pull_request_target: + types: [labeled] + paths: + - glide-core/src/** + - submodules/** + - node/** + - utils/cluster_manager.py + - .github/workflows/node.yml + - .github/workflows/build-node-wrapper/action.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/test-benchmark/action.yml + - .github/workflows/lint-rust/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json + schedule: + - cron: "0 1 * * *" + concurrency: group: node-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -38,37 +55,81 @@ env: CARGO_TERM_COLOR: always jobs: - load-engine-matrix: + check-label: runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + # If the trigger is a label we want to check if it is a CoreChanges label, + # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved + environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Check for CoreChanges label + if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + run: | + echo "This workflow only runs for the CoreChanges label on pull requests" + exit 1 + + load-engine-matrix: + runs-on: ubuntu-latest + # We want to run this job only if the previous job outputs should_run=true, + # Which means that if the trigger was a label we run this job just if the label was CoreChanges + needs: check-label + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + create-version-matrix: + runs-on: ubuntu-latest + needs: check-label + outputs: + matrix: ${{ steps.create-version-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create version matrix + id: create-version-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo 'matrix={"node":["16.x","20.x"]}' >> $GITHUB_OUTPUT + else + echo 'matrix={"node":["16.x","17.x","18.x","19.x","20.x"]}' >> $GITHUB_OUTPUT + fi - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT test-ubuntu-latest: runs-on: ubuntu-latest - needs: load-engine-matrix - timeout-minutes: 25 + needs: [load-engine-matrix, create-version-matrix] + timeout-minutes: 15 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - + node: ${{ fromJson(needs.create-version-matrix.outputs.matrix).node }} + steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Use Node.js 16.x + - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: ${{ matrix.node }} - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper @@ -118,6 +179,7 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest + needs: check-label steps: - uses: actions/checkout@v4 with: @@ -128,39 +190,41 @@ jobs: cargo-toml-folder: ./node/rust-client name: lint node rust - # build-macos-latest: - # runs-on: macos-latest - # timeout-minutes: 25 - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: recursive - # - name: Set up Homebrew - # uses: Homebrew/actions/setup-homebrew@master - - # - name: Install NodeJS - # run: | - # brew update - # brew upgrade || true - # brew install node - - # - name: Downgrade npm major version to 8 - # run: | - # npm i -g npm@8 - - # - name: Build Node wrapper - # uses: ./.github/workflows/build-node-wrapper - # with: - # os: "macos" - # named_os: "darwin" - # arch: "arm64" - # target: "aarch64-apple-darwin" - # github-token: ${{ secrets.GITHUB_TOKEN }} - # engine-version: "7.2.5" + build-macos-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + runs-on: macos-latest + needs: check-label + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Install NodeJS + run: | + brew update + brew upgrade || true + brew install node + + - name: Downgrade npm major version to 8 + run: | + npm i -g npm@8 - # - name: Test compatibility - # run: npm test -- -t "set and get flow works" - # working-directory: ./node + - name: Build Node wrapper + uses: ./.github/workflows/build-node-wrapper + with: + os: "macos" + named_os: "darwin" + arch: "arm64" + target: "aarch64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: "7.2.5" + + - name: Test compatibility + run: npm test -- -t "set and get flow works" + working-directory: ./node # - name: Upload test reports # if: always() @@ -174,8 +238,10 @@ jobs: # benchmarks/results/** build-amazonlinux-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' runs-on: ubuntu-latest container: amazonlinux:latest + needs: check-label timeout-minutes: 15 steps: - name: Install git @@ -221,7 +287,9 @@ jobs: benchmarks/results/** build-and-test-linux-musl-on-x86: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' name: Build and test Node wrapper on Linux musl + needs: check-label runs-on: ubuntu-latest container: image: node:alpine diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 7e453178a6..37e05bfd7e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -31,6 +31,23 @@ on: - .github/json_matrices/build-matrix.json workflow_dispatch: + pull_request_target: + types: [labeled] + paths: + - python/** + - glide-core/src/** + - submodules/** + - utils/cluster_manager.py + - .github/workflows/python.yml + - .github/workflows/build-python-wrapper/action.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/test-benchmark/action.yml + - .github/workflows/lint-rust/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json + schedule: + - cron: "0 2 * * *" + concurrency: group: python-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -39,45 +56,71 @@ permissions: contents: read jobs: + check-label: + runs-on: ubuntu-latest + # If the trigger is a label we want to check if it is a CoreChanges label, + # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved + environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + steps: + - name: Check for CoreChanges label + if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + run: | + echo "This workflow only runs for the CoreChanges label on pull requests" + exit 1 + load-engine-matrix: + runs-on: ubuntu-latest + # We want to run this job only if the previous job outputs should_run=true, + # Which means that if the trigger was a label we run this job just if the label was CoreChanges + needs: check-label + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then + echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + create-version-matrix: runs-on: ubuntu-latest + needs: check-label outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + matrix: ${{ steps.create-version-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix + + - name: Create version matrix + id: create-version-matrix + shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - - test: - runs-on: ${{ matrix.host.RUNNER }} - needs: load-engine-matrix + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then + echo 'matrix={"python":["3.8","3.12"]}' >> $GITHUB_OUTPUT + else + echo 'matrix={"python":["3.8","3.9","3.10","3.11","3.12"]}' >> $GITHUB_OUTPUT + fi + + test-ubuntu-latest: + runs-on: ubuntu-latest + needs: [load-engine-matrix, create-version-matrix] timeout-minutes: 35 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" - - "3.12" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} steps: - uses: actions/checkout@v4 with: @@ -134,6 +177,15 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** + test-pubsub-ubuntu-latest: + runs-on: ubuntu-latest + needs: [load-engine-matrix, create-version-matrix] + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} test-pubsub: runs-on: ${{ matrix.host.RUNNER }} @@ -197,6 +249,7 @@ jobs: lint: runs-on: ubuntu-latest + needs: check-label timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -235,10 +288,72 @@ jobs: working-directory: ./python run: | black --check --diff . + test-macos-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + runs-on: macos-latest + needs: load-engine-matrix + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: "macos" + target: "aarch64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test with pytest + working-directory: ./python + run: | + source .env/bin/activate + pytest --asyncio-mode=auto + + test-pubsub-macos-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + runs-on: macos-latest + needs: load-engine-matrix + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: "macos" + target: "aarch64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test pubsub with pytest + working-directory: ./python + run: | + source .env/bin/activate + cd python/tests/ + pytest --asyncio-mode=auto -k test_pubsub build-amazonlinux-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' runs-on: ubuntu-latest container: amazonlinux:latest + needs: check-label timeout-minutes: 15 steps: - name: Install git From fc4cb44ec9526fbad0efe41aaaac797043d04b3d Mon Sep 17 00:00:00 2001 From: avifenesh Date: Sun, 28 Jul 2024 12:38:57 +0000 Subject: [PATCH 2/2] limit CI Signed-off-by: avifenesh --- .github/DEVELOPER.md | 110 ++++++++++++ .github/json_matrices/build-matrix.json | 18 +- .github/workflows/java.yml | 84 +++++---- .github/workflows/lint-rust/action.yml | 2 - .github/workflows/node.yml | 63 +++---- .github/workflows/python.yml | 225 +++++++++--------------- 6 files changed, 289 insertions(+), 213 deletions(-) create mode 100644 .github/DEVELOPER.md diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md new file mode 100644 index 0000000000..14e9602434 --- /dev/null +++ b/.github/DEVELOPER.md @@ -0,0 +1,110 @@ +# CI/CD Workflow Guide + + +TODO: Add a description of the CI/CD workflow and its components. + + +### Overview + +Our CI/CD pipeline tests and builds our project across multiple languages, versions, and environments. This guide outlines the key components and processes of our workflow. + +### Workflow Triggers + +- Push to `main` branch +- Pull requests +- Scheduled runs (daily) +- Manual trigger (workflow_dispatch) +- Pull request with `Core changes` label + +### Language-Specific Workflows + +#### Python (python.yml) + +#### Node.js (node.yml) + +#### Java (java.yml) + +Each language has its own workflow file with similar structure but language-specific steps. + +### Shared Components + +#### Run Field + +The `run` field in both engine-matrix.json and build-matrix.json is used to specify when a particular configuration should be executed: + +- In engine-matrix.json: + + - `"run": "always"`: The engine version will be tested in every workflow run. + - If `run` is omitted: The engine version will only be tested in full matrix runs (e.g., when the `Core changes` label is applied). + +- In build-matrix.json: + - The `run` array specifies which language workflows should use this host configuration. + - `"always"` in the array means the host will be used in every workflow run for the specified languages. + +This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage. + +#### Engine Matrix (engine-matrix.json) + +Defines the versions of Valkey engine to test against: + +```json +[ + { "type": "valkey", "version": "7.2.5", "run": "always" }, + { "type": "valkey", "version": "redis-7.0.15" }, + { "type": "valkey", "version": "redis-6.2.14", "run": "always" } +] +``` + +#### Build Matrix (build-matrix.json) + +Defines the host environments for testing: + +```json +[ + { + "OS": "ubuntu", + "RUNNER": "ubuntu-latest", + "TARGET": "x86_64-unknown-linux-gnu", + "run": ["always", "python", "node", "java"] + } + // ... other configurations +] +``` + +#### Adding New Versions or Hosts + +To add a new engine version, update engine-matrix.json. +To add a new host environment, update build-matrix.json. + +Ensure new entries have all required fields: + +For engine versions: `type`, `version`, `run` (optional) +For hosts: `OS`, `RUNNER`, `TARGET`, `run` + +#### Triggering Workflows + +Push to main or create a pull request to run workflows automatically. +Add `Core changes` label to a pull request to trigger full matrix tests. +Use workflow_dispatch for manual triggers. + +### Mutual vs. Language-Specific Components + +#### Mutual + +`Engine matrix` +`Build matrix` +`Shared dependencies installation` +`Linting (Rust)` + +#### Language-Specific + +`Package manager commands` +`Testing frameworks` +`Build processes` +`Version matrices` + +### Customizing Workflows + +Modify `[language].yml` files to adjust language-specific steps. +Update matrix files to change tested versions or environments. +Adjust cron schedules in workflow files for different timing of scheduled runs. diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index e8589b84f8..98219d7848 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -5,7 +5,8 @@ "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", - "PACKAGE_MANAGERS": ["pypi", "npm"] + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["always", "python", "node", "java"] }, { "OS": "ubuntu", @@ -14,7 +15,8 @@ "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm"], - "CONTAINER": "2_28" + "CONTAINER": "2_28", + "run": ["python", "node"] }, { "OS": "macos", @@ -22,7 +24,8 @@ "RUNNER": "macos-12", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm"] + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["python", "node"] }, { "OS": "macos", @@ -30,7 +33,8 @@ "RUNNER": "macos-latest", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm"] + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["python", "node", "java"] }, { "OS": "ubuntu", @@ -40,7 +44,8 @@ "RUNNER": ["self-hosted", "Linux", "ARM64"], "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged --rm", - "PACKAGE_MANAGERS": ["npm"] + "PACKAGE_MANAGERS": ["npm"], + "run": "node" }, { "OS": "ubuntu", @@ -50,6 +55,7 @@ "RUNNER": "ubuntu-latest", "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged", - "PACKAGE_MANAGERS": ["npm"] + "PACKAGE_MANAGERS": ["npm"], + "run": "node" } ] diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index c2a181fd7d..1f95cc29be 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -46,23 +46,26 @@ concurrency: cancel-in-progress: true jobs: - check-label: + check_should_run: runs-on: ubuntu-latest - # If the trigger is a label we want to check if it is a CoreChanges label, - # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved - environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + outputs: + should_run: ${{ steps.check_should_run.outputs.should_run }} steps: - - name: Check for CoreChanges label - if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + - name: Check for Core changes label + id: check_should_run run: | - echo "This workflow only runs for the CoreChanges label on pull requests" - exit 1 + if [[ ("${{ github.event.label.name }}" == "Core changes" && "${{github.event_name}}" == 'pull_request_target') || "${{github.event_name}}" != 'pull_request_target' ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + fi load-engine-matrix: runs-on: ubuntu-latest # We want to run this job only if the previous job outputs should_run=true, - # Which means that if the trigger was a label we run this job just if the label was CoreChanges - needs: check-label + # Which means that if the trigger was a label we run this job just if the label was Core changes + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} outputs: matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: @@ -79,38 +82,59 @@ jobs: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT fi - create-host-matrix: + load-host-matrix: + runs-on: ubuntu-latest + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} + outputs: + matrix: ${{ steps.load-host-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: load host matrix + id: load-host-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo 'matrix={"include":' $(jq '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json) '}' >> $GITHUB_OUTPUT + else + echo 'matrix={"include":' $(jq '[.[] | select(.run | type == "array" and contains(["java"]))]' .github/json_matrices/build-matrix.json) '}' >> $GITHUB_OUTPUT + fi + + create-version-matrix: runs-on: ubuntu-latest - needs: check-label + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} outputs: - matrix: ${{ steps.create-host-matrix.outputs.matrix }} + matrix: ${{ steps.create-version-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Create host matrix - id: create-host-matrix + - name: Create version matrix + id: create-version-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and Core changes + # For other events we want to run only the always matrix which is first and last version supported run: | if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then - echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"}]}' >> $GITHUB_OUTPUT + echo 'matrix={"java":["11","17"]}' >> $GITHUB_OUTPUT else - echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"},{"OS":"macos","RUNNER":"macos-latest","TARGET":"aarch64-apple-darwin"}]}' >> $GITHUB_OUTPUT + echo 'matrix={"java":["11","17"]}' >> $GITHUB_OUTPUT fi build-and-test-java-client: - needs: [load-engine-matrix, create-host-matrix] + needs: [load-engine-matrix, load-host-matrix, create-version-matrix] timeout-minutes: 35 strategy: # Run all jobs fail-fast: false matrix: - java: - - 11 - - 17 + java: ${{ fromJson(needs.create-version-matrix.outputs.matrix).java }} engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: ${{ fromJson(needs.create-host-matrix.outputs.matrix).include }} - + host: ${{ fromJson(needs.load-host-matrix.outputs.matrix).include }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -166,16 +190,15 @@ jobs: java/client/build/reports/spotbugs/** build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event.label.name == 'CoreChanges') + if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event.label.name == 'Core changes') + needs: [load-engine-matrix, create-version-matrix] strategy: # Run all jobs fail-fast: false matrix: - java: - - 11 - - 17 + java: ${{ fromJson(needs.create-version-matrix.outputs.matrix).java }} + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} runs-on: ubuntu-latest - needs: check-label container: amazonlinux:latest timeout-minutes: 35 steps: @@ -201,7 +224,7 @@ jobs: os: "amazon-linux" target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: ${{ matrix.engine.version }} - name: Install protoc (protobuf) uses: arduino/setup-protoc@v3 @@ -231,7 +254,8 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest - needs: check-label + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/lint-rust/action.yml b/.github/workflows/lint-rust/action.yml index 8a7cdf185f..531acae1be 100644 --- a/.github/workflows/lint-rust/action.yml +++ b/.github/workflows/lint-rust/action.yml @@ -24,8 +24,6 @@ runs: github-token: ${{ inputs.github-token }} - uses: Swatinem/rust-cache@v2 - with: - github-token: ${{ inputs.github-token }} - run: cargo fmt --all -- --check working-directory: ${{ inputs.cargo-toml-folder }} diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index cf8693878b..204c023c78 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -55,23 +55,25 @@ env: CARGO_TERM_COLOR: always jobs: - check-label: + check_should_run: runs-on: ubuntu-latest - # If the trigger is a label we want to check if it is a CoreChanges label, - # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved - environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + outputs: + should_run: ${{ steps.check_should_run.outputs.should_run }} steps: - - name: Check for CoreChanges label - if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + - name: Check for Core changes label + id: check_should_run run: | - echo "This workflow only runs for the CoreChanges label on pull requests" - exit 1 - + if [[ ("${{ github.event.label.name }}" == "Core changes" && "${{github.event_name}}" == 'pull_request_target') || "${{github.event_name}}" != 'pull_request_target' ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + fi load-engine-matrix: runs-on: ubuntu-latest # We want to run this job only if the previous job outputs should_run=true, - # Which means that if the trigger was a label we run this job just if the label was CoreChanges - needs: check-label + # Which means that if the trigger was a label we run this job just if the label was Core changes + needs: check_should_run + if: needs.check_should_run.outputs.should_run == 'true' outputs: matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: @@ -81,7 +83,7 @@ jobs: - name: Load the engine matrix id: load-engine-matrix shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges + # We want to run the full matrix only for scheduled runs and Core changes # For other events we want to run only the always matrix which is first and last version supported run: | if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then @@ -91,7 +93,8 @@ jobs: fi create-version-matrix: runs-on: ubuntu-latest - needs: check-label + needs: check_should_run + if: needs.check_should_run.outputs.should_run == 'true' outputs: matrix: ${{ steps.create-version-matrix.outputs.matrix }} steps: @@ -101,7 +104,7 @@ jobs: - name: Create version matrix id: create-version-matrix shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges + # We want to run the full matrix only for scheduled runs and Core changes # For other events we want to run only the always matrix which is first and last version supported run: | if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then @@ -114,13 +117,12 @@ jobs: test-ubuntu-latest: runs-on: ubuntu-latest needs: [load-engine-matrix, create-version-matrix] - timeout-minutes: 15 + timeout-minutes: 25 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - node: ${{ fromJson(needs.create-version-matrix.outputs.matrix).node }} - + node: ${{ fromJson(needs.create-version-matrix.outputs.matrix).node }} steps: - uses: actions/checkout@v4 with: @@ -128,6 +130,8 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true with: node-version: ${{ matrix.node }} @@ -179,7 +183,8 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest - needs: check-label + needs: check_should_run + if: needs.check_should_run.outputs.should_run == 'true' steps: - uses: actions/checkout@v4 with: @@ -191,9 +196,8 @@ jobs: name: lint node rust build-macos-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: github.event_name == 'schedule' || github.event.label.name == 'Core changes' runs-on: macos-latest - needs: check-label timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -222,8 +226,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node # - name: Upload test reports @@ -238,10 +242,9 @@ jobs: # benchmarks/results/** build-amazonlinux-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: github.event_name == 'schedule' || github.event.label.name == 'Core changes' runs-on: ubuntu-latest container: amazonlinux:latest - needs: check-label timeout-minutes: 15 steps: - name: Install git @@ -271,8 +274,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node - name: Upload test reports @@ -287,9 +290,8 @@ jobs: benchmarks/results/** build-and-test-linux-musl-on-x86: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: github.event_name == 'schedule' || github.event.label.name == 'Core changes' name: Build and test Node wrapper on Linux musl - needs: check-label runs-on: ubuntu-latest container: image: node:alpine @@ -322,9 +324,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - shell: bash - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node - name: Upload test reports diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 37e05bfd7e..274cefee68 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -56,33 +56,35 @@ permissions: contents: read jobs: - check-label: + check_should_run: runs-on: ubuntu-latest - # If the trigger is a label we want to check if it is a CoreChanges label, - # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved - environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + outputs: + should_run: ${{ steps.check_should_run.outputs.should_run }} steps: - - name: Check for CoreChanges label - if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + - name: Check for Core changes label + id: check_should_run run: | - echo "This workflow only runs for the CoreChanges label on pull requests" - exit 1 + if [[ ("${{ github.event.label.name }}" == "Core changes" && "${{github.event_name}}" == 'pull_request_target') || "${{github.event_name}}" != 'pull_request_target' ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + fi load-engine-matrix: runs-on: ubuntu-latest # We want to run this job only if the previous job outputs should_run=true, - # Which means that if the trigger was a label we run this job just if the label was CoreChanges - needs: check-label + # Which means that if the trigger was a label we run this job just if the label was Core changes + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} outputs: matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Load the engine matrix id: load-engine-matrix shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges + # We want to run the full matrix only for scheduled runs and Core changes # For other events we want to run only the always matrix which is first and last version supported run: | if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then @@ -90,9 +92,30 @@ jobs: else echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT fi + + load-host-matrix: + runs-on: ubuntu-latest + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} + outputs: + matrix: ${{ steps.load-host-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: load host matrix + id: load-host-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo 'matrix={"include":' $(jq '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json) '}' >> $GITHUB_OUTPUT + else + echo 'matrix={"include":' $(jq '[.[] | select(.run | type == "array" and contains(["pyton"]))]' .github/json_matrices/build-matrix.json) '}' >> $GITHUB_OUTPUT + fi + create-version-matrix: runs-on: ubuntu-latest - needs: check-label + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} outputs: matrix: ${{ steps.create-version-matrix.outputs.matrix }} steps: @@ -103,7 +126,7 @@ jobs: id: create-version-matrix shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges + # We want to run the full matrix only for scheduled runs and Core changes # For other events we want to run only the always matrix which is first and last version supported run: | if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then @@ -112,15 +135,16 @@ jobs: echo 'matrix={"python":["3.8","3.9","3.10","3.11","3.12"]}' >> $GITHUB_OUTPUT fi - test-ubuntu-latest: - runs-on: ubuntu-latest - needs: [load-engine-matrix, create-version-matrix] + test-python-client: + runs-on: ${{ matrix.host.RUNNER }} + needs: [load-engine-matrix, create-version-matrix, load-host-matrix] timeout-minutes: 35 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} + host: ${{ fromJson(needs.load-host-matrix.outputs.matrix).include }} steps: - uses: actions/checkout@v4 with: @@ -177,79 +201,54 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** - test-pubsub-ubuntu-latest: - runs-on: ubuntu-latest - needs: [load-engine-matrix, create-version-matrix] + test-pubsub-python-client: + runs-on: ${{ matrix.host.RUNNER }} + needs: [load-engine-matrix, create-version-matrix, load-host-matrix] timeout-minutes: 35 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} - - test-pubsub: - runs-on: ${{ matrix.host.RUNNER }} - needs: load-engine-matrix - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" - - "3.12" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test pubsub with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} - path: | - python/python/tests/pytest_report.html + host: ${{ fromJson(needs.load-host-matrix.outputs.matrix).include }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test pubsub with pytest + working-directory: ./python + run: | + source .env/bin/activate + cd python/tests/ + pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} + path: | + python/python/tests/pytest_report.html lint: runs-on: ubuntu-latest - needs: check-label + needs: check_should_run + if: ${{ needs.check_should_run.outputs.should_run == 'true' }} timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -288,72 +287,10 @@ jobs: working-directory: ./python run: | black --check --diff . - test-macos-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' - runs-on: macos-latest - needs: load-engine-matrix - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "macos" - target: "aarch64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test with pytest - working-directory: ./python - run: | - source .env/bin/activate - pytest --asyncio-mode=auto - - test-pubsub-macos-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' - runs-on: macos-latest - needs: load-engine-matrix - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "macos" - target: "aarch64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test pubsub with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto -k test_pubsub - build-amazonlinux-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: github.event_name == 'schedule' || github.event.label.name == 'Core changes' runs-on: ubuntu-latest container: amazonlinux:latest - needs: check-label timeout-minutes: 15 steps: - name: Install git