limit pr run to ubuntu latest X first and last version supported, add…

.github/DEVELOPER.md

@@ -0,0 +1,110 @@
+# CI/CD Workflow Guide
+
+<span style="color: green;">
+TODO: Add a description of the CI/CD workflow and its components.
+</span>
+
+### Overview
+
+Our CI/CD pipeline tests and builds our project across multiple languages, versions, and environments. This guide outlines the key components and processes of our workflow.
+
+### Workflow Triggers
+
+-   Push to `main` branch
+-   Pull requests
+-   Scheduled runs (daily)
+-   Manual trigger (workflow_dispatch)
+-   Pull request with `Core changes` label
+
+### Language-Specific Workflows
+
+#### Python (python.yml)
+
+#### Node.js (node.yml)
+
+#### Java (java.yml)
+
+Each language has its own workflow file with similar structure but language-specific steps.
+
+### Shared Components
+
+#### Run Field
+
+The `run` field in both engine-matrix.json and build-matrix.json is used to specify when a particular configuration should be executed:
+
+-   In engine-matrix.json:
+
+    -   `"run": "always"`: The engine version will be tested in every workflow run.
+    -   If `run` is omitted: The engine version will only be tested in full matrix runs (e.g., when the `Core changes` label is applied).
+
+-   In build-matrix.json:
+    -   The `run` array specifies which language workflows should use this host configuration.
+    -   `"always"` in the array means the host will be used in every workflow run for the specified languages.
+
+This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage.
+
+#### Engine Matrix (engine-matrix.json)
+
+Defines the versions of Valkey engine to test against:
+
+```json
+[
+    { "type": "valkey", "version": "7.2.5", "run": "always" },
+    { "type": "valkey", "version": "redis-7.0.15" },
+    { "type": "valkey", "version": "redis-6.2.14", "run": "always" }
+]
+```
+
+#### Build Matrix (build-matrix.json)
+
+Defines the host environments for testing:
+
+```json
+[
+    {
+        "OS": "ubuntu",
+        "RUNNER": "ubuntu-latest",
+        "TARGET": "x86_64-unknown-linux-gnu",
+        "run": ["always", "python", "node", "java"]
+    }
+    // ... other configurations
+]
+```
+
+#### Adding New Versions or Hosts
+
+To add a new engine version, update engine-matrix.json.
+To add a new host environment, update build-matrix.json.
+
+Ensure new entries have all required fields:
+
+For engine versions: `type`, `version`, `run` (optional)
+For hosts: `OS`, `RUNNER`, `TARGET`, `run`
+
+#### Triggering Workflows
+
+Push to main or create a pull request to run workflows automatically.
+Add `Core changes` label to a pull request to trigger full matrix tests.
+Use workflow_dispatch for manual triggers.
+
+### Mutual vs. Language-Specific Components
+
+#### Mutual
+
+`Engine matrix`
+`Build matrix`
+`Shared dependencies installation`
+`Linting (Rust)`
+
+#### Language-Specific
+
+`Package manager commands`
+`Testing frameworks`
+`Build processes`
+`Version matrices`
+
+### Customizing Workflows
+
+Modify `[language].yml` files to adjust language-specific steps.
+Update matrix files to change tested versions or environments.
+Adjust cron schedules in workflow files for different timing of scheduled runs.

.github/json_matrices/build-matrix.json

@@ -5,7 +5,8 @@
         "RUNNER": "ubuntu-latest",
         "ARCH": "x64",
         "TARGET": "x86_64-unknown-linux-gnu",
-        "PACKAGE_MANAGERS": ["pypi", "npm"]
+        "PACKAGE_MANAGERS": ["pypi", "npm"],
+        "run": ["always", "python", "node", "java"]
     },
     {
         "OS": "ubuntu",
@@ -14,23 +15,26 @@
         "ARCH": "arm64",
         "TARGET": "aarch64-unknown-linux-gnu",
         "PACKAGE_MANAGERS": ["pypi", "npm"],
-        "CONTAINER": "2_28"
+        "CONTAINER": "2_28",
+        "run": ["python", "node"]
     },
     {
         "OS": "macos",
         "NAMED_OS": "darwin",
         "RUNNER": "macos-12",
         "ARCH": "x64",
         "TARGET": "x86_64-apple-darwin",
-        "PACKAGE_MANAGERS": ["pypi", "npm"]
+        "PACKAGE_MANAGERS": ["pypi", "npm"],
+        "run": ["python", "node"]
     },
     {
         "OS": "macos",
         "NAMED_OS": "darwin",
         "RUNNER": "macos-latest",
         "ARCH": "arm64",
         "TARGET": "aarch64-apple-darwin",
-        "PACKAGE_MANAGERS": ["pypi", "npm"]
+        "PACKAGE_MANAGERS": ["pypi", "npm"],
+        "run": ["python", "node", "java"]
     },
     {
         "OS": "ubuntu",
@@ -40,7 +44,8 @@
         "RUNNER": ["self-hosted", "Linux", "ARM64"],
         "IMAGE": "node:alpine",
         "CONTAINER_OPTIONS": "--user root --privileged --rm",
-        "PACKAGE_MANAGERS": ["npm"]
+        "PACKAGE_MANAGERS": ["npm"],
+        "run": "node"
     },
     {
         "OS": "ubuntu",
@@ -50,6 +55,7 @@
         "RUNNER": "ubuntu-latest",
         "IMAGE": "node:alpine",
         "CONTAINER_OPTIONS": "--user root --privileged",
-        "PACKAGE_MANAGERS": ["npm"]
+        "PACKAGE_MANAGERS": ["npm"],
+        "run": "node"
     }
 ]

.github/json_matrices/engine-matrix.json

@@ -1,6 +1,16 @@
 [
-  {
-    "type": "valkey",
-    "version": "7.2.5"
-  }
+    {
+        "type": "valkey",
+        "version": "7.2.5",
+        "run": "always"
+    },
+    {
+        "type": "valkey",
+        "version": "redis-7.0.15"
+    },
+    {
+        "type": "valkey",
+        "version": "redis-6.2.14",
+        "run": "always"
+    }
 ]

.github/workflows/java.yml

@@ -26,47 +26,115 @@ on:
             - .github/json_matrices/build-matrix.json
     workflow_dispatch:
 
+    pull_request_target:
+        types: [labeled]
+        paths:
+            - glide-core/src/**
+            - submodules/**
+            - java/**
+            - .github/workflows/java.yml
+            - .github/workflows/install-shared-dependencies/action.yml
+            - .github/workflows/test-benchmark/action.yml
+            - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
+            - .github/json_matrices/build-matrix.json
+    schedule:
+        - cron: "0 0 * * *"
+
 concurrency:
     group: java-${{ github.head_ref || github.ref }}
     cancel-in-progress: true
 
 jobs:
+    check_should_run:
+      runs-on: ubuntu-latest
+      outputs:
+        should_run: ${{ steps.check_should_run.outputs.should_run }}
+      steps:
+        - name: Check for Core changes label
+          id: check_should_run
+          run: |
+            if [[ ("${{ github.event.label.name }}" == "Core changes" &&  "${{github.event_name}}" == 'pull_request_target') || "${{github.event_name}}" != 'pull_request_target' ]]; then
+              echo "should_run=true" >> $GITHUB_OUTPUT
+            else
+              echo "should_run=false" >> $GITHUB_OUTPUT
+            fi
+
     load-engine-matrix:
-        runs-on: ubuntu-latest
-        outputs:
-            matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
-        steps:
-            - name: Checkout
-              uses: actions/checkout@v4
+      runs-on: ubuntu-latest
+      # We want to run this job only if the previous job outputs should_run=true,
+      # Which means that if the trigger was a label we run this job just if the label was Core changes
+      needs: check_should_run  
+      if: ${{ needs.check_should_run.outputs.should_run == 'true' }}   
+      outputs:
+        matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
+      steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+
+          - name: Load the engine matrix
+            id: load-engine-matrix
+            shell: bash
+            run: |
+              if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then
+                echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
+              else
+                echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
+              fi
+
+    load-host-matrix:
+      runs-on: ubuntu-latest
+      needs: check_should_run
+      if: ${{ needs.check_should_run.outputs.should_run == 'true' }}
+      outputs:
+          matrix: ${{ steps.load-host-matrix.outputs.matrix }}
+      steps:
+          - name: Checkout
+            uses: actions/checkout@v4
 
-            - name: Load the engine matrix
-              id: load-engine-matrix
-              shell: bash
-              run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
+          - name: load host matrix
+            id: load-host-matrix
+            shell: bash
+            run: |
+              if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then
+                echo 'matrix={"include":' $(jq '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json) '}' >> $GITHUB_OUTPUT
+              else
+                echo 'matrix={"include":' $(jq '[.[] | select(.run | type == "array" and contains(["java"]))]' .github/json_matrices/build-matrix.json) '}' >> $GITHUB_OUTPUT
+              fi
+
+    create-version-matrix:
+      runs-on: ubuntu-latest
+      needs: check_should_run
+      if: ${{ needs.check_should_run.outputs.should_run == 'true' }}
+      outputs:
+          matrix: ${{ steps.create-version-matrix.outputs.matrix }}
+      steps:
+          - name: Checkout
+            uses: actions/checkout@v4
+
+          - name: Create version matrix
+            id: create-version-matrix
+
+            shell: bash
+            # We want to run the full matrix only for scheduled runs and Core changes
+            # For other events we want to run only the always matrix which is first and last version supported
+            run: |
+              if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then
+                echo 'matrix={"java":["11","17"]}' >> $GITHUB_OUTPUT
+              else
+                echo 'matrix={"java":["11","17"]}' >> $GITHUB_OUTPUT
+              fi
 
     build-and-test-java-client:
-        needs: load-engine-matrix
+        needs: [load-engine-matrix, load-host-matrix, create-version-matrix]
         timeout-minutes: 35
         strategy:
             # Run all jobs
             fail-fast: false
             matrix:
-                java:
-                    # - 11
-                    - 17
+                java: ${{ fromJson(needs.create-version-matrix.outputs.matrix).java }}
                 engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
-                host:
-                    - {
-                        OS: ubuntu,
-                        RUNNER: ubuntu-latest,
-                        TARGET: x86_64-unknown-linux-gnu
-                    }
-                    # - {
-                    #     OS: macos,
-                    #     RUNNER: macos-latest,
-                    #     TARGET: aarch64-apple-darwin
-                    # }
-
+                host: ${{ fromJson(needs.load-host-matrix.outputs.matrix).include }}
         runs-on: ${{ matrix.host.RUNNER }}
 
         steps:
@@ -122,14 +190,14 @@ jobs:
                       java/client/build/reports/spotbugs/**
 
     build-amazonlinux-latest:
-        if: github.repository_owner == 'valkey-io'
+        if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event.label.name == 'Core changes')
+        needs: [load-engine-matrix, create-version-matrix]
         strategy:
             # Run all jobs
             fail-fast: false
             matrix:
-                java:
-                    # - 11
-                    - 17
+                java: ${{ fromJson(needs.create-version-matrix.outputs.matrix).java }}
+                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
         runs-on: ubuntu-latest
         container: amazonlinux:latest
         timeout-minutes: 35
@@ -156,7 +224,7 @@ jobs:
                   os: "amazon-linux"
                   target: "x86_64-unknown-linux-gnu"
                   github-token: ${{ secrets.GITHUB_TOKEN }}
-                  engine-version: "7.2.5"
+                  engine-version: ${{ matrix.engine.version }}
 
             - name: Install protoc (protobuf)
               uses: arduino/setup-protoc@v3
@@ -186,6 +254,8 @@ jobs:
     lint-rust:
         timeout-minutes: 15
         runs-on: ubuntu-latest
+        needs: check_should_run
+        if: ${{ needs.check_should_run.outputs.should_run == 'true' }}
         steps:
             - uses: actions/checkout@v4
               with:

.github/workflows/lint-rust/action.yml

@@ -24,8 +24,6 @@ runs:
               github-token: ${{ inputs.github-token }}
 
         - uses: Swatinem/rust-cache@v2
-          with:
-              github-token: ${{ inputs.github-token }}
 
         - run: cargo fmt --all -- --check
           working-directory: ${{ inputs.cargo-toml-folder }}