From e98ad97d8a592027103525434c41c2f4578deba9 Mon Sep 17 00:00:00 2001 From: tjzhang-BQ <111323543+tjzhang-BQ@users.noreply.github.com> Date: Fri, 18 Oct 2024 08:38:43 -0700 Subject: [PATCH] Node: Add CI support for server modules (#2472) * Node: add server modules CI support --------- Signed-off-by: TJ Zhang Signed-off-by: Yury-Fridlyand Signed-off-by: Chloe Co-authored-by: TJ Zhang Co-authored-by: Yury-Fridlyand Co-authored-by: Chloe --- .../workflows/build-node-wrapper/action.yml | 2 +- .../install-shared-dependencies/action.yml | 8 +- .github/workflows/node.yml | 78 +++++++++++++++---- CHANGELOG.md | 1 + node/package.json | 5 +- node/tests/ServerModules.test.ts | 60 ++++++++++++++ node/tests/TestUtilities.ts | 1 + 7 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 node/tests/ServerModules.test.ts diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml index 98246df22f..aa1200fbd5 100644 --- a/.github/workflows/build-node-wrapper/action.yml +++ b/.github/workflows/build-node-wrapper/action.yml @@ -31,7 +31,7 @@ inputs: required: true engine-version: description: "Engine version to install" - required: true + required: false type: string publish: description: "Enable building the wrapper in release mode" diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index abca1966cd..1cb56e63f0 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -22,10 +22,9 @@ inputs: - aarch64-unknown-linux-musl - x86_64-unknown-linux-musl engine-version: - description: "Engine version to install" - required: true - type: string - + description: "Engine version to install" + required: false + type: string github-token: description: "GITHUB_TOKEN, GitHub App installation access token" required: true @@ -72,6 +71,7 @@ runs: github-token: ${{ inputs.github-token }} - name: Install Valkey + if: ${{ inputs.engine-version != '' }} uses: ./.github/workflows/install-valkey with: engine-version: ${{ inputs.engine-version }} diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 32db45e5c5..634219ea15 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -15,6 +15,7 @@ on: - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - .github/json_matrices/build-matrix.json + - .github/workflows/start-self-hosted-runner/action.yml pull_request: paths: - glide-core/src/** @@ -28,6 +29,7 @@ on: - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - .github/json_matrices/build-matrix.json + - .github/workflows/start-self-hosted-runner/action.yml workflow_dispatch: concurrency: @@ -39,17 +41,17 @@ env: jobs: load-engine-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT test-ubuntu-latest: runs-on: ubuntu-latest @@ -84,18 +86,18 @@ jobs: - name: test hybrid node modules - commonjs run: | - npm install --package-lock-only - npm ci - npm run build-and-test + npm install --package-lock-only + npm ci + npm run build-and-test working-directory: ./node/hybrid-node-tests/commonjs-test env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-commonjs.html - name: test hybrid node modules - ecma run: | - npm install --package-lock-only - npm ci - npm run build-and-test + npm install --package-lock-only + npm ci + npm run build-and-test working-directory: ./node/hybrid-node-tests/ecmascript-test env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-ecma.html @@ -269,3 +271,45 @@ jobs: node/test-report*.html utils/clusters/** benchmarks/results/** + + test-modules: + if: (github.repository_owner == 'valkey-io' && github.event_name == 'workflow_dispatch') || github.event.pull_request.head.repo.owner.login == 'valkey-io' + environment: AWS_ACTIONS + name: Running Module Tests + runs-on: [self-hosted, linux, ARM64] + timeout-minutes: 15 + + steps: + - name: Setup self-hosted runner access + run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide + + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Use Node.js 18.x + uses: actions/setup-node@v4 + with: + node-version: 18.x + + - name: Build Node wrapper + uses: ./.github/workflows/build-node-wrapper + with: + os: ubuntu + named_os: linux + arch: arm64 + target: aarch64-unknown-linux-gnu + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: test + run: npm run test-modules -- --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} --tls=true + working-directory: ./node + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-report-node-modules-ubuntu + path: | + node/test-report*.html diff --git a/CHANGELOG.md b/CHANGELOG.md index a90f628944..6dd52a4859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ #### Operational Enhancements * Java: Add modules CI ([#2388](https://github.com/valkey-io/valkey-glide/pull/2388), [#2404](https://github.com/valkey-io/valkey-glide/pull/2404), [#2416](https://github.com/valkey-io/valkey-glide/pull/2416)) +* Node: Add modules CI ([#2472](https://github.com/valkey-io/valkey-glide/pull/2472)) ## 1.1.0 (2024-09-24) diff --git a/node/package.json b/node/package.json index 20dbeea80b..1979e408e4 100644 --- a/node/package.json +++ b/node/package.json @@ -31,14 +31,15 @@ "build-protobuf": "npm run compile-protobuf-files && npm run fix-protobuf-file", "compile-protobuf-files": "cd src && pbjs -t static-module -o ProtobufMessage.js ../../glide-core/src/protobuf/*.proto && pbts -o ProtobufMessage.d.ts ProtobufMessage.js", "fix-protobuf-file": "replace 'this\\.encode\\(message, writer\\)\\.ldelim' 'this.encode(message, writer && writer.len ? writer.fork() : writer).ldelim' src/ProtobufMessage.js", - "test": "npm run build-test-utils && jest --verbose --runInBand --testPathIgnorePatterns='RedisModules'", + "test": "npm run build-test-utils && jest --verbose --runInBand --testPathIgnorePatterns='ServerModules'", "build-test-utils": "cd ../utils && npm i && npm run build", "lint:fix": "npm run install-linting && npx eslint -c ../eslint.config.mjs --fix && npm run prettier:format", "lint": "npm run install-linting && npx eslint -c ../eslint.config.mjs && npm run prettier:check:ci", "install-linting": "cd ../ & npm install", "prepack": "npmignore --auto", "prettier:check:ci": "npx prettier --check . --ignore-unknown '!**/*.{js,d.ts}'", - "prettier:format": "npx prettier --write . --ignore-unknown '!**/*.{js,d.ts}'" + "prettier:format": "npx prettier --write . --ignore-unknown '!**/*.{js,d.ts}'", + "test-modules": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='ServerModules'" }, "devDependencies": { "@jest/globals": "^29.7.0", diff --git a/node/tests/ServerModules.test.ts b/node/tests/ServerModules.test.ts new file mode 100644 index 0000000000..6a11884385 --- /dev/null +++ b/node/tests/ServerModules.test.ts @@ -0,0 +1,60 @@ +/** + * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 + */ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, +} from "@jest/globals"; +import { GlideClusterClient, InfoOptions, ProtocolVersion } from ".."; +import { ValkeyCluster } from "../../utils/TestUtils"; +import { + flushAndCloseClient, + getClientConfigurationOption, + getServerVersion, + parseCommandLineArgs, + parseEndpoints, +} from "./TestUtilities"; + +const TIMEOUT = 50000; +describe("GlideJson", () => { + const testsFailed = 0; + let cluster: ValkeyCluster; + let client: GlideClusterClient; + beforeAll(async () => { + const clusterAddresses = parseCommandLineArgs()["cluster-endpoints"]; + cluster = await ValkeyCluster.initFromExistingCluster( + true, + parseEndpoints(clusterAddresses), + getServerVersion, + ); + }, 20000); + + afterEach(async () => { + await flushAndCloseClient(true, cluster.getAddresses(), client); + }); + + afterAll(async () => { + if (testsFailed === 0) { + await cluster.close(); + } + }, TIMEOUT); + + it.each([ProtocolVersion.RESP2, ProtocolVersion.RESP3])( + "ServerModules check modules loaded", + async (protocol) => { + client = await GlideClusterClient.createClient( + getClientConfigurationOption(cluster.getAddresses(), protocol), + ); + const info = await client.info({ + sections: [InfoOptions.Modules], + route: "randomNode", + }); + expect(info).toContain("# json_core_metrics"); + expect(info).toContain("# search_index_stats"); + }, + ); +}); diff --git a/node/tests/TestUtilities.ts b/node/tests/TestUtilities.ts index 6da0a39f00..c3fac91e09 100644 --- a/node/tests/TestUtilities.ts +++ b/node/tests/TestUtilities.ts @@ -387,6 +387,7 @@ export const getClientConfigurationOption = ( port, })), protocol, + useTLS: parseCommandLineArgs()["tls"] == "true", ...configOverrides, }; };