From e136a034068b49bf299b2335555258db6b37df43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Loren=20=E2=98=BA=EF=B8=8F?= <251288+lorensr@users.noreply.github.com> Date: Sat, 2 Apr 2022 22:08:22 -0400 Subject: [PATCH] Add Protobuf sample (#113) * Update to 0.19.0 * Update update script * Revert "Update to 0.19.0" This reverts commit 7edc75e6c699609235b87844e82497ac76a95d61. * Non-mono * Mono * Fix build * Fix linting * Update interceptors version * Add protobuf sample * Add generated files to .gitignore * README and prettier * Fix proto build Was having this error during building: https://github.com/protobufjs/protobuf.js/issues/1368 --- .eslintignore | 3 +- .prettierignore | 4 +- .scripts/copy-shared-files.mjs | 14 ++++-- README.md | 1 + protobufs/.eslintignore | 4 +- protobufs/.eslintrc.js | 55 ++++++++++++---------- protobufs/.gitignore | 5 +- protobufs/.prettierignore | 2 + protobufs/README.md | 28 +++++++++++ protobufs/package.json | 45 ++++++++++++++++++ protobufs/protos/messages.proto | 9 ++++ protobufs/protos/namespaced-messages.proto | 8 ++++ protobufs/protos/root.js | 8 ++++ protobufs/src/activities.ts | 7 +++ protobufs/src/client.ts | 30 ++++++++++++ protobufs/src/payload-converter.ts | 6 +++ protobufs/src/worker.ts | 20 ++++++++ protobufs/src/workflows.ts | 14 ++++++ 18 files changed, 230 insertions(+), 33 deletions(-) create mode 100644 protobufs/.prettierignore create mode 100644 protobufs/README.md create mode 100644 protobufs/package.json create mode 100644 protobufs/protos/messages.proto create mode 100644 protobufs/protos/namespaced-messages.proto create mode 100644 protobufs/protos/root.js create mode 100644 protobufs/src/activities.ts create mode 100644 protobufs/src/client.ts create mode 100644 protobufs/src/payload-converter.ts create mode 100644 protobufs/src/worker.ts create mode 100644 protobufs/src/workflows.ts diff --git a/.eslintignore b/.eslintignore index 82a9a730..555ee9ca 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1,2 @@ -.eslintrc.js \ No newline at end of file +.eslintrc.js +protobufs/protos/ \ No newline at end of file diff --git a/.prettierignore b/.prettierignore index 6c5c2401..0b40c553 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,4 +4,6 @@ node_modules build lib .next -.vscode \ No newline at end of file +.vscode +protobufs/protos/json-module.js +protobufs/protos/root.d.ts \ No newline at end of file diff --git a/.scripts/copy-shared-files.mjs b/.scripts/copy-shared-files.mjs index 2c584a65..8799b35b 100644 --- a/.scripts/copy-shared-files.mjs +++ b/.scripts/copy-shared-files.mjs @@ -6,9 +6,15 @@ const ADDITIONAL_SAMPLES = []; // Some samples have different config files from those in .shared/ // that we don't want to overwrite const TSCONFIG_EXCLUDE = ['nextjs-ecommerce-oneclick', 'monorepo-folders', 'fetch-esm', 'production', 'hello-world-js']; -const GITIGNORE_EXCLUDE = ['nextjs-ecommerce-oneclick', 'monorepo-folders', 'production', 'hello-world-js']; -const ESLINTRC_EXCLUDE = ['nextjs-ecommerce-oneclick', 'monorepo-folders', 'fetch-esm', 'hello-world-js']; -const ESLINTIGNORE_EXCLUDE = ['production', 'hello-world-js']; +const GITIGNORE_EXCLUDE = [ + 'nextjs-ecommerce-oneclick', + 'monorepo-folders', + 'production', + 'hello-world-js', + 'protobufs', +]; +const ESLINTRC_EXCLUDE = ['nextjs-ecommerce-oneclick', 'monorepo-folders', 'fetch-esm', 'hello-world-js', 'protobufs']; +const ESLINTIGNORE_EXCLUDE = ['production', 'hello-world-js', 'protobufs']; const POST_CREATE_EXCLUDE = [ 'timer-examples', @@ -28,7 +34,7 @@ $.verbose = false; let [answer] = await question( `Running pre-commit hook. -This will overwrite any changes made to most config files in samples (like ${chalk.bold('hello-world/tsconfig.json')}). +This will overwrite changes made to most config files in samples (like ${chalk.bold('hello-world/tsconfig.json')}). Proceed? [Y/n] ` ); diff --git a/README.md b/README.md index b8acfdaf..bf10184e 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Each directory in this repo is a sample Temporal project built with the [TypeScr - [**Patching**](https://docs.temporal.io/docs/typescript/patching/): Patch in new Workflow code when making updates to Workflows that have executions in progress in production. - [**Logging**](https://github.com/temporalio/samples-typescript/tree/main/logging-sinks): Use Sinks to extract data out of Workflows for logging/metrics/tracing purposes. - [**Instrumentation**](https://github.com/temporalio/samples-typescript/tree/main/instrumentation): Use a [winston](https://github.com/winstonjs/winston) logger to get logs out of all SDK components and get metrics and traces out of Rust Core. +- [**Protobufs**](https://github.com/temporalio/samples-typescript/tree/main/protobufs): Use [Protobufs](https://docs.temporal.io/docs/typescript/data-converters#protobufs). - [**Custom Payload Converter**](https://github.com/temporalio/samples-typescript/tree/main/ejson): Customize data serialization by creating a `PayloadConverter` that uses EJSON to convert Dates, binary, and regexes. #### Advanced APIs diff --git a/protobufs/.eslintignore b/protobufs/.eslintignore index 7bd99a41..d30e19dd 100644 --- a/protobufs/.eslintignore +++ b/protobufs/.eslintignore @@ -1,3 +1,5 @@ node_modules lib -.eslintrc.js \ No newline at end of file +.eslintrc.js +protos + diff --git a/protobufs/.eslintrc.js b/protobufs/.eslintrc.js index 0cbdf98e..7aaed8d2 100644 --- a/protobufs/.eslintrc.js +++ b/protobufs/.eslintrc.js @@ -1,33 +1,38 @@ module.exports = { root: true, parser: '@typescript-eslint/parser', - parserOptions: { - project: './tsconfig.json', - tsconfigRootDir: __dirname, - }, plugins: ['@typescript-eslint', 'deprecation'], - extends: [ - 'eslint:recommended', - 'plugin:@typescript-eslint/eslint-recommended', - 'plugin:@typescript-eslint/recommended', - 'prettier', - ], - rules: { - // recommended for safety - '@typescript-eslint/no-floating-promises': 'error', // forgetting to await Activities and Workflow APIs is bad - 'deprecation/deprecation': 'warn', + overrides: [ + { + files: ['*.ts'], + parserOptions: { + project: './tsconfig.json', + tsconfigRootDir: __dirname, + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/eslint-recommended', + 'plugin:@typescript-eslint/recommended', + 'prettier', + ], + rules: { + // recommended for safety + '@typescript-eslint/no-floating-promises': 'error', // forgetting to await Activities and Workflow APIs is bad + 'deprecation/deprecation': 'warn', - // code style preference - 'object-shorthand': ['error', 'always'], + // code style preference + 'object-shorthand': ['error', 'always'], - // relaxed rules, for convenience - '@typescript-eslint/no-unused-vars': [ - 'warn', - { - argsIgnorePattern: '^_', - varsIgnorePattern: '^_', + // relaxed rules, for convenience + '@typescript-eslint/no-unused-vars': [ + 'warn', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + }, + ], + '@typescript-eslint/no-explicit-any': 'off', + }, }, - ], - '@typescript-eslint/no-explicit-any': 'off', - }, + }, + ], }; diff --git a/protobufs/.gitignore b/protobufs/.gitignore index a9f4ed54..1a1f34ca 100644 --- a/protobufs/.gitignore +++ b/protobufs/.gitignore @@ -1,2 +1,5 @@ lib -node_modules \ No newline at end of file +node_modules +protos/json-module.js +protos/root.d.ts + diff --git a/protobufs/.prettierignore b/protobufs/.prettierignore new file mode 100644 index 00000000..6cfeae5b --- /dev/null +++ b/protobufs/.prettierignore @@ -0,0 +1,2 @@ +protos/json-module.js +protos/root.d.ts \ No newline at end of file diff --git a/protobufs/README.md b/protobufs/README.md new file mode 100644 index 00000000..046aa6fa --- /dev/null +++ b/protobufs/README.md @@ -0,0 +1,28 @@ +# Protobufs + +Use [Protobufs](https://docs.temporal.io/docs/typescript/data-converters#protobufs). + +- Example proto files: + - [protos/messages.proto](protos/messages.proto) + - [protos/namespaced-messages.proto](protos/namespaced-messages.proto) +- Scripts for compiling protos: [package.json](package.json) +- Root file: [protos/root.js](protos/root.js) +- Payload Converter: [src/payload-converter.ts](src/payload-converter.ts) + +We provide the Payload Converter to the Client ([src/client.ts](src/client.ts)) and Worker ([src/worker.ts](src/worker.ts)), and then we can use Protobufs in the Client, Workflow ([src/workflows.ts](src/workflows.ts)), and Activity ([src/activities.ts](src/activities.ts)). + +### Running this sample + +1. Make sure Temporal Server is running locally (see the [quick install guide](https://docs.temporal.io/docs/server/quick-install/)). +1. `npm install` to install dependencies. +1. `npm run start.watch` to start the Worker. +1. In another shell, `npm run workflow` to run the Workflow Client. + +The client should log the Workflow ID that is started, and you should see it reflected in Temporal Web UI. + +Optionally, you can also uncomment the `await handle.result()`, rerun, and see the client script return: + +```bash +Started workflow my-business-id-b6155489-920f-41a8-9e88-c17c24d47ee9 +{ sentence: 'Proto is 2 years old.' } +``` diff --git a/protobufs/package.json b/protobufs/package.json new file mode 100644 index 00000000..09c20323 --- /dev/null +++ b/protobufs/package.json @@ -0,0 +1,45 @@ +{ + "name": "temporal-protobufs", + "version": "0.1.0", + "private": true, + "scripts": { + "start": "ts-node src/worker.ts", + "start.watch": "nodemon src/worker.ts", + "workflow": "ts-node src/client.ts", + "build": "npm-run-all build:protos build:ts", + "build.watch": "npm-run-all build:protos build:ts-watch", + "build:ts": "tsc --build", + "build:ts-watch": "tsc --build --watch", + "build:protos": "pbjs -t json-module -w commonjs -r protobuf-sample -o protos/json-module.js protos/*.proto & pbjs -t static-module protos/*.proto | pbts -o protos/root.d.ts -", + "lint": "eslint ." + }, + "nodemonConfig": { + "execMap": { + "ts": "ts-node" + }, + "ext": "ts", + "watch": [ + "src" + ] + }, + "dependencies": { + "temporalio": "0.19.x", + "uuid": "^8.3.2" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.0", + "@types/uuid": "^8.3.4", + "@typescript-eslint/eslint-plugin": "^5.0.0", + "@typescript-eslint/parser": "^5.0.0", + "eslint": "^7.32.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-deprecation": "^1.2.1", + "installing": "^1.0.0", + "jsdoc": "^3.6.10", + "nodemon": "^2.0.12", + "npm-run-all": "^4.1.5", + "protobufjs": "^6.11.2", + "ts-node": "^10.2.1", + "typescript": "^4.4.2" + } +} diff --git a/protobufs/protos/messages.proto b/protobufs/protos/messages.proto new file mode 100644 index 00000000..12151a9c --- /dev/null +++ b/protobufs/protos/messages.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +message ProtoResult { + string sentence = 1; +} + +message BinaryMessage { + bytes data = 1; +} \ No newline at end of file diff --git a/protobufs/protos/namespaced-messages.proto b/protobufs/protos/namespaced-messages.proto new file mode 100644 index 00000000..651c113f --- /dev/null +++ b/protobufs/protos/namespaced-messages.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package foo.bar; + +message ProtoInput { + string name = 1; + int32 age = 2; +} \ No newline at end of file diff --git a/protobufs/protos/root.js b/protobufs/protos/root.js new file mode 100644 index 00000000..86528606 --- /dev/null +++ b/protobufs/protos/root.js @@ -0,0 +1,8 @@ +// json-module.js is generated with: +// pbjs -t json-module -w commonjs -o json-module.js protos/*.proto + +// @@@SNIPSTART typescript-protobuf-root +const { patchProtobufRoot } = require('@temporalio/common/lib/converter/patch-protobuf-root'); +const unpatchedRoot = require('./json-module'); +module.exports = patchProtobufRoot(unpatchedRoot); +// @@@SNIPEND diff --git a/protobufs/src/activities.ts b/protobufs/src/activities.ts new file mode 100644 index 00000000..0d5a9410 --- /dev/null +++ b/protobufs/src/activities.ts @@ -0,0 +1,7 @@ +// @@@SNIPSTART typescript-protobuf-activity +import { foo, ProtoResult } from '../protos/root'; + +export async function protoActivity(input: foo.bar.ProtoInput): Promise { + return ProtoResult.create({ sentence: `${input.name} is ${input.age} years old.` }); +} +// @@@SNIPEND diff --git a/protobufs/src/client.ts b/protobufs/src/client.ts new file mode 100644 index 00000000..4c060dea --- /dev/null +++ b/protobufs/src/client.ts @@ -0,0 +1,30 @@ +// @@@SNIPSTART typescript-protobuf-client +import { Connection, WorkflowClient } from '@temporalio/client'; +import { v4 as uuid } from 'uuid'; +import { foo, ProtoResult } from '../protos/root'; +import { example } from './workflows'; + +async function run() { + const client = new WorkflowClient(new Connection().service, { + dataConverter: { payloadConverterPath: require.resolve('./payload-converter') }, + }); + + const handle = await client.start(example, { + args: [foo.bar.ProtoInput.create({ name: 'Proto', age: 2 })], + // can't do: + // args: [new foo.bar.ProtoInput({ name: 'Proto', age: 2 })], + taskQueue: 'protobufs', + workflowId: 'my-business-id-' + uuid(), + }); + + console.log(`Started workflow ${handle.workflowId}`); + + const result: ProtoResult = await handle.result(); + console.log(result.toJSON()); +} +// @@@SNIPEND + +run().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/protobufs/src/payload-converter.ts b/protobufs/src/payload-converter.ts new file mode 100644 index 00000000..811e6e29 --- /dev/null +++ b/protobufs/src/payload-converter.ts @@ -0,0 +1,6 @@ +// @@@SNIPSTART typescript-protobuf-converter +import { DefaultPayloadConverterWithProtobufs } from '@temporalio/common/lib/protobufs'; +import root from '../protos/root'; + +export const payloadConverter = new DefaultPayloadConverterWithProtobufs({ protobufRoot: root }); +// @@@SNIPEND diff --git a/protobufs/src/worker.ts b/protobufs/src/worker.ts new file mode 100644 index 00000000..a254ddf0 --- /dev/null +++ b/protobufs/src/worker.ts @@ -0,0 +1,20 @@ +import { Worker } from '@temporalio/worker'; +import * as activities from './activities'; + +async function run() { + // @@@SNIPSTART typescript-protobuf-worker + const worker = await Worker.create({ + workflowsPath: require.resolve('./workflows'), + activities, + taskQueue: 'protobufs', + dataConverter: { payloadConverterPath: require.resolve('./payload-converter') }, + }); + // @@@SNIPEND + + await worker.run(); +} + +run().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/protobufs/src/workflows.ts b/protobufs/src/workflows.ts new file mode 100644 index 00000000..a09f4fad --- /dev/null +++ b/protobufs/src/workflows.ts @@ -0,0 +1,14 @@ +// @@@SNIPSTART typescript-protobuf-workflow +import { proxyActivities } from '@temporalio/workflow'; +import { foo, ProtoResult } from '../protos/root'; +import type * as activities from './activities'; + +const { protoActivity } = proxyActivities({ + startToCloseTimeout: '1 minute', +}); + +export async function example(input: foo.bar.ProtoInput): Promise { + const result = await protoActivity(input); + return result; +} +// @@@SNIPEND