diff --git a/.gitignore b/.gitignore index 4cfeddc..53a24ba 100644 --- a/.gitignore +++ b/.gitignore @@ -353,6 +353,7 @@ yarn.lock package-json.lock *.cs *.ts +!Generation/JavaScript/Build/*.ts !Source/**/Artifacts diff --git a/Generation/JavaScript/Build/Compilers.ts b/Generation/JavaScript/Build/Compilers.ts new file mode 100644 index 0000000..0697978 --- /dev/null +++ b/Generation/JavaScript/Build/Compilers.ts @@ -0,0 +1,24 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import execa from 'execa'; + +const ProtocPath = require.resolve('grpc-tools/bin/protoc.js'); +const ProtocPluginPath = require.resolve('grpc-tools/bin/protoc_plugin.js'); +const ProtocTSPluginPath = require.resolve('grpc_tools_node_protoc_ts/bin/protoc-gen-ts'); + +export const protoc = async (...args: string[]): Promise => { + try { + await execa('node', [ProtocPath, `--plugin=protoc-gen-grpc=${ProtocPluginPath}`, ...args]); + } catch (error) { + throw error.stderr; + } +} + +export const protocTS = async (...args: string[]): Promise => { + try { + await execa('node', [ProtocPath, `--plugin=protoc-gen-ts=${ProtocTSPluginPath}`, ...args]); + } catch (error) { + throw error.stderr; + } +} diff --git a/Generation/JavaScript/Build/GenerateOptions.ts b/Generation/JavaScript/Build/GenerateOptions.ts new file mode 100644 index 0000000..1a5961a --- /dev/null +++ b/Generation/JavaScript/Build/GenerateOptions.ts @@ -0,0 +1,13 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import { GenerationTarget } from './GenerationTarget'; + +export type GenerateOptions = { + readonly target: GenerationTarget; + readonly output: string; + readonly paths: readonly string[]; + readonly includes: readonly string[]; + readonly rewrites: readonly {readonly from: string, readonly to: string, readonly package: boolean}[]; + readonly skipEmptyFiles: boolean; +} \ No newline at end of file diff --git a/Generation/JavaScript/Build/GenerationTarget.ts b/Generation/JavaScript/Build/GenerationTarget.ts new file mode 100644 index 0000000..46c5171 --- /dev/null +++ b/Generation/JavaScript/Build/GenerationTarget.ts @@ -0,0 +1,7 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +export enum GenerationTarget { + Node = 'grpc-node', + Web = 'grpc-web' +} diff --git a/Generation/JavaScript/Build/Generator.ts b/Generation/JavaScript/Build/Generator.ts new file mode 100644 index 0000000..06e531b --- /dev/null +++ b/Generation/JavaScript/Build/Generator.ts @@ -0,0 +1,167 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import os from 'os'; +import process from 'process'; +import { dirname as pathDirname, join as pathJoin, relative as pathRelative, normalize as pathNormalize, resolve as pathResolve } from 'path'; +import { mkdir, mkdtemp, readdir, readFile, rmdir, stat, writeFile } from 'fs/promises'; + +import { protoc, protocTS } from './Compilers'; +import { GenerateOptions } from './GenerateOptions'; +import { GenerationTarget } from './GenerationTarget'; + +const regexpEscape = (expression: string): string => + expression.replace('\\','\\\\').replace('/','\\/').replace('(','\\(').replace(')','\\)'); + +export class Generator { + async generate(options: GenerateOptions): Promise { + try { + console.log('Generating code for', options.target); + console.log('With includes', options.includes.join(' ')); + console.log('With rewrites', options.rewrites.map(_ => `${_.from}:${_.to}`).join(' ')); + console.log('To directory', options.output); + + await this.ensureCleanOutputDirectory(options); + const protoFiles = await this.findAllProtoFilesIn(...options.paths); + + const tmpDir = await this.createTemporaryBuildDirectory(); + + switch (options.target) { + case GenerationTarget.Node: + await this.generateNodeCode(options, protoFiles, tmpDir); + break; + default: + throw new Error(`Target '${options.target}' not implemented`); + } + + await this.moveGeneratedFilesToOutputDirectory(options, tmpDir); + } catch (error) { + console.error('Generation failed', error); + } + } + + private async generateNodeCode(options: GenerateOptions, protoFiles: string[], buildDir: string) { + for (const protoFile of protoFiles) { + console.log('Generating', protoFile); + await protoc( + `--js_out=import_style=commonjs,binary:${buildDir}`, + `--grpc_out=grpc_js:${buildDir}`, + ...options.includes.map(_ => `-I${_}`), + protoFile); + await protocTS( + `--ts_out=grpc_js:${buildDir}`, + ...options.includes.map(_ => `-I${_}`), + protoFile); + } + } + + private async moveGeneratedFilesToOutputDirectory(options: GenerateOptions, buildDir: string): Promise { + const generatedFiles = await this.findAllFilesIn(buildDir); + + for (const generatedFile of generatedFiles) { + const filePath = pathRelative(buildDir, generatedFile); + const rewrittenPath = this.getRewrittenFilePath(options, filePath); + + const contents = (await readFile(generatedFile)).toString(); + const [rewrittenContents, shouldInclude] = this.getRewrittenFileContents(options, contents); + + if (!shouldInclude) continue; + + const movedFilePath = pathJoin(options.output, rewrittenPath); + const movedFileDirectory = pathDirname(movedFilePath); + await mkdir(movedFileDirectory, { recursive: true }); + await writeFile(movedFilePath, rewrittenContents); + } + + await rmdir(buildDir, { recursive: true }); + } + + private getRewrittenFilePath(options: GenerateOptions, filePath: string): string { + for (const rewrite of options.rewrites) { + if (!rewrite.package && filePath.startsWith(rewrite.from)) { + return rewrite.to + filePath.substr(rewrite.from.length); + } + } + return filePath; + } + + private getRewrittenFileContents(options: GenerateOptions, contents: string): [string, boolean] { + if (contents.startsWith('// GENERATED CODE -- NO SERVICES IN PROTO') && options.skipEmptyFiles) { + return [contents, false]; + } + + const importReplacements = this.findImportsToReplaceInContent(options, contents, `from "`, `";`); + const requireReplacements = this.findImportsToReplaceInContent(options, contents, `require('`, `');`); + + for (const [from, to] of importReplacements.concat(requireReplacements)) { + contents = contents.replace(from, to); + } + + return [contents, true]; + } + + private findImportsToReplaceInContent(options: GenerateOptions, contents: string, prefix: string, postfix: string): [string, string][] { + const replacements: [string, string][] = []; + for (const {from, to, package: pkg} of options.rewrites) { + const re = new RegExp(`${regexpEscape(prefix)}(.*${regexpEscape(from)}.*)${regexpEscape(postfix)}`, 'g'); + for (const match of contents.matchAll(re)) { + let [before, after] = match[1].split(from, 2); + + let replacement = match[0]; + if (pkg) { + replacement = prefix+to+after+postfix; + } else { + if (before.endsWith('../')) { + before = before.substr(0, before.length-3); + } + replacement = prefix+before+to+after+postfix; + } + replacements.push([match[0], replacement]); + } + } + return replacements; + } + + private async ensureCleanOutputDirectory(options: GenerateOptions): Promise { + const currentDirectory = pathResolve(process.cwd()); + const outputDirectory = pathResolve(options.output); + if (currentDirectory.startsWith(outputDirectory)) { + console.log('Output directory includes current directory, not cleaning'); + return; + } + try { + const info = await stat(outputDirectory); + if (info.isDirectory()) { + await rmdir(outputDirectory, { recursive: true }); + } else { + throw new Error(`Output directory '${options.output}' is not a directory`); + } + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + + await mkdir(outputDirectory, { recursive: true }); + } + + private async createTemporaryBuildDirectory(): Promise { + return await mkdtemp(pathJoin(os.tmpdir(), 'dolittle-grpc-')); + } + + private async findAllFilesIn(...paths: string[]): Promise { + const files: string[] = []; + for (const path of paths) { + const fileInfo = await stat(path); + if (fileInfo.isDirectory()) { + const filesInDirectory = (await readdir(path)).map(_ => pathJoin(path, _)); + files.push(...await this.findAllFilesIn(...filesInDirectory)); + } else if (fileInfo.isFile()) { + files.push(path); + } + } + return files; + } + + private async findAllProtoFilesIn(...paths: string[]): Promise { + return (await this.findAllFilesIn(...paths)).filter(_ => _.endsWith('.proto')); + } +} \ No newline at end of file diff --git a/Generation/JavaScript/Build/dolittle_proto_build b/Generation/JavaScript/Build/dolittle_proto_build new file mode 100755 index 0000000..9752549 --- /dev/null +++ b/Generation/JavaScript/Build/dolittle_proto_build @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('./Distribution/index.js') diff --git a/Generation/JavaScript/Build/generateAction.ts b/Generation/JavaScript/Build/generateAction.ts new file mode 100644 index 0000000..19d9df8 --- /dev/null +++ b/Generation/JavaScript/Build/generateAction.ts @@ -0,0 +1,35 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import { GenerateOptions } from './GenerateOptions'; +import { GenerationTarget } from './GenerationTarget'; + +type GenerateActionCallback = (options: GenerateOptions) => Promise | void; + +type Options = { + O: string, + I: string[], + R: string[], + skipEmptyFiles?: boolean, +} + +export const generateAction = (target: GenerationTarget, action: GenerateActionCallback) => { + return (paths: string[], options: Options) => { + action({ + target, + output: options.O, + paths, + includes: options.I ?? [], + rewrites: (options.R ?? []).map(_ => { + const [from, to] = _.split(':',2); + const [pkg, pkgName] = to.split('=', 2); + if (pkg === 'pkg') { + return { from, to: pkgName, package: true }; + } else { + return { from, to, package: false }; + } + }), + skipEmptyFiles: !!options.skipEmptyFiles, + }); + } +} \ No newline at end of file diff --git a/Generation/JavaScript/Build/index.ts b/Generation/JavaScript/Build/index.ts new file mode 100644 index 0000000..d20bb86 --- /dev/null +++ b/Generation/JavaScript/Build/index.ts @@ -0,0 +1,39 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import 'process'; +import { Command, Option } from 'commander'; + +import { repeated } from './repeated'; +import { generateAction } from './generateAction'; +import { GenerationTarget } from './GenerationTarget'; +import { Generator } from './Generator'; + +const generator = new Generator(); + +const program = new Command('dolittle_proto_build'); + +const output = new Option('-O ', 'Output path').default('./build'); +const includes = new Option('-I ', 'Include path (multiple allowed)'); +const rewrite = new Option('-R ', 'Rewrite file paths (multiple allowed)'); +const skipEmptyFiles = new Option('--skip-empty-files', 'Remove files generated without any content'); + +program + .command('grpc-node ') + .addOption(output) + .addOption(repeated(includes)) + .addOption(repeated(rewrite)) + .addOption(skipEmptyFiles) + .description('Generate gRPC code for NodeJS') + .action(generateAction(GenerationTarget.Node, options => generator.generate(options))); + +program + .command('grpc-web ') + .addOption(output) + .addOption(repeated(includes)) + .addOption(repeated(rewrite)) + .addOption(skipEmptyFiles) + .description('Generate gRPC code for browsers') + .action(generateAction(GenerationTarget.Web, options => generator.generate(options))); + +program.parse(process.argv); diff --git a/Generation/JavaScript/Build/package.json b/Generation/JavaScript/Build/package.json new file mode 100644 index 0000000..cb8ac41 --- /dev/null +++ b/Generation/JavaScript/Build/package.json @@ -0,0 +1,35 @@ +{ + "name": "@dolittle/protobuf.contracts", + "private": "true", + "author": "Dolittle", + "description": "", + "version": "0.0.0", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/dolittle-tools/JavaScript.Protobuf.git" + }, + "bugs": { + "url": "https://github.com/dolittle-tools/JavaScript.Protobuf/issues" + }, + "homepage": "https://github.com/dolittle-tools/JavaScript.Protobuf#readme", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc" + }, + "bin": { + "dolittle_proto_build": "./dolittle_proto_build" + }, + "dependencies": { + "commander": "7.2.0", + "execa": "5.0.0", + "grpc-tools": "1.11.2", + "grpc_tools_node_protoc_ts": "5.3.2" + }, + "devDependencies": { + "@tsconfig/node12": "1.0.7", + "@types/node": "14.14.37" + } +} diff --git a/Generation/JavaScript/Build/repeated.ts b/Generation/JavaScript/Build/repeated.ts new file mode 100644 index 0000000..4b9b412 --- /dev/null +++ b/Generation/JavaScript/Build/repeated.ts @@ -0,0 +1,15 @@ +// Copyright (c) Dolittle. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import commander from 'commander'; + +const parser = (next: string, previous: string[]): string[] => { + previous = previous ?? []; + previous.push(next); + return previous; +} + +export const repeated = (option: commander.Option): commander.Option => { + option.argParser(parser); + return option; +} diff --git a/Generation/JavaScript/Build/tsconfig.json b/Generation/JavaScript/Build/tsconfig.json new file mode 100644 index 0000000..197e80f --- /dev/null +++ b/Generation/JavaScript/Build/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@tsconfig/node12/tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "./Distribution", + "baseUrl": "./", + }, + "exclude": ["**/*.d.ts", "node_modules"], + "include": ["**/*.ts", "**/*.js"] +} \ No newline at end of file diff --git a/Generation/JavaScript/package.json b/Generation/JavaScript/package.json index 119c439..e789024 100644 --- a/Generation/JavaScript/package.json +++ b/Generation/JavaScript/package.json @@ -1,11 +1,14 @@ { "private": true, "workspaces": [ + "Build", "Fundamentals", "Runtime" ], "devDependencies": { - "@dolittle/protobuf.build": "3.2.0", "typescript": "^3.8.3" - } + }, + "bin": { + "dolittle_proto_build": "Build/dolittle_proto_build" +} } \ No newline at end of file