Merge pull request #95 from dolittle/use-new-protobuf-msbuild

Update versions

.github/workflows/ci.yml

@@ -75,6 +75,9 @@ jobs:
           files-to-update: |
             Generation/JavaScript/Fundamentals/VersionInfo.js
             Generation/JavaScript/Runtime/VersionInfo.js
+      - name: Build
+        working-directory: ./Generation/JavaScript/Build
+        run: yarn build
       - name: Build Fundamentals
         working-directory: ./Generation/JavaScript/Fundamentals
         run: yarn build
@@ -175,6 +178,9 @@ jobs:
           files-to-update: |
             Generation/JavaScript/Fundamentals/VersionInfo.js
             Generation/JavaScript/Runtime/VersionInfo.js
+      - name: Build
+        working-directory: ./Generation/JavaScript/Build
+        run: yarn build
       - name: Build Fundamentals
         working-directory: ./Generation/JavaScript/Fundamentals
         run: yarn build

.gitignore

@@ -353,6 +353,8 @@ yarn.lock
 package-json.lock
 *.cs
 *.ts
+!Generation/JavaScript/Build/*.ts
+Generation/JavaScript/Build/Distribution
 
 !Source/**/Artifacts
 

Generation/CSharp/Fundamentals/Fundamentals.csproj

@@ -1,14 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
+    <Import Project="../../../default.props"/>
     <PropertyGroup>
-        <TargetFramework>netstandard2.0</TargetFramework>
         <AssemblyName>Dolittle.Contracts</AssemblyName>
-        <DolittleProtoProject>../../../Source/Fundamentals</DolittleProtoProject>
-        <DolittleProtoRoot>../../../Source</DolittleProtoRoot>
-        <DolittleProtoKeepFiles>VersionInfo.cs</DolittleProtoKeepFiles>
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Dolittle.Common" Version="2.*" PrivateAssets="All"/>
-        <PackageReference Include="Dolittle.Protobuf.MSBuild" Version="3.3.1"/>
+        <Protobuf Include="../../../Source/Fundamentals/**/*.proto"
+                    ProtoRoot="../../../Source"
+                    OutputDir="%(RecursiveDir)"
+                    GrpcServices="None" />
     </ItemGroup>
 </Project>

Generation/CSharp/Runtime/Runtime.csproj

@@ -1,18 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
+    <Import Project="../../../default.props"/>
     <PropertyGroup>
-        <TargetFramework>netstandard2.0</TargetFramework>
         <AssemblyName>Dolittle.Runtime.Contracts</AssemblyName>
-        <DolittleProtoProject>../../../Source/Runtime</DolittleProtoProject>
-        <DolittleProtoRoot>../../../Source</DolittleProtoRoot>
-        <DolittleProtoKeepFiles>VersionInfo.cs</DolittleProtoKeepFiles>
     </PropertyGroup>
 
     <ItemGroup>
         <ProjectReference Include="../Fundamentals/Fundamentals.csproj" />
     </ItemGroup>
-
     <ItemGroup>
-        <PackageReference Include="Dolittle.Common" Version="2.*" PrivateAssets="All"/>
-        <PackageReference Include="Dolittle.Protobuf.MSBuild" Version="3.3.1"/>
+        <Protobuf Include="../../../Source/Runtime/**/*.proto"
+                    ProtoRoot="../../../Source"
+                    OutputDir="%(RecursiveDir)"
+                    GrpcServices="Both" />
     </ItemGroup>
+
 </Project>

Generation/JavaScript/Build/Compilers.ts

@@ -0,0 +1,24 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import execa from 'execa';
+
+const ProtocPath = require.resolve('grpc-tools/bin/protoc.js');
+const ProtocPluginPath = require.resolve('grpc-tools/bin/protoc_plugin.js');
+const ProtocTSPluginPath = require.resolve('grpc_tools_node_protoc_ts/bin/protoc-gen-ts');
+
+export const protoc = async (...args: string[]): Promise<void> => {
+    try {
+        await execa('node', [ProtocPath, `--plugin=protoc-gen-grpc=${ProtocPluginPath}`, ...args]);
+    } catch (error) {
+        throw error.stderr;
+    }
+}
+
+export const protocTS = async (...args: string[]): Promise<void> => {
+    try {
+        await execa('node', [ProtocPath, `--plugin=protoc-gen-ts=${ProtocTSPluginPath}`, ...args]);
+    } catch (error) {
+        throw error.stderr;
+    }
+}

Generation/JavaScript/Build/GenerateOptions.ts

@@ -0,0 +1,13 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { GenerationTarget } from './GenerationTarget';
+
+export type GenerateOptions = {
+    readonly target: GenerationTarget;
+    readonly output: string;
+    readonly paths: readonly string[];
+    readonly includes: readonly string[];
+    readonly rewrites: readonly {readonly from: string, readonly to: string, readonly package: boolean}[];
+    readonly skipEmptyFiles: boolean;
+}

Generation/JavaScript/Build/GenerationTarget.ts

@@ -0,0 +1,7 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+export enum GenerationTarget {
+    Node = 'grpc-node',
+    Web = 'grpc-web'
+}

Generation/JavaScript/Build/Generator.ts

@@ -0,0 +1,167 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import os from 'os';
+import process from 'process';
+import { dirname as pathDirname, join as pathJoin, relative as pathRelative, normalize as pathNormalize, resolve as pathResolve } from 'path';
+import { mkdir, mkdtemp, readdir, readFile, rmdir, stat, writeFile } from 'fs/promises';
+
+import { protoc, protocTS } from './Compilers';
+import { GenerateOptions } from './GenerateOptions';
+import { GenerationTarget } from './GenerationTarget';
+
+const regexpEscape = (expression: string): string => 
+    expression.replace('\\','\\\\').replace('/','\\/').replace('(','\\(').replace(')','\\)');
+
+export class Generator {
+    async generate(options: GenerateOptions): Promise<void> {
+        try {
+            console.log('Generating code for', options.target);
+            console.log('With includes', options.includes.join(' '));
+            console.log('With rewrites', options.rewrites.map(_ => `${_.from}:${_.to}`).join(' '));
+            console.log('To directory', options.output);
+
+            await this.ensureCleanOutputDirectory(options);
+            const protoFiles = await this.findAllProtoFilesIn(...options.paths);
+
+            const tmpDir = await this.createTemporaryBuildDirectory();
+
+            switch (options.target) {
+                case GenerationTarget.Node:
+                    await this.generateNodeCode(options, protoFiles, tmpDir);
+                    break;
+                default:
+                    throw new Error(`Target '${options.target}' not implemented`);
+            }
+
+            await this.moveGeneratedFilesToOutputDirectory(options, tmpDir);
+        } catch (error) {
+            console.error('Generation failed', error);
+        }
+    }
+
+    private async generateNodeCode(options: GenerateOptions, protoFiles: string[], buildDir: string) {
+        for (const protoFile of protoFiles) {
+            console.log('Generating', protoFile);
+            await protoc(
+                `--js_out=import_style=commonjs,binary:${buildDir}`,
+                `--grpc_out=grpc_js:${buildDir}`,
+                ...options.includes.map(_ => `-I${_}`),
+                protoFile);
+            await protocTS(
+                `--ts_out=grpc_js:${buildDir}`,
+                ...options.includes.map(_ => `-I${_}`),
+                protoFile);
+        }
+    }
+
+    private async moveGeneratedFilesToOutputDirectory(options: GenerateOptions, buildDir: string): Promise<void> {
+        const generatedFiles = await this.findAllFilesIn(buildDir);
+
+        for (const generatedFile of generatedFiles) {
+            const filePath = pathRelative(buildDir, generatedFile);
+            const rewrittenPath = this.getRewrittenFilePath(options, filePath);
+
+            const contents = (await readFile(generatedFile)).toString();
+            const [rewrittenContents, shouldInclude] = this.getRewrittenFileContents(options, contents);
+
+            if (!shouldInclude) continue;
+
+            const movedFilePath = pathJoin(options.output, rewrittenPath);
+            const movedFileDirectory = pathDirname(movedFilePath);
+            await mkdir(movedFileDirectory, { recursive: true });
+            await writeFile(movedFilePath, rewrittenContents);
+        }
+
+        await rmdir(buildDir, { recursive: true });
+    }
+
+    private getRewrittenFilePath(options: GenerateOptions, filePath: string): string {
+        for (const rewrite of options.rewrites) {
+            if (!rewrite.package && filePath.startsWith(rewrite.from)) {
+                return rewrite.to + filePath.substr(rewrite.from.length);
+            }
+        }
+        return filePath;
+    }
+
+    private getRewrittenFileContents(options: GenerateOptions, contents: string): [string, boolean] {
+        if (contents.startsWith('// GENERATED CODE -- NO SERVICES IN PROTO') && options.skipEmptyFiles) {
+            return [contents, false];
+        }
+
+        const importReplacements = this.findImportsToReplaceInContent(options, contents, `from "`, `";`);
+        const requireReplacements = this.findImportsToReplaceInContent(options, contents, `require('`, `');`);
+
+        for (const [from, to] of importReplacements.concat(requireReplacements)) {
+            contents = contents.replace(from, to);
+        }
+
+        return [contents, true];
+    }
+
+    private findImportsToReplaceInContent(options: GenerateOptions, contents: string, prefix: string, postfix: string): [string, string][] {
+        const replacements: [string, string][] = [];
+        for (const {from, to, package: pkg} of options.rewrites) {
+            const re = new RegExp(`${regexpEscape(prefix)}(.*${regexpEscape(from)}.*)${regexpEscape(postfix)}`, 'g');
+            for (const match of contents.matchAll(re)) {
+                let [before, after] = match[1].split(from, 2);
+
+                let replacement = match[0];
+                if (pkg) {
+                    replacement = prefix+to+after+postfix;
+                } else {
+                    if (before.endsWith('../')) {
+                        before = before.substr(0, before.length-3);
+                    }
+                    replacement = prefix+before+to+after+postfix;
+                }
+                replacements.push([match[0], replacement]);
+            }
+        }
+        return replacements;
+    }
+
+    private async ensureCleanOutputDirectory(options: GenerateOptions): Promise<void> {
+        const currentDirectory = pathResolve(process.cwd());
+        const outputDirectory = pathResolve(options.output);
+        if (currentDirectory.startsWith(outputDirectory)) {
+            console.log('Output directory includes current directory, not cleaning');
+            return;
+        }
+        try {
+            const info = await stat(outputDirectory);
+            if (info.isDirectory()) {
+                await rmdir(outputDirectory, { recursive: true });
+            } else {
+                throw new Error(`Output directory '${options.output}' is not a directory`);
+            }
+        } catch (error) {
+            if (error.code !== 'ENOENT') throw error;
+        }
+
+        await mkdir(outputDirectory, { recursive: true });
+    }
+
+    private async createTemporaryBuildDirectory(): Promise<string> {
+        return await mkdtemp(pathJoin(os.tmpdir(), 'dolittle-grpc-'));
+    }
+
+    private async findAllFilesIn(...paths: string[]): Promise<string[]> {
+        const files: string[] = [];
+        for (const path of paths) {
+            const fileInfo = await stat(path);
+            if (fileInfo.isDirectory()) {
+                const filesInDirectory = (await readdir(path)).map(_ => pathJoin(path, _));
+                files.push(...await this.findAllFilesIn(...filesInDirectory));
+            } else if (fileInfo.isFile()) {
+                files.push(path);
+            }
+        }
+        return files;
+    }
+
+    private async findAllProtoFilesIn(...paths: string[]): Promise<string[]> {
+        return (await this.findAllFilesIn(...paths)).filter(_ => _.endsWith('.proto'));
+    }
+}

Generation/JavaScript/Build/dolittle_proto_build

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('./Distribution/index.js')

Generation/JavaScript/Build/generateAction.ts

@@ -0,0 +1,35 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { GenerateOptions } from './GenerateOptions';
+import { GenerationTarget } from './GenerationTarget';
+
+type GenerateActionCallback = (options: GenerateOptions) => Promise<void> | void;
+
+type Options = {
+    O: string,
+    I: string[],
+    R: string[],
+    skipEmptyFiles?: boolean,
+}
+
+export const generateAction = (target: GenerationTarget, action: GenerateActionCallback) => {
+    return (paths: string[], options: Options) => {
+        action({ 
+            target,
+            output: options.O,
+            paths,
+            includes: options.I ?? [],
+            rewrites: (options.R ?? []).map(_ => {
+                const [from, to] = _.split(':',2);
+                const [pkg, pkgName] = to.split('=', 2);
+                if (pkg === 'pkg') {
+                    return { from, to: pkgName, package: true };
+                } else {
+                    return { from, to, package: false };
+                }
+            }),
+            skipEmptyFiles: !!options.skipEmptyFiles,
+        });
+    }
+}

Generation/JavaScript/Build/index.ts

@@ -0,0 +1,39 @@
+// Copyright (c) Dolittle. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import 'process';
+import { Command, Option } from 'commander';
+
+import { repeated } from './repeated';
+import { generateAction } from './generateAction';
+import { GenerationTarget } from './GenerationTarget';
+import { Generator } from './Generator';
+
+const generator = new Generator();
+
+const program = new Command('dolittle_proto_build');
+
+const output = new Option('-O <path>', 'Output path').default('./build');
+const includes = new Option('-I <path>', 'Include path (multiple allowed)');
+const rewrite = new Option('-R <rewrite>', 'Rewrite file paths (multiple allowed)');
+const skipEmptyFiles = new Option('--skip-empty-files', 'Remove files generated without any content');
+
+program
+    .command('grpc-node <paths...>')
+    .addOption(output)
+    .addOption(repeated(includes))
+    .addOption(repeated(rewrite))
+    .addOption(skipEmptyFiles)
+    .description('Generate gRPC code for NodeJS')
+    .action(generateAction(GenerationTarget.Node, options => generator.generate(options)));
+
+program
+    .command('grpc-web <paths...>')
+    .addOption(output)
+    .addOption(repeated(includes))
+    .addOption(repeated(rewrite))
+    .addOption(skipEmptyFiles)
+    .description('Generate gRPC code for browsers')
+    .action(generateAction(GenerationTarget.Web, options => generator.generate(options)));
+
+program.parse(process.argv);