From 91399fe232aae777968ff73de390503f95b87163 Mon Sep 17 00:00:00 2001 From: Alex McKinney Date: Mon, 12 Aug 2024 17:41:57 -0400 Subject: [PATCH] (feature, cli): Upload source files to S3 (#4286) --- packages/cli/docs-preview/package.json | 2 +- packages/cli/docs-resolver/package.json | 2 +- .../remote-workspace-runner/package.json | 1 + .../src/SourceUploader.ts | 124 ++++++++++++++++++ .../src/runRemoteGenerationForGenerator.ts | 38 +++++- packages/cli/register/package.json | 2 +- pnpm-lock.yaml | 34 ++--- 7 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts diff --git a/packages/cli/docs-preview/package.json b/packages/cli/docs-preview/package.json index f4bb7c56310..4f93f479e7f 100644 --- a/packages/cli/docs-preview/package.json +++ b/packages/cli/docs-preview/package.json @@ -28,7 +28,7 @@ }, "dependencies": { "@fern-api/docs-resolver": "workspace:*", - "@fern-api/fdr-sdk": "0.98.20-33071ab6e", + "@fern-api/fdr-sdk": "0.98.20-86eba53a1", "@fern-api/fs-utils": "workspace:*", "@fern-api/ir-sdk": "workspace:*", "@fern-api/logger": "workspace:*", diff --git a/packages/cli/docs-resolver/package.json b/packages/cli/docs-resolver/package.json index f849b493986..d4f5b407706 100644 --- a/packages/cli/docs-resolver/package.json +++ b/packages/cli/docs-resolver/package.json @@ -30,7 +30,7 @@ "@fern-api/configuration": "workspace:*", "@fern-api/core-utils": "workspace:*", "@fern-api/docs-markdown-utils": "workspace:*", - "@fern-api/fdr-sdk": "0.98.20-33071ab6e", + "@fern-api/fdr-sdk": "0.98.20-86eba53a1", "@fern-api/fs-utils": "workspace:*", "@fern-api/ir-generator": "workspace:*", "@fern-api/ir-sdk": "workspace:*", diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/package.json b/packages/cli/generation/remote-generation/remote-workspace-runner/package.json index 8200c465271..f59565ee06c 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/package.json +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/package.json @@ -32,6 +32,7 @@ "@fern-api/core": "workspace:*", "@fern-api/core-utils": "workspace:*", "@fern-api/docs-resolver": "workspace:*", + "@fern-api/logging-execa": "workspace:*", "@fern-fern/fdr-cjs-sdk": "0.1.0", "@fern-api/fs-utils": "workspace:*", "@fern-api/ir-generator": "workspace:*", diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts new file mode 100644 index 00000000000..f5bf9c1f3d5 --- /dev/null +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/SourceUploader.ts @@ -0,0 +1,124 @@ +import { AbsoluteFilePath, join, RelativeFilePath } from "@fern-api/fs-utils"; +import { ApiDefinitionSource, SourceConfig } from "@fern-api/ir-sdk"; +import { loggingExeca } from "@fern-api/logging-execa"; +import { InteractiveTaskContext } from "@fern-api/task-context"; +import { IdentifiableSource } from "@fern-api/workspace-loader"; +import { FernRegistry as FdrAPI } from "@fern-fern/fdr-cjs-sdk"; +import { readFile, unlink } from "fs/promises"; +import tmp from "tmp-promise"; + +const PROTOBUF_ZIP_FILENAME = "proto.zip"; + +export type SourceType = "asyncapi" | "openapi" | "protobuf"; + +export class SourceUploader { + public sourceTypes: Set; + private context: InteractiveTaskContext; + private sources: Record; + + constructor(context: InteractiveTaskContext, sources: IdentifiableSource[]) { + this.context = context; + this.sources = Object.fromEntries(sources.map((source) => [source.id, source])); + this.sourceTypes = new Set(Object.values(this.sources).map((source) => source.type)); + } + + public async uploadSources( + sources: Record + ): Promise { + for (const [id, source] of Object.entries(sources)) { + const identifiableSource = this.getSourceOrThrow(id); + await this.uploadSource(identifiableSource, source.uploadUrl); + } + return this.convertFdrSourceUploadsToSourceConfig(sources); + } + + private async uploadSource(source: IdentifiableSource, uploadURL: string): Promise { + const uploadCommand = await this.getUploadCommand(source); + const fileData = await readFile(uploadCommand.absoluteFilePath); + const response = await fetch(uploadURL, { + method: "PUT", + body: fileData, + headers: { + "Content-Type": "application/octet-stream" + } + }); + await uploadCommand.cleanup(); + if (!response.ok) { + this.context.failAndThrow( + `Failed to upload source file: ${source.absoluteFilePath}. Status: ${response.status}, ${response.statusText}` + ); + } + } + + private async getUploadCommand( + source: IdentifiableSource + ): Promise<{ absoluteFilePath: AbsoluteFilePath; cleanup: () => Promise }> { + if (source.type === "protobuf") { + const absoluteFilePath = await this.zipSource(source.absoluteFilePath); + return { + absoluteFilePath, + cleanup: async () => { + this.context.logger.debug(`Removing ${absoluteFilePath}`); + await unlink(absoluteFilePath); + } + }; + } + return { + absoluteFilePath: source.absoluteFilePath, + cleanup: async () => { + // Do nothing. + } + }; + } + + private async zipSource(absolutePathToSource: AbsoluteFilePath): Promise { + const tmpDir = await tmp.dir(); + const destination = join(AbsoluteFilePath.of(tmpDir.path), RelativeFilePath.of(PROTOBUF_ZIP_FILENAME)); + + this.context.logger.debug(`Zipping source ${absolutePathToSource} into ${destination}`); + await loggingExeca(this.context.logger, "zip", ["-r", destination, "."], { + cwd: absolutePathToSource, + doNotPipeOutput: true + }); + + return destination; + } + + private convertFdrSourceUploadsToSourceConfig( + sources: Record + ): SourceConfig { + const apiDefinitionSources: ApiDefinitionSource[] = []; + for (const [id, sourceUpload] of Object.entries(sources)) { + const identifiableSource = this.getSourceOrThrow(id); + switch (identifiableSource.type) { + case "protobuf": + apiDefinitionSources.push( + ApiDefinitionSource.proto({ + id, + protoRootUrl: sourceUpload.downloadUrl + }) + ); + continue; + case "openapi": + apiDefinitionSources.push(ApiDefinitionSource.openapi()); + continue; + case "asyncapi": + // AsyncAPI sources aren't modeled in the IR yet. + continue; + } + } + return { + sources: apiDefinitionSources + }; + } + + private getSourceOrThrow(id: string): IdentifiableSource { + const source = this.sources[id]; + if (source == null) { + this.context.failAndThrow( + `Internal error; server responded with source id "${id}" which does not exist in the workspace.` + ); + } + return source; + } +} diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts index 15cd9ab70fa..34de2691d73 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForGenerator.ts @@ -5,12 +5,13 @@ import { AbsoluteFilePath } from "@fern-api/fs-utils"; import { generateIntermediateRepresentation } from "@fern-api/ir-generator"; import { convertIrToFdrApi } from "@fern-api/register"; import { InteractiveTaskContext } from "@fern-api/task-context"; -import { FernWorkspace } from "@fern-api/workspace-loader"; +import { FernWorkspace, IdentifiableSource } from "@fern-api/workspace-loader"; import { FernRegistry as FdrAPI, FernRegistryClient as FdrClient } from "@fern-fern/fdr-cjs-sdk"; import { FernFiddle } from "@fern-fern/fiddle-sdk"; import { createAndStartJob } from "./createAndStartJob"; import { pollJobAndReportStatus } from "./pollJobAndReportStatus"; import { RemoteTaskHandler } from "./RemoteTaskHandler"; +import { SourceUploader } from "./SourceUploader"; export async function runRemoteGenerationForGenerator({ projectConfig, @@ -57,15 +58,35 @@ export async function runRemoteGenerationForGenerator({ version: version ?? (await computeSemanticVersion({ fdr, packageName, generatorInvocation })) }); + const sources = workspace.getSources(); const apiDefinition = convertIrToFdrApi({ ir, snippetsConfig: {} }); const response = await fdr.api.v1.register.registerApiDefinition({ orgId: organization, apiId: ir.apiName.originalName, - definition: apiDefinition + definition: apiDefinition, + sources: sources.length > 0 ? convertToFdrApiDefinitionSources(sources) : undefined }); + let fdrApiDefinitionId; + let sourceUploads; if (response.ok) { fdrApiDefinitionId = response.body.apiDefinitionId; + sourceUploads = response.body.sources; + } + + const sourceUploader = new SourceUploader(interactiveTaskContext, sources); + if (sourceUploads == null && sourceUploader.sourceTypes.has("protobuf")) { + // We only fail hard if we need to upload Protobuf source files. Unlike OpenAPI, these + // files are required for successful code generation. + interactiveTaskContext.failAndThrow("Did not successfully upload Protobuf source files."); + } + + if (sourceUploads != null) { + interactiveTaskContext.logger.debug("Uploading source files ..."); + const sourceConfig = await sourceUploader.uploadSources(sourceUploads); + + interactiveTaskContext.logger.debug("Setting IR source configuration ..."); + ir.sourceConfig = sourceConfig; } const job = await createAndStartJob({ @@ -161,3 +182,16 @@ async function computeSemanticVersion({ } return response.body.version; } + +function convertToFdrApiDefinitionSources( + sources: IdentifiableSource[] +): Record { + return Object.fromEntries( + Object.values(sources).map((source) => [ + source.id, + { + type: source.type === "protobuf" ? "proto" : source.type + } + ]) + ); +} diff --git a/packages/cli/register/package.json b/packages/cli/register/package.json index efc9983951e..89c72f16d00 100644 --- a/packages/cli/register/package.json +++ b/packages/cli/register/package.json @@ -31,7 +31,7 @@ "@fern-api/configuration": "workspace:*", "@fern-api/core": "workspace:*", "@fern-api/core-utils": "workspace:*", - "@fern-api/fdr-sdk": "0.98.20-33071ab6e", + "@fern-api/fdr-sdk": "0.98.20-86eba53a1", "@fern-api/ir-generator": "workspace:*", "@fern-api/ir-sdk": "workspace:*", "@fern-api/task-context": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e5fab0ab06a..f6cb1ec494c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3177,19 +3177,6 @@ importers: specifier: 4.6.4 version: 4.6.4 - packages/cli/cli/dist/dev: {} - - packages/cli/cli/dist/local: - devDependencies: - globals: - specifier: link:@types/vitest/globals - version: link:@types/vitest/globals - vitest: - specifier: ^2.0.5 - version: 2.0.5(@types/node@18.7.18)(jsdom@20.0.3)(sass@1.72.0)(terser@5.31.5) - - packages/cli/cli/dist/prod: {} - packages/cli/configuration: dependencies: '@fern-api/core-utils': @@ -3327,8 +3314,8 @@ importers: specifier: workspace:* version: link:../docs-resolver '@fern-api/fdr-sdk': - specifier: 0.98.20-33071ab6e - version: 0.98.20-33071ab6e(typescript@4.6.4) + specifier: 0.98.20-86eba53a1 + version: 0.98.20-86eba53a1(typescript@4.6.4) '@fern-api/fs-utils': specifier: workspace:* version: link:../../commons/fs-utils @@ -3427,8 +3414,8 @@ importers: specifier: workspace:* version: link:../docs-markdown-utils '@fern-api/fdr-sdk': - specifier: 0.98.20-33071ab6e - version: 0.98.20-33071ab6e(typescript@4.6.4) + specifier: 0.98.20-86eba53a1 + version: 0.98.20-86eba53a1(typescript@4.6.4) '@fern-api/fs-utils': specifier: workspace:* version: link:../../commons/fs-utils @@ -4017,6 +4004,9 @@ importers: '@fern-api/logger': specifier: workspace:* version: link:../../../logger + '@fern-api/logging-execa': + specifier: workspace:* + version: link:../../../../commons/logging-execa '@fern-api/register': specifier: workspace:* version: link:../../../register @@ -4633,8 +4623,8 @@ importers: specifier: workspace:* version: link:../../commons/core-utils '@fern-api/fdr-sdk': - specifier: 0.98.20-33071ab6e - version: 0.98.20-33071ab6e(typescript@4.6.4) + specifier: 0.98.20-86eba53a1 + version: 0.98.20-86eba53a1(typescript@4.6.4) '@fern-api/ir-generator': specifier: workspace:* version: link:../generation/ir-generator @@ -6689,8 +6679,8 @@ packages: '@exodus/schemasafe@1.0.0': resolution: {integrity: sha512-2cyupPIZI69HQxEAPllLXBjQp4njDKkOjYRCYxvMZe3/LY9pp9fBM3Tb1wiFAdP6Emo4v3OEbCLGj6u73Q5KLw==} - '@fern-api/fdr-sdk@0.98.20-33071ab6e': - resolution: {integrity: sha512-dvyh7sDjWw1Zs8LRlLVxm2SKChH5ZY0wbybe4c2+I64i7Voy3/YQ9J03nx9IrMtsrbifiTDDtj6IxSJycsUeVA==} + '@fern-api/fdr-sdk@0.98.20-86eba53a1': + resolution: {integrity: sha512-gQsTxrmG69hE/W68e3RpS0Jiq+hD1OPwwEgPi7EjFHcUy0Ru9WlSDOGJ0Uv1WsS84AHAz84OrUs+KEQwUwDeyA==} '@fern-api/venus-api-sdk@0.0.38': resolution: {integrity: sha512-1JjuctZwyPu4jN51bBqjIy+uCBPGa/ROcDfqN0UnGeVEW7NyztkGh/qOFwnN6K07VMMql1AIF4Zov+MVqTPwmw==} @@ -13712,7 +13702,7 @@ snapshots: '@exodus/schemasafe@1.0.0': {} - '@fern-api/fdr-sdk@0.98.20-33071ab6e(typescript@4.6.4)': + '@fern-api/fdr-sdk@0.98.20-86eba53a1(typescript@4.6.4)': dependencies: dayjs: 1.11.11 fast-deep-equal: 3.1.3