diff --git a/.pnp.cjs b/.pnp.cjs index 196ae4bc9c7..e6b5fe355a5 100644 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -7812,6 +7812,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@types/js-yaml", "npm:4.0.8"],\ ["@types/lodash-es", "npm:4.17.12"],\ ["@types/node", "npm:18.7.18"],\ + ["@types/object-hash", "npm:3.0.6"],\ ["@types/tar", "npm:6.1.11"],\ ["axios", "npm:0.28.0"],\ ["chalk", "npm:5.3.0"],\ @@ -7820,6 +7821,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["jest", "virtual:816fb67d993b0978271f762d4ccbec7209ef2546c234ca6e241662d44336c8e32c1c3c07189cfe14b67974a4840e1ed140408a7403bf9deb68c1953445072efe#npm:29.7.0"],\ ["js-yaml", "npm:4.1.0"],\ ["lodash-es", "npm:4.17.21"],\ + ["object-hash", "npm:3.0.0"],\ ["organize-imports-cli", "npm:0.10.0"],\ ["prettier", "npm:2.7.1"],\ ["tar", "npm:6.2.1"],\ diff --git a/packages/cli/workspace-loader/package.json b/packages/cli/workspace-loader/package.json index 92fb3e9e2a4..892f33f49dd 100644 --- a/packages/cli/workspace-loader/package.json +++ b/packages/cli/workspace-loader/package.json @@ -42,6 +42,7 @@ "chalk": "^5.3.0", "js-yaml": "^4.1.0", "lodash-es": "^4.17.21", + "object-hash": "^3.0.0", "tar": "^6.2.1", "tmp-promise": "^3.0.3", "zod": "^3.22.3" @@ -51,6 +52,7 @@ "@types/js-yaml": "^4.0.8", "@types/lodash-es": "^4.17.12", "@types/node": "^18.7.18", + "@types/object-hash": "^3.0.6", "@types/tar": "^6.1.11", "depcheck": "^1.4.6", "eslint": "^8.56.0", diff --git a/packages/cli/workspace-loader/src/loadDependency.ts b/packages/cli/workspace-loader/src/loadDependency.ts index 4f8680b2a88..af63b3d6e07 100644 --- a/packages/cli/workspace-loader/src/loadDependency.ts +++ b/packages/cli/workspace-loader/src/loadDependency.ts @@ -1,14 +1,16 @@ import { dependenciesYml } from "@fern-api/configuration"; import { createFiddleService } from "@fern-api/core"; import { assertNever, noop, visitObject } from "@fern-api/core-utils"; -import { AbsoluteFilePath } from "@fern-api/fs-utils"; +import { AbsoluteFilePath, doesPathExist, join, RelativeFilePath } from "@fern-api/fs-utils"; import { parseVersion } from "@fern-api/semver-utils"; import { TaskContext } from "@fern-api/task-context"; import { RootApiFileSchema, YAML_SCHEMA_VERSION } from "@fern-api/yaml-schema"; import { FernFiddle } from "@fern-fern/fiddle-sdk"; import axios from "axios"; import { createWriteStream } from "fs"; +import { mkdir, readFile, writeFile } from "fs/promises"; import { isEqual } from "lodash-es"; +import { homedir } from "os"; import path from "path"; import { pipeline } from "stream/promises"; import tar from "tar"; @@ -97,6 +99,22 @@ export async function loadDependency({ } } +const DEPENDENCIES_FOLDER_NAME = "dependencies"; +const DEFINITION_FOLDER_NAME = "definition"; +const METADATA_RESPONSE_FILENAME = "metadata.json"; +const LOCAL_STORAGE_FOLDER = process.env.LOCAL_STORAGE_FOLDER ?? ".fern"; + +function getPathToLocalStorageDependency(dependency: dependenciesYml.VersionedDependency): AbsoluteFilePath { + return join( + AbsoluteFilePath.of(homedir()), + RelativeFilePath.of(LOCAL_STORAGE_FOLDER), + RelativeFilePath.of(DEPENDENCIES_FOLDER_NAME), + RelativeFilePath.of(dependency.organization), + RelativeFilePath.of(dependency.apiName), + RelativeFilePath.of(dependency.version) + ); +} + async function validateLocalDependencyAndGetDefinition({ dependency, context, @@ -121,7 +139,10 @@ async function validateLocalDependencyAndGetDefinition({ return undefined; } - return await loadDependencyWorkspaceResult.workspace.getDefinition({ context }, settings); + const definition = await loadDependencyWorkspaceResult.workspace.getDefinition({ context }, settings); + context.logger.info("Loaded..."); + + return definition; } async function validateVersionedDependencyAndGetDefinition({ @@ -135,83 +156,97 @@ async function validateVersionedDependencyAndGetDefinition({ cliVersion: string; settings?: OSSWorkspace.Settings; }): Promise { - // load API - context.logger.info("Downloading manifest..."); - const response = await FIDDLE.definitionRegistry.get( - FernFiddle.OrganizationId(dependency.organization), - FernFiddle.ApiId(dependency.apiName), - dependency.version - ); - if (!response.ok) { - response.error._visit({ - orgDoesNotExistError: () => { - context.failWithoutThrowing("Organization does not exist"); - }, - apiDoesNotExistError: () => { - context.failWithoutThrowing("API does not exist"); - }, - versionDoesNotExistError: () => { - context.failWithoutThrowing("Version does not exist"); - }, - _other: (error) => { - context.failWithoutThrowing("Failed to download API manifest", error); - } - }); - return undefined; - } + const pathToDependency: AbsoluteFilePath = getPathToLocalStorageDependency(dependency); + const pathToDefinition = join(pathToDependency, RelativeFilePath.of(DEPENDENCIES_FOLDER_NAME)); + const pathToMetadata = join(pathToDependency, RelativeFilePath.of(METADATA_RESPONSE_FILENAME)); + + let metadata: FernFiddle.Api; + if (!(await doesPathExist(pathToDefinition)) || !(await doesPathExist(pathToMetadata))) { + // load API + context.logger.info("Downloading manifest..."); + const response = await FIDDLE.definitionRegistry.get( + FernFiddle.OrganizationId(dependency.organization), + FernFiddle.ApiId(dependency.apiName), + dependency.version + ); + if (!response.ok) { + response.error._visit({ + orgDoesNotExistError: () => { + context.failWithoutThrowing("Organization does not exist"); + }, + apiDoesNotExistError: () => { + context.failWithoutThrowing("API does not exist"); + }, + versionDoesNotExistError: () => { + context.failWithoutThrowing("Version does not exist"); + }, + _other: (error) => { + context.failWithoutThrowing("Failed to download API manifest", error); + } + }); + return undefined; + } - const parsedYamlVersionOfDependency = - response.body.yamlSchemaVersion != null ? parseInt(response.body.yamlSchemaVersion) : undefined; - const parsedCliVersion = parseVersion(cliVersion); - const parsedCliVersionOfDependency = parseVersion(response.body.cliVersion); + const parsedYamlVersionOfDependency = + response.body.yamlSchemaVersion != null ? parseInt(response.body.yamlSchemaVersion) : undefined; + const parsedCliVersion = parseVersion(cliVersion); + const parsedCliVersionOfDependency = parseVersion(response.body.cliVersion); - // ensure dependency is on the same YAML_SCHEMA_VERSION - if (parsedYamlVersionOfDependency != null) { - if (parsedYamlVersionOfDependency > YAML_SCHEMA_VERSION) { + // ensure dependency is on the same YAML_SCHEMA_VERSION + if (parsedYamlVersionOfDependency != null) { + if (parsedYamlVersionOfDependency > YAML_SCHEMA_VERSION) { + context.failWithoutThrowing( + `${dependency.organization}/${dependency.apiName}@${dependency.version} on a higher version of fern. Upgrade this workspace to ${response.body.cliVersion}` + ); + return undefined; + } else if (parsedYamlVersionOfDependency < YAML_SCHEMA_VERSION) { + context.failWithoutThrowing( + `${dependency.organization}/${dependency.apiName}@${dependency.version} on a lower version of fern. Upgrade it to ${cliVersion}` + ); + return undefined; + } + } + // otherwise, ensure CLI versions are on the same major + minor versions + else if ( + parsedCliVersion.major !== parsedCliVersionOfDependency.major || + parsedCliVersion.minor !== parsedCliVersionOfDependency.minor + ) { context.failWithoutThrowing( - `${dependency.organization}/${dependency.apiName}@${dependency.version} on a higher version of fern. Upgrade this workspace to ${response.body.cliVersion}` + `CLI version is ${response.body.cliVersion}. Expected ${parsedCliVersion.major}.${parsedCliVersion.minor}.x (to match current workspace).` ); return undefined; - } else if (parsedYamlVersionOfDependency < YAML_SCHEMA_VERSION) { - context.failWithoutThrowing( - `${dependency.organization}/${dependency.apiName}@${dependency.version} on a lower version of fern. Upgrade it to ${cliVersion}` - ); + } + + // download API + context.logger.info("Downloading..."); + context.logger.debug("Remote URL: " + response.body.definitionS3DownloadUrl); + + await mkdir(pathToDefinition, { recursive: true }); + + try { + await downloadDependency({ + s3PreSignedReadUrl: response.body.definitionS3DownloadUrl, + absolutePathToLocalOutput: pathToDefinition + }); + } catch (error) { + context.failWithoutThrowing("Failed to download API", error); return undefined; } - } - // otherwise, ensure CLI versions are on the same major + minor versions - else if ( - parsedCliVersion.major !== parsedCliVersionOfDependency.major || - parsedCliVersion.minor !== parsedCliVersionOfDependency.minor - ) { - context.failWithoutThrowing( - `CLI version is ${response.body.cliVersion}. Expected ${parsedCliVersion.major}.${parsedCliVersion.minor}.x (to match current workspace).` - ); - return undefined; - } - // download API - context.logger.info("Downloading..."); - const pathToDependency = AbsoluteFilePath.of((await tmp.dir()).path); - context.logger.debug("Remote URL: " + response.body.definitionS3DownloadUrl); - try { - await downloadDependency({ - s3PreSignedReadUrl: response.body.definitionS3DownloadUrl, - absolutePathToLocalOutput: pathToDependency - }); - } catch (error) { - context.failWithoutThrowing("Failed to download API", error); - return undefined; + metadata = response.body; + await writeFile(pathToMetadata, JSON.stringify(metadata)); + } else { + metadata = JSON.parse((await readFile(pathToMetadata)).toString()); } - // parse workspace context.logger.info("Parsing..."); const loadDependencyWorkspaceResult = await loadAPIWorkspace({ - absolutePathToWorkspace: pathToDependency, + absolutePathToWorkspace: pathToDefinition, context, - cliVersion: response.body.cliVersion, + cliVersion: metadata.cliVersion, workspaceName: undefined }); + if (!loadDependencyWorkspaceResult.didSucceed) { context.failWithoutThrowing( "Failed to parse dependency after downloading", diff --git a/packages/cli/workspace-loader/src/workspaces/FernWorkspace.ts b/packages/cli/workspace-loader/src/workspaces/FernWorkspace.ts index 276d9c5ffc0..182c25379ca 100644 --- a/packages/cli/workspace-loader/src/workspaces/FernWorkspace.ts +++ b/packages/cli/workspace-loader/src/workspaces/FernWorkspace.ts @@ -1,6 +1,7 @@ import { DEFINITION_DIRECTORY, dependenciesYml, generatorsYml } from "@fern-api/configuration"; import { AbsoluteFilePath, join, RelativeFilePath } from "@fern-api/fs-utils"; import { TaskContext } from "@fern-api/task-context"; +import hash from "object-hash"; import { handleFailedWorkspaceParserResultRaw } from "../handleFailedWorkspaceParserResult"; import { listFernFiles } from "../listFernFiles"; import { parseYamlFiles } from "../parseYamlFiles"; @@ -75,7 +76,7 @@ export class LazyFernWorkspace extends AbstractAPIWorkspace = {}; constructor({ absoluteFilepath, @@ -105,65 +106,64 @@ export class LazyFernWorkspace extends AbstractAPIWorkspace { - if (this.downloaded) { - context?.logger.disable(); + const key = hash(settings ?? {}); + let workspace = this.fernWorkspaces[key]; + + if (workspace == null) { + const defaultedContext = context || this.context; + const absolutePathToDefinition = join(this.absoluteFilepath, RelativeFilePath.of(DEFINITION_DIRECTORY)); + const dependenciesConfiguration = await dependenciesYml.loadDependenciesConfiguration({ + absolutePathToWorkspace: this.absoluteFilepath, + context: defaultedContext + }); + + const yamlFiles = await listFernFiles(absolutePathToDefinition, "{yml,yaml}"); + + const parseResult = await parseYamlFiles(yamlFiles); + if (!parseResult.didSucceed) { + handleFailedWorkspaceParserResultRaw(parseResult.failures, defaultedContext.logger); + return defaultedContext.failAndThrow(); + } + + const structuralValidationResult = validateStructureOfYamlFiles({ + files: parseResult.files, + absolutePathToDefinition + }); + if (!structuralValidationResult.didSucceed) { + handleFailedWorkspaceParserResultRaw(structuralValidationResult.failures, defaultedContext.logger); + return defaultedContext.failAndThrow(); + } + + const processPackageMarkersResult = await processPackageMarkers({ + dependenciesConfiguration, + structuralValidationResult, + context: defaultedContext, + cliVersion: this.cliVersion, + settings + }); + if (!processPackageMarkersResult.didSucceed) { + handleFailedWorkspaceParserResultRaw(processPackageMarkersResult.failures, defaultedContext.logger); + return defaultedContext.failAndThrow(); + } + + workspace = new FernWorkspace({ + absoluteFilepath: this.absoluteFilepath, + generatorsConfiguration: this.generatorsConfiguration, + dependenciesConfiguration, + workspaceName: this.workspaceName, + definition: { + absoluteFilepath: absolutePathToDefinition, + rootApiFile: structuralValidationResult.rootApiFile, + namedDefinitionFiles: structuralValidationResult.namedDefinitionFiles, + packageMarkers: processPackageMarkersResult.packageMarkers, + importedDefinitions: processPackageMarkersResult.importedDefinitions + }, + changelog: this.changelog + }); + + this.fernWorkspaces[key] = workspace; } - const defaultedContext = context || this.context; - const absolutePathToDefinition = join(this.absoluteFilepath, RelativeFilePath.of(DEFINITION_DIRECTORY)); - const dependenciesConfiguration = await dependenciesYml.loadDependenciesConfiguration({ - absolutePathToWorkspace: this.absoluteFilepath, - context: defaultedContext - }); - - const yamlFiles = await listFernFiles(absolutePathToDefinition, "{yml,yaml}"); - - const parseResult = await parseYamlFiles(yamlFiles); - if (!parseResult.didSucceed) { - handleFailedWorkspaceParserResultRaw(parseResult.failures, defaultedContext.logger); - return defaultedContext.failAndThrow(); - } - - const structuralValidationResult = validateStructureOfYamlFiles({ - files: parseResult.files, - absolutePathToDefinition - }); - if (!structuralValidationResult.didSucceed) { - handleFailedWorkspaceParserResultRaw(structuralValidationResult.failures, defaultedContext.logger); - return defaultedContext.failAndThrow(); - } - - const processPackageMarkersResult = await processPackageMarkers({ - dependenciesConfiguration, - structuralValidationResult, - context: defaultedContext, - cliVersion: this.cliVersion, - settings - }); - if (!processPackageMarkersResult.didSucceed) { - handleFailedWorkspaceParserResultRaw(processPackageMarkersResult.failures, defaultedContext.logger); - return defaultedContext.failAndThrow(); - } - - if (!this.downloaded) { - this.downloaded = true; - } else { - context?.logger.enable(); - } - - return new FernWorkspace({ - absoluteFilepath: this.absoluteFilepath, - generatorsConfiguration: this.generatorsConfiguration, - dependenciesConfiguration, - workspaceName: this.workspaceName, - definition: { - absoluteFilepath: absolutePathToDefinition, - rootApiFile: structuralValidationResult.rootApiFile, - namedDefinitionFiles: structuralValidationResult.namedDefinitionFiles, - packageMarkers: processPackageMarkersResult.packageMarkers, - importedDefinitions: processPackageMarkersResult.importedDefinitions - }, - changelog: this.changelog - }); + return workspace; } } diff --git a/yarn.lock b/yarn.lock index 9dd5d6e5512..d863599f315 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4684,6 +4684,7 @@ __metadata: "@types/js-yaml": ^4.0.8 "@types/lodash-es": ^4.17.12 "@types/node": ^18.7.18 + "@types/object-hash": ^3.0.6 "@types/tar": ^6.1.11 axios: ^0.28.0 chalk: ^5.3.0 @@ -4692,6 +4693,7 @@ __metadata: jest: ^29.7.0 js-yaml: ^4.1.0 lodash-es: ^4.17.21 + object-hash: ^3.0.0 organize-imports-cli: ^0.10.0 prettier: ^2.7.1 tar: ^6.2.1