Skip to content

Commit

Permalink
(feat): cli caches api dependencies (#4201)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsinghvi
dsinghvi authored Aug 6, 2024
1 parent 80930d1 commit 4a7e107
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 123 deletions.
2 changes: 2 additions & 0 deletions .pnp.cjs
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions packages/cli/workspace-loader/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
"chalk": "^5.3.0",
"js-yaml": "^4.1.0",
"lodash-es": "^4.17.21",
"object-hash": "^3.0.0",
"tar": "^6.2.1",
"tmp-promise": "^3.0.3",
"zod": "^3.22.3"
Expand All @@ -51,6 +52,7 @@
"@types/js-yaml": "^4.0.8",
"@types/lodash-es": "^4.17.12",
"@types/node": "^18.7.18",
"@types/object-hash": "^3.0.6",
"@types/tar": "^6.1.11",
"depcheck": "^1.4.6",
"eslint": "^8.56.0",
Expand Down
163 changes: 99 additions & 64 deletions packages/cli/workspace-loader/src/loadDependency.ts
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import { dependenciesYml } from "@fern-api/configuration";
import { createFiddleService } from "@fern-api/core";
import { assertNever, noop, visitObject } from "@fern-api/core-utils";
import { AbsoluteFilePath } from "@fern-api/fs-utils";
import { AbsoluteFilePath, doesPathExist, join, RelativeFilePath } from "@fern-api/fs-utils";
import { parseVersion } from "@fern-api/semver-utils";
import { TaskContext } from "@fern-api/task-context";
import { RootApiFileSchema, YAML_SCHEMA_VERSION } from "@fern-api/yaml-schema";
import { FernFiddle } from "@fern-fern/fiddle-sdk";
import axios from "axios";
import { createWriteStream } from "fs";
import { mkdir, readFile, writeFile } from "fs/promises";
import { isEqual } from "lodash-es";
import { homedir } from "os";
import path from "path";
import { pipeline } from "stream/promises";
import tar from "tar";
Expand Down Expand Up @@ -97,6 +99,22 @@ export async function loadDependency({
}
}

const DEPENDENCIES_FOLDER_NAME = "dependencies";
const DEFINITION_FOLDER_NAME = "definition";
const METADATA_RESPONSE_FILENAME = "metadata.json";
const LOCAL_STORAGE_FOLDER = process.env.LOCAL_STORAGE_FOLDER ?? ".fern";

function getPathToLocalStorageDependency(dependency: dependenciesYml.VersionedDependency): AbsoluteFilePath {
return join(
AbsoluteFilePath.of(homedir()),
RelativeFilePath.of(LOCAL_STORAGE_FOLDER),
RelativeFilePath.of(DEPENDENCIES_FOLDER_NAME),
RelativeFilePath.of(dependency.organization),
RelativeFilePath.of(dependency.apiName),
RelativeFilePath.of(dependency.version)
);
}

async function validateLocalDependencyAndGetDefinition({
dependency,
context,
Expand All @@ -121,7 +139,10 @@ async function validateLocalDependencyAndGetDefinition({
return undefined;
}

return await loadDependencyWorkspaceResult.workspace.getDefinition({ context }, settings);
const definition = await loadDependencyWorkspaceResult.workspace.getDefinition({ context }, settings);
context.logger.info("Loaded...");

return definition;
}

async function validateVersionedDependencyAndGetDefinition({
Expand All @@ -135,83 +156,97 @@ async function validateVersionedDependencyAndGetDefinition({
cliVersion: string;
settings?: OSSWorkspace.Settings;
}): Promise<FernDefinition | undefined> {
// load API
context.logger.info("Downloading manifest...");
const response = await FIDDLE.definitionRegistry.get(
FernFiddle.OrganizationId(dependency.organization),
FernFiddle.ApiId(dependency.apiName),
dependency.version
);
if (!response.ok) {
response.error._visit({
orgDoesNotExistError: () => {
context.failWithoutThrowing("Organization does not exist");
},
apiDoesNotExistError: () => {
context.failWithoutThrowing("API does not exist");
},
versionDoesNotExistError: () => {
context.failWithoutThrowing("Version does not exist");
},
_other: (error) => {
context.failWithoutThrowing("Failed to download API manifest", error);
}
});
return undefined;
}
const pathToDependency: AbsoluteFilePath = getPathToLocalStorageDependency(dependency);
const pathToDefinition = join(pathToDependency, RelativeFilePath.of(DEPENDENCIES_FOLDER_NAME));
const pathToMetadata = join(pathToDependency, RelativeFilePath.of(METADATA_RESPONSE_FILENAME));

let metadata: FernFiddle.Api;
if (!(await doesPathExist(pathToDefinition)) || !(await doesPathExist(pathToMetadata))) {
// load API
context.logger.info("Downloading manifest...");
const response = await FIDDLE.definitionRegistry.get(
FernFiddle.OrganizationId(dependency.organization),
FernFiddle.ApiId(dependency.apiName),
dependency.version
);
if (!response.ok) {
response.error._visit({
orgDoesNotExistError: () => {
context.failWithoutThrowing("Organization does not exist");
},
apiDoesNotExistError: () => {
context.failWithoutThrowing("API does not exist");
},
versionDoesNotExistError: () => {
context.failWithoutThrowing("Version does not exist");
},
_other: (error) => {
context.failWithoutThrowing("Failed to download API manifest", error);
}
});
return undefined;
}

const parsedYamlVersionOfDependency =
response.body.yamlSchemaVersion != null ? parseInt(response.body.yamlSchemaVersion) : undefined;
const parsedCliVersion = parseVersion(cliVersion);
const parsedCliVersionOfDependency = parseVersion(response.body.cliVersion);
const parsedYamlVersionOfDependency =
response.body.yamlSchemaVersion != null ? parseInt(response.body.yamlSchemaVersion) : undefined;
const parsedCliVersion = parseVersion(cliVersion);
const parsedCliVersionOfDependency = parseVersion(response.body.cliVersion);

// ensure dependency is on the same YAML_SCHEMA_VERSION
if (parsedYamlVersionOfDependency != null) {
if (parsedYamlVersionOfDependency > YAML_SCHEMA_VERSION) {
// ensure dependency is on the same YAML_SCHEMA_VERSION
if (parsedYamlVersionOfDependency != null) {
if (parsedYamlVersionOfDependency > YAML_SCHEMA_VERSION) {
context.failWithoutThrowing(
`${dependency.organization}/${dependency.apiName}@${dependency.version} on a higher version of fern. Upgrade this workspace to ${response.body.cliVersion}`
);
return undefined;
} else if (parsedYamlVersionOfDependency < YAML_SCHEMA_VERSION) {
context.failWithoutThrowing(
`${dependency.organization}/${dependency.apiName}@${dependency.version} on a lower version of fern. Upgrade it to ${cliVersion}`
);
return undefined;
}
}
// otherwise, ensure CLI versions are on the same major + minor versions
else if (
parsedCliVersion.major !== parsedCliVersionOfDependency.major ||
parsedCliVersion.minor !== parsedCliVersionOfDependency.minor
) {
context.failWithoutThrowing(
`${dependency.organization}/${dependency.apiName}@${dependency.version} on a higher version of fern. Upgrade this workspace to ${response.body.cliVersion}`
`CLI version is ${response.body.cliVersion}. Expected ${parsedCliVersion.major}.${parsedCliVersion.minor}.x (to match current workspace).`
);
return undefined;
} else if (parsedYamlVersionOfDependency < YAML_SCHEMA_VERSION) {
context.failWithoutThrowing(
`${dependency.organization}/${dependency.apiName}@${dependency.version} on a lower version of fern. Upgrade it to ${cliVersion}`
);
}

// download API
context.logger.info("Downloading...");
context.logger.debug("Remote URL: " + response.body.definitionS3DownloadUrl);

await mkdir(pathToDefinition, { recursive: true });

try {
await downloadDependency({
s3PreSignedReadUrl: response.body.definitionS3DownloadUrl,
absolutePathToLocalOutput: pathToDefinition
});
} catch (error) {
context.failWithoutThrowing("Failed to download API", error);
return undefined;
}
}
// otherwise, ensure CLI versions are on the same major + minor versions
else if (
parsedCliVersion.major !== parsedCliVersionOfDependency.major ||
parsedCliVersion.minor !== parsedCliVersionOfDependency.minor
) {
context.failWithoutThrowing(
`CLI version is ${response.body.cliVersion}. Expected ${parsedCliVersion.major}.${parsedCliVersion.minor}.x (to match current workspace).`
);
return undefined;
}

// download API
context.logger.info("Downloading...");
const pathToDependency = AbsoluteFilePath.of((await tmp.dir()).path);
context.logger.debug("Remote URL: " + response.body.definitionS3DownloadUrl);
try {
await downloadDependency({
s3PreSignedReadUrl: response.body.definitionS3DownloadUrl,
absolutePathToLocalOutput: pathToDependency
});
} catch (error) {
context.failWithoutThrowing("Failed to download API", error);
return undefined;
metadata = response.body;
await writeFile(pathToMetadata, JSON.stringify(metadata));
} else {
metadata = JSON.parse((await readFile(pathToMetadata)).toString());
}

// parse workspace
context.logger.info("Parsing...");
const loadDependencyWorkspaceResult = await loadAPIWorkspace({
absolutePathToWorkspace: pathToDependency,
absolutePathToWorkspace: pathToDefinition,
context,
cliVersion: response.body.cliVersion,
cliVersion: metadata.cliVersion,
workspaceName: undefined
});

if (!loadDependencyWorkspaceResult.didSucceed) {
context.failWithoutThrowing(
"Failed to parse dependency after downloading",
Expand Down
118 changes: 59 additions & 59 deletions packages/cli/workspace-loader/src/workspaces/FernWorkspace.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { DEFINITION_DIRECTORY, dependenciesYml, generatorsYml } from "@fern-api/configuration";
import { AbsoluteFilePath, join, RelativeFilePath } from "@fern-api/fs-utils";
import { TaskContext } from "@fern-api/task-context";
import hash from "object-hash";
import { handleFailedWorkspaceParserResultRaw } from "../handleFailedWorkspaceParserResult";
import { listFernFiles } from "../listFernFiles";
import { parseYamlFiles } from "../parseYamlFiles";
Expand Down Expand Up @@ -75,7 +76,7 @@ export class LazyFernWorkspace extends AbstractAPIWorkspace<OSSWorkspace.Setting

private context: TaskContext;
private cliVersion: string;
private downloaded = false;
private fernWorkspaces: Record<string, FernWorkspace> = {};

constructor({
absoluteFilepath,
Expand Down Expand Up @@ -105,65 +106,64 @@ export class LazyFernWorkspace extends AbstractAPIWorkspace<OSSWorkspace.Setting
{ context }: { context?: TaskContext },
settings?: OSSWorkspace.Settings
): Promise<FernWorkspace> {
if (this.downloaded) {
context?.logger.disable();
const key = hash(settings ?? {});
let workspace = this.fernWorkspaces[key];

if (workspace == null) {
const defaultedContext = context || this.context;
const absolutePathToDefinition = join(this.absoluteFilepath, RelativeFilePath.of(DEFINITION_DIRECTORY));
const dependenciesConfiguration = await dependenciesYml.loadDependenciesConfiguration({
absolutePathToWorkspace: this.absoluteFilepath,
context: defaultedContext
});

const yamlFiles = await listFernFiles(absolutePathToDefinition, "{yml,yaml}");

const parseResult = await parseYamlFiles(yamlFiles);
if (!parseResult.didSucceed) {
handleFailedWorkspaceParserResultRaw(parseResult.failures, defaultedContext.logger);
return defaultedContext.failAndThrow();
}

const structuralValidationResult = validateStructureOfYamlFiles({
files: parseResult.files,
absolutePathToDefinition
});
if (!structuralValidationResult.didSucceed) {
handleFailedWorkspaceParserResultRaw(structuralValidationResult.failures, defaultedContext.logger);
return defaultedContext.failAndThrow();
}

const processPackageMarkersResult = await processPackageMarkers({
dependenciesConfiguration,
structuralValidationResult,
context: defaultedContext,
cliVersion: this.cliVersion,
settings
});
if (!processPackageMarkersResult.didSucceed) {
handleFailedWorkspaceParserResultRaw(processPackageMarkersResult.failures, defaultedContext.logger);
return defaultedContext.failAndThrow();
}

workspace = new FernWorkspace({
absoluteFilepath: this.absoluteFilepath,
generatorsConfiguration: this.generatorsConfiguration,
dependenciesConfiguration,
workspaceName: this.workspaceName,
definition: {
absoluteFilepath: absolutePathToDefinition,
rootApiFile: structuralValidationResult.rootApiFile,
namedDefinitionFiles: structuralValidationResult.namedDefinitionFiles,
packageMarkers: processPackageMarkersResult.packageMarkers,
importedDefinitions: processPackageMarkersResult.importedDefinitions
},
changelog: this.changelog
});

this.fernWorkspaces[key] = workspace;
}

const defaultedContext = context || this.context;
const absolutePathToDefinition = join(this.absoluteFilepath, RelativeFilePath.of(DEFINITION_DIRECTORY));
const dependenciesConfiguration = await dependenciesYml.loadDependenciesConfiguration({
absolutePathToWorkspace: this.absoluteFilepath,
context: defaultedContext
});

const yamlFiles = await listFernFiles(absolutePathToDefinition, "{yml,yaml}");

const parseResult = await parseYamlFiles(yamlFiles);
if (!parseResult.didSucceed) {
handleFailedWorkspaceParserResultRaw(parseResult.failures, defaultedContext.logger);
return defaultedContext.failAndThrow();
}

const structuralValidationResult = validateStructureOfYamlFiles({
files: parseResult.files,
absolutePathToDefinition
});
if (!structuralValidationResult.didSucceed) {
handleFailedWorkspaceParserResultRaw(structuralValidationResult.failures, defaultedContext.logger);
return defaultedContext.failAndThrow();
}

const processPackageMarkersResult = await processPackageMarkers({
dependenciesConfiguration,
structuralValidationResult,
context: defaultedContext,
cliVersion: this.cliVersion,
settings
});
if (!processPackageMarkersResult.didSucceed) {
handleFailedWorkspaceParserResultRaw(processPackageMarkersResult.failures, defaultedContext.logger);
return defaultedContext.failAndThrow();
}

if (!this.downloaded) {
this.downloaded = true;
} else {
context?.logger.enable();
}

return new FernWorkspace({
absoluteFilepath: this.absoluteFilepath,
generatorsConfiguration: this.generatorsConfiguration,
dependenciesConfiguration,
workspaceName: this.workspaceName,
definition: {
absoluteFilepath: absolutePathToDefinition,
rootApiFile: structuralValidationResult.rootApiFile,
namedDefinitionFiles: structuralValidationResult.namedDefinitionFiles,
packageMarkers: processPackageMarkersResult.packageMarkers,
importedDefinitions: processPackageMarkersResult.importedDefinitions
},
changelog: this.changelog
});
return workspace;
}
}
Loading

0 comments on commit 4a7e107

Please sign in to comment.