(beta): introduce new api configuration in generators.yml (#3121)

* (beta): introduce new api configuration in generators.yml * (chore): fix tests * (feature): support asyncapi overrides * (fix): fix depcheck * (fix): ete and unit tests work * (fix): try again
fern-api · Mar 7, 2024 · f4eddd4 · f4eddd4
1 parent 3ffef03
commit f4eddd4
Show file tree

Hide file tree

Showing 40 changed files with 496 additions and 515 deletions.
diff --git a/.pnp.cjs b/.pnp.cjs
diff --git a/packages/cli/cli/package.json b/packages/cli/cli/package.json
@@ -57,6 +57,7 @@
     "@fern-api/mock": "workspace:*",
     "@fern-api/openapi-ir-sdk": "workspace:*",
     "@fern-api/openapi-ir-to-fern": "workspace:*",
+    "@fern-api/openapi-parser": "workspace:*",
     "@fern-api/posthog-manager": "workspace:*",
     "@fern-api/project-configuration": "workspace:*",
     "@fern-api/project-loader": "workspace:*",

diff --git a/packages/cli/cli/src/cli.ts b/packages/cli/cli/src/cli.ts
@@ -730,11 +730,6 @@ function addWriteOverridesCommand(cli: Argv<GlobalCliOptions>, cliContext: CliCo
                 description:
                     "When generating the initial overrides, also stub the models (in addition to the endpoints)",
                 default: false
-            }),
-            yargs.option("existing-overrides", {
-                string: true,
-                description:
-                    "The existing overrides file to add on to instead of writing a new one, we will default to the one specified in generators.yml."
             })
         ],
         async (argv) => {
@@ -747,7 +742,6 @@ function addWriteOverridesCommand(cli: Argv<GlobalCliOptions>, cliContext: CliCo
                     defaultToAllApiWorkspaces: true
                 }),
                 includeModels: !(argv.excludeModels as boolean),
-                overridesFilepath: argv.existingOverrides as string,
                 cliContext
             });
         }

diff --git a/packages/cli/cli/src/commands/format/formatWorkspaces.ts b/packages/cli/cli/src/commands/format/formatWorkspaces.ts
@@ -13,7 +13,7 @@ export async function formatWorkspaces({
 }): Promise<void> {
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
-            if (workspace.type === "openapi") {
+            if (workspace.type === "oss") {
                 return;
             }
             await cliContext.runTaskForWorkspace(workspace, async (context) => {

diff --git a/packages/cli/cli/src/commands/generate-fdr/generateFdrApiDefinitionForWorkspaces.ts b/packages/cli/cli/src/commands/generate-fdr/generateFdrApiDefinitionForWorkspaces.ts
@@ -23,7 +23,7 @@ export async function generateFdrApiDefinitionForWorkspaces({
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
                 const fernWorkspace =
-                    workspace.type === "openapi"
+                    workspace.type === "oss"
                         ? await convertOpenApiWorkspaceToFernWorkspace(workspace, context)
                         : workspace;
 

diff --git a/packages/cli/cli/src/commands/generate-ir/generateIrForWorkspaces.ts b/packages/cli/cli/src/commands/generate-ir/generateIrForWorkspaces.ts
@@ -2,10 +2,10 @@ import { Audiences } from "@fern-api/config-management-commons";
 import { AbsoluteFilePath, stringifyLargeObject } from "@fern-api/fs-utils";
 import { GenerationLanguage } from "@fern-api/generators-configuration";
 import { migrateIntermediateRepresentationThroughVersion } from "@fern-api/ir-migrations";
+import { serialization as IrSerialization } from "@fern-api/ir-sdk";
 import { Project } from "@fern-api/project-loader";
 import { TaskContext } from "@fern-api/task-context";
 import { convertOpenApiWorkspaceToFernWorkspace, FernWorkspace } from "@fern-api/workspace-loader";
-import { serialization as IrSerialization } from "@fern-api/ir-sdk";
 import { writeFile } from "fs/promises";
 import path from "path";
 import { CliContext } from "../../cli-context/CliContext";
@@ -32,7 +32,7 @@ export async function generateIrForWorkspaces({
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
                 const fernWorkspace =
-                    workspace.type === "openapi"
+                    workspace.type === "oss"
                         ? await convertOpenApiWorkspaceToFernWorkspace(workspace, context)
                         : workspace;
 

diff --git a/packages/cli/cli/src/commands/generate-openapi-ir/generateOpenAPIIrForWorkspaces.ts b/packages/cli/cli/src/commands/generate-openapi-ir/generateOpenAPIIrForWorkspaces.ts
@@ -1,7 +1,7 @@
 import { AbsoluteFilePath, stringifyLargeObject } from "@fern-api/fs-utils";
 import { serialization } from "@fern-api/openapi-ir-sdk";
+import { parse } from "@fern-api/openapi-parser";
 import { Project } from "@fern-api/project-loader";
-import { getOpenAPIIRFromOpenAPIWorkspace } from "@fern-api/workspace-loader";
 import { writeFile } from "fs/promises";
 import path from "path";
 import { CliContext } from "../../cli-context/CliContext";
@@ -23,7 +23,10 @@ export async function generateOpenAPIIrForWorkspaces({
                     return;
                 }
 
-                const openAPIIr = await getOpenAPIIRFromOpenAPIWorkspace(workspace, context);
+                const openAPIIr = await parse({
+                    workspace,
+                    taskContext: context
+                });
 
                 const irOutputFilePath = path.resolve(irFilepath);
                 const openApiIrJson = await serialization.OpenApiIntermediateRepresentation.jsonOrThrow(openAPIIr, {

diff --git a/packages/cli/cli/src/commands/generate-overrides/writeOverridesForWorkspaces.ts b/packages/cli/cli/src/commands/generate-overrides/writeOverridesForWorkspaces.ts
@@ -1,33 +1,30 @@
 import { dirname, join, RelativeFilePath } from "@fern-api/fs-utils";
 import { getEndpointLocation } from "@fern-api/openapi-ir-to-fern";
+import { parse } from "@fern-api/openapi-parser";
 import { Project } from "@fern-api/project-loader";
 import { TaskContext } from "@fern-api/task-context";
-import { getOpenAPIIRFromOpenAPIWorkspace, OpenAPIWorkspace } from "@fern-api/workspace-loader";
+import { OSSWorkspace } from "@fern-api/workspace-loader";
 import { readFile, writeFile } from "fs/promises";
 import yaml from "js-yaml";
 import { CliContext } from "../../cli-context/CliContext";
 
 export async function writeOverridesForWorkspaces({
     project,
     includeModels,
-    overridesFilepath,
     cliContext
 }: {
     project: Project;
     includeModels: boolean;
-    overridesFilepath: string | undefined;
     cliContext: CliContext;
 }): Promise<void> {
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                if (workspace.type === "openapi") {
+                if (workspace.type === "oss") {
                     await writeDefinitionForOpenAPIWorkspace({
                         workspace,
                         context,
-                        includeModels,
-                        overridesFilepath:
-                            overridesFilepath ?? workspace.generatorsConfiguration?.absolutePathToOpenAPIOverrides
+                        includeModels
                     });
                 } else {
                     context.logger.warn("Skipping fern workspace definition generation");
@@ -55,66 +52,70 @@ async function readExistingOverrides(overridesFilepath: string, context: TaskCon
 async function writeDefinitionForOpenAPIWorkspace({
     workspace,
     includeModels,
-    overridesFilepath,
     context
 }: {
-    workspace: OpenAPIWorkspace;
+    workspace: OSSWorkspace;
     includeModels: boolean;
-    overridesFilepath: string | undefined;
     context: TaskContext;
 }): Promise<void> {
-    const openApiIr = await getOpenAPIIRFromOpenAPIWorkspace(workspace, context);
-
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    let existingOverrides: any = {};
-    if (overridesFilepath !== undefined) {
-        existingOverrides = await readExistingOverrides(overridesFilepath, context);
-    }
-
-    const paths: Record<string, Record<string, unknown>> = "path" in existingOverrides
-        ? (existingOverrides.path as Record<string, Record<string, unknown>>)
-        : {};
-    for (const endpoint of openApiIr.endpoints) {
-        const endpointLocation = getEndpointLocation(endpoint);
-        if (!(endpoint.path in paths)) {
-            paths[endpoint.path] = {};
+    for (const spec of workspace.specs) {
+        const ir = await parse({
+            workspace: {
+                specs: [spec]
+            },
+            taskContext: context
+        });
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        let existingOverrides: any = {};
+        if (spec.absoluteFilepathToOverrides !== undefined) {
+            existingOverrides = await readExistingOverrides(spec.absoluteFilepathToOverrides, context);
         }
-        const pathItem = paths[endpoint.path];
-        if (pathItem != null && pathItem[endpoint.method] == null) {
-            const groupName = endpointLocation.file
-                .split("/")
-                .map((part) => part.replace(".yml", ""))
-                .filter((part) => part !== "__package__");
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const sdkMethodNameExtensions: Record<string, any> = {};
-            if (groupName.length > 0) {
-                sdkMethodNameExtensions["x-fern-sdk-group-name"] = groupName;
+
+        const paths: Record<string, Record<string, unknown>> = "path" in existingOverrides
+            ? (existingOverrides.path as Record<string, Record<string, unknown>>)
+            : {};
+        for (const endpoint of ir.endpoints) {
+            const endpointLocation = getEndpointLocation(endpoint);
+            if (!(endpoint.path in paths)) {
+                paths[endpoint.path] = {};
+            }
+            const pathItem = paths[endpoint.path];
+            if (pathItem != null && pathItem[endpoint.method] == null) {
+                const groupName = endpointLocation.file
+                    .split("/")
+                    .map((part) => part.replace(".yml", ""))
+                    .filter((part) => part !== "__package__");
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const sdkMethodNameExtensions: Record<string, any> = {};
+                if (groupName.length > 0) {
+                    sdkMethodNameExtensions["x-fern-sdk-group-name"] = groupName;
+                }
+                sdkMethodNameExtensions["x-fern-sdk-method-name"] = endpointLocation.endpointId;
+                pathItem[endpoint.method.toLowerCase()] = sdkMethodNameExtensions;
+            } else if (existingOverrides == null) {
+                context.logger.warn(`Endpoint ${endpoint.path} ${endpoint.method} is defined multiple times`);
             }
-            sdkMethodNameExtensions["x-fern-sdk-method-name"] = endpointLocation.endpointId;
-            pathItem[endpoint.method.toLowerCase()] = sdkMethodNameExtensions;
-        } else if (existingOverrides == null) {
-            context.logger.warn(`Endpoint ${endpoint.path} ${endpoint.method} is defined multiple times`);
         }
-    }
-    const schemas: Record<string, Record<string, unknown>> = "path" in existingOverrides
-        ? (existingOverrides.path as Record<string, Record<string, unknown>>)
-        : {};
-    if (includeModels) {
-        for (const [schemaId, schema] of Object.entries(openApiIr.schemas)) {
-            if (schemaId in schemas) {
-                continue;
+        const schemas: Record<string, Record<string, unknown>> = "path" in existingOverrides
+            ? (existingOverrides.path as Record<string, Record<string, unknown>>)
+            : {};
+        if (includeModels) {
+            for (const [schemaId, schema] of Object.entries(ir.schemas)) {
+                if (schemaId in schemas) {
+                    continue;
+                }
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const typeNameOverride: Record<string, any> = {};
+                typeNameOverride["x-fern-type-name"] =
+                    "nameOverride" in schema ? schema.nameOverride ?? schemaId : schemaId;
+                schemas[schemaId] = typeNameOverride;
             }
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const typeNameOverride: Record<string, any> = {};
-            typeNameOverride["x-fern-type-name"] =
-                "nameOverride" in schema ? schema.nameOverride ?? schemaId : schemaId;
-            schemas[schemaId] = typeNameOverride;
         }
-    }
-    const components: Record<string, Record<string, unknown>> = { schemas };
+        const components: Record<string, Record<string, unknown>> = { schemas };
 
-    await writeFile(
-        join(dirname(workspace.absolutePathToOpenAPI), RelativeFilePath.of("openapi-overrides.yml")),
-        yaml.dump({ paths, components })
-    );
+        await writeFile(
+            join(dirname(spec.absoluteFilepath), RelativeFilePath.of("openapi-overrides.yml")),
+            yaml.dump({ paths, components })
+        );
+    }
 }
diff --git a/packages/cli/cli/src/commands/register/registerWorkspacesV1.ts b/packages/cli/cli/src/commands/register/registerWorkspacesV1.ts
@@ -35,7 +35,7 @@ export async function registerWorkspacesV1({
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                if (workspace.type === "openapi") {
+                if (workspace.type === "oss") {
                     context.failWithoutThrowing("Registering from OpenAPI not currently supported.");
                     return;
                 }

diff --git a/packages/cli/cli/src/commands/register/registerWorkspacesV2.ts b/packages/cli/cli/src/commands/register/registerWorkspacesV2.ts
@@ -16,7 +16,7 @@ export async function registerWorkspacesV2({
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                if (workspace.type === "openapi") {
+                if (workspace.type === "oss") {
                     context.failWithoutThrowing("Cannot register OpenAPI workspace");
                 } else {
                     await registerApi({

diff --git a/packages/cli/cli/src/commands/validate/validateWorkspaces.ts b/packages/cli/cli/src/commands/validate/validateWorkspaces.ts
@@ -23,7 +23,7 @@ export async function validateWorkspaces({
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                if (workspace.type === "openapi") {
+                if (workspace.type === "oss") {
                     const fernWorkspace = await convertOpenApiWorkspaceToFernWorkspace(workspace, context);
                     await validateAPIWorkspaceAndLogIssues({ workspace: fernWorkspace, context, logWarnings });
                 } else {

diff --git a/packages/cli/cli/src/commands/write-definition/writeDefinitionForWorkspaces.ts b/packages/cli/cli/src/commands/write-definition/writeDefinitionForWorkspaces.ts
@@ -6,7 +6,7 @@ import {
     convertOpenApiWorkspaceToFernWorkspace,
     FernDefinition,
     FernWorkspace,
-    OpenAPIWorkspace
+    OSSWorkspace
 } from "@fern-api/workspace-loader";
 import chalk from "chalk";
 import { mkdir, rmdir, writeFile } from "fs/promises";
@@ -24,7 +24,7 @@ export async function writeDefinitionForWorkspaces({
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                if (workspace.type === "openapi") {
+                if (workspace.type === "oss") {
                     await writeDefinitionForOpenAPIWorkspace({ workspace, context });
                 } else {
                     await writeDefinitionForFernWorkspace({ workspace, context });
@@ -62,7 +62,7 @@ async function writeDefinitionForOpenAPIWorkspace({
     workspace,
     context
 }: {
-    workspace: OpenAPIWorkspace;
+    workspace: OSSWorkspace;
     context: TaskContext;
 }): Promise<void> {
     const fernWorkspace = await convertOpenApiWorkspaceToFernWorkspace(workspace, context);

diff --git a/packages/cli/cli/tsconfig.json b/packages/cli/cli/tsconfig.json
@@ -24,6 +24,7 @@
     { "path": "../mock" },
     { "path": "../openapi-ir-sdk" },
     { "path": "../openapi-ir-to-fern" },
+    { "path": "../openapi-parser" },
     { "path": "../posthog-manager" },
     { "path": "../project-loader" },
     { "path": "../register" },

diff --git a/packages/cli/config-management/generators-configuration/src/GeneratorsConfiguration.ts b/packages/cli/config-management/generators-configuration/src/GeneratorsConfiguration.ts
@@ -5,15 +5,25 @@ import { FernFiddle } from "@fern-fern/fiddle-sdk";
 import { GeneratorsConfigurationSchema } from "./schemas/GeneratorsConfigurationSchema";
 
 export interface GeneratorsConfiguration {
-    absolutePathToConfiguration: AbsoluteFilePath;
-    absolutePathToOpenAPI: AbsoluteFilePath | undefined;
-    absolutePathToOpenAPIOverrides: AbsoluteFilePath | undefined;
-    absolutePathToAsyncAPI: AbsoluteFilePath | undefined;
-    disableOpenAPIExamples: boolean | undefined;
-    rawConfiguration: GeneratorsConfigurationSchema;
+    api?: APIDefinition;
     defaultGroup: string | undefined;
     groups: GeneratorGroup[];
     whitelabel: FernFiddle.WhitelabelConfig | undefined;
+
+    rawConfiguration: GeneratorsConfigurationSchema;
+    absolutePathToConfiguration: AbsoluteFilePath;
+}
+
+export type APIDefinition = SingleNamespaceAPIDefinition;
+
+export interface SingleNamespaceAPIDefinition {
+    type: "singleNamespace";
+    definitions: APIDefinitionLocation[];
+}
+
+export interface APIDefinitionLocation {
+    path: string;
+    overrides: string | undefined;
 }
 
 export interface GeneratorGroup {

diff --git a/...g-management/generators-configuration/src/__test__/convertGeneratorsConfiguration.test.ts b/...g-management/generators-configuration/src/__test__/convertGeneratorsConfiguration.test.ts
@@ -123,31 +123,4 @@ describe("convertGeneratorsConfiguration", () => {
 
         expect(converted.groups[0]?.generators[0]?.outputMode?.type).toEqual("githubV2");
     });
-
-    it("OpenAPI legacy", async () => {
-        const converted = await convertGeneratorsConfiguration({
-            absolutePathToGeneratorsConfiguration: AbsoluteFilePath.of(__filename),
-            rawGeneratorsConfiguration: {
-                openapi: "path/to/openapi.yml",
-                ["openapi-overrides"]: "path/to/overrides.yml"
-            }
-        });
-
-        expect(converted.disableOpenAPIExamples).toEqual(undefined);
-    });
-
-    it("OpenAPI object", async () => {
-        const converted = await convertGeneratorsConfiguration({
-            absolutePathToGeneratorsConfiguration: AbsoluteFilePath.of(__filename),
-            rawGeneratorsConfiguration: {
-                openapi: {
-                    path: "path/to/openapi.yml",
-                    overrides: "path/to/overrides.yml",
-                    ["disable-examples"]: true
-                }
-            }
-        });
-
-        expect(converted.disableOpenAPIExamples).toEqual(true);
-    });
 });