chore: implement api-definition migrator and add fixtures (#1552)

.github/workflows/ci.yml

@@ -63,6 +63,9 @@ jobs:
       - name: Install
         uses: ./.github/actions/install
 
+      - name: Build components
+        run: pnpm turbo --filter=@fern-ui/components compile
+
       - name: Run Chromatic
         uses: chromaui/action@latest
         # Chromatic CI config: https://www.chromatic.com/docs/github-actions/

.gitignore

@@ -53,6 +53,7 @@ lerna-debug.log*
 /playwright/.cache/
 
 *storybook.log
+storybook-static/
 
 #ide
 .idea/

fern/apis/fdr/definition/api/latest/endpoint.yml

@@ -18,6 +18,7 @@ types:
     extends:
       - commons.WithDescription
       - commons.WithAvailability
+      - commons.WithNamespace
     properties:
       id: rootCommons.EndpointId # or original endpoint id?
       method: rootCommons.HttpMethod
@@ -50,7 +51,7 @@ types:
   HttpRequestBodyShape:
     union:
       object: type.ObjectType
-      reference: type.TypeReference
+      alias: type.TypeReference
       bytes: type.BytesRequest
       formData: type.FormDataRequest
 
@@ -64,7 +65,7 @@ types:
   HttpResponseBodyShape:
     union:
       object: type.ObjectType
-      reference: type.TypeReference
+      alias: type.TypeReference
       fileDownload: FileDownloadResponseBodyShape
       streamingText: {}
       stream: StreamResponse
@@ -96,15 +97,15 @@ types:
   ExampleEndpointCall:
     extends: commons.WithDescription
     properties:
-      name: optional<string>
       path: string
-      pathParameters: map<rootCommons.PropertyKey, unknown>
-      queryParameters: map<rootCommons.PropertyKey, unknown>
-      headers: map<rootCommons.PropertyKey, unknown>
-      requestBody: optional<ExampleEndpointRequest>
       responseStatusCode: integer
+      name: optional<string>
+      pathParameters: optional<map<rootCommons.PropertyKey, unknown>>
+      queryParameters: optional<map<rootCommons.PropertyKey, unknown>>
+      headers: optional<map<rootCommons.PropertyKey, unknown>>
+      requestBody: optional<ExampleEndpointRequest>
       responseBody: optional<ExampleEndpointResponse>
-      snippets: list<CodeSnippet>
+      snippets: optional<map<Language, list<CodeSnippet>>>
 
   CodeSnippet:
     extends: commons.WithDescription

fern/apis/fdr/definition/api/latest/type.yml

@@ -87,24 +87,23 @@ types:
 
   DiscriminatedUnionType:
     properties:
-      discriminant: string
+      discriminant: rootCommons.PropertyKey
       variants: list<DiscriminatedUnionVariant>
 
   DiscriminatedUnionVariant:
     extends:
       - commons.WithDescription
       - commons.WithAvailability
+      - ObjectType
     properties:
       discriminantValue: string
       displayName: optional<string>
-      additionalProperties: ObjectType
 
   FormDataRequest:
     extends:
       - commons.WithDescription
       - commons.WithAvailability
     properties:
-      name: string
       fields: list<FormDataField>
 
   FormDataField:
@@ -114,7 +113,7 @@ types:
       property: FormDataProperty
 
   FormDataProperty:
-    extends: ObjectProperty
+    extends: ObjectProperty # has description and availability
     properties:
       contentType: optional<ContentType>
 

fern/apis/fdr/definition/api/latest/webhook.yml

@@ -11,13 +11,14 @@ types:
     extends:
       - commons.WithDescription
       - commons.WithAvailability
+      - commons.WithNamespace
     properties:
       id: rootCommons.WebhookId
       method: WebhookHttpMethod
       path: list<string>
-      headers: list<type.ObjectProperty>
-      payload: WebhookPayload
-      examples: list<v1Read.ExampleWebhookPayload>
+      headers: optional<list<type.ObjectProperty>>
+      payload: optional<WebhookPayload>
+      examples: optional<list<v1Read.ExampleWebhookPayload>>
 
   WebhookPayload:
     extends: commons.WithDescription

fern/apis/fdr/definition/api/latest/websocket.yml

@@ -18,6 +18,7 @@ types:
     extends:
       - commons.WithDescription
       - commons.WithAvailability
+      - commons.WithNamespace
     properties:
       id: rootCommons.WebSocketId
       path: list<commons.PathPart>
@@ -46,9 +47,9 @@ types:
   ExampleWebSocketSession:
     extends: commons.WithDescription
     properties:
-      name: optional<string>
       path: string
-      pathParameters: map<rootCommons.PropertyKey, unknown>
-      queryParameters: map<rootCommons.PropertyKey, unknown>
-      requestHeaders: map<rootCommons.PropertyKey, unknown>
-      messages: list<ExampleWebSocketMessage>
+      name: optional<string>
+      pathParameters: optional<map<rootCommons.PropertyKey, unknown>>
+      queryParameters: optional<map<rootCommons.PropertyKey, unknown>>
+      requestHeaders: optional<map<rootCommons.PropertyKey, unknown>>
+      messages: optional<list<ExampleWebSocketMessage>>

package.json

@@ -31,7 +31,7 @@
     "docs:build": "turbo docs:build",
     "docs:start": "turbo docs:start",
     "docs:e2e": "vitest --run test/docs-e2e/**/* --globals",
-    "fdr:generate": "pnpm fern generate --api fdr --local && pnpm --filter=@fern-api/fdr-sdk compile"
+    "fdr:generate": "pnpm fern generate --api fdr --local && pnpm turbo --filter=@fern-api/fdr-sdk compile"
   },
   "devDependencies": {
     "@babel/core": "^7.23.7",

packages/commons/core-utils/package.json

@@ -12,9 +12,37 @@
   ],
   "type": "module",
   "source": "src/index.ts",
-  "module": "src/index.ts",
   "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "exports": {
+    "./assertNever": {
+      "types": "./src/assertNever.ts",
+      "default": "./dist/assertNever.js"
+    },
+    "./titleCase": {
+      "types": "./src/titleCase.ts",
+      "default": "./dist/titleCase.js"
+    },
+    "./unknownToString": {
+      "types": "./src/unknownToString.ts",
+      "default": "./dist/unknownToString.js"
+    },
+    "./visitDiscriminatedUnion": {
+      "types": "./src/visitDiscriminatedUnion.ts",
+      "default": "./dist/visitDiscriminatedUnion.js"
+    },
+    "./isPlainObject": {
+      "types": "./src/objects/isPlainObject.ts",
+      "default": "./dist/objects/isPlainObject.js"
+    },
+    "./identity": {
+      "types": "./src/identity.ts",
+      "default": "./dist/identity.js"
+    },
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
   "sideEffects": false,
   "scripts": {
     "clean": "rm -rf ./lib && tsc --build --clean",

packages/commons/core-utils/src/assertNever.ts

@@ -4,3 +4,5 @@ export function assertNever(x: never): never {
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
 export function assertNeverNoThrow(_: never): void {}
+
+export default assertNever;

packages/commons/core-utils/src/identity.ts

@@ -1,3 +1,5 @@
 export function identity<T>(value: T): T {
     return value;
 }
+
+export default identity;

packages/commons/core-utils/src/objects/isPlainObject.ts

@@ -16,3 +16,5 @@ export function isPlainObject(value: unknown): value is Record<string, unknown>
 function isObjectLike(value: unknown) {
     return typeof value === "object" && value != null;
 }
+
+export default isPlainObject;

packages/commons/core-utils/src/titleCase.ts

@@ -17,3 +17,5 @@ export function titleCase(name: string): string {
     const versionedTitle = titleCased.replace(/V\s(\d)/g, "V$1");
     return versionedTitle;
 }
+
+export default titleCase;

packages/commons/core-utils/src/unknownToString.ts

@@ -10,3 +10,5 @@ export function unknownToString(value: unknown, { renderNull = false }: Opts = {
     }
     return JSON.stringify(value);
 }
+
+export default unknownToString;

packages/commons/core-utils/src/visitDiscriminatedUnion.ts

@@ -31,3 +31,5 @@ export function visitDiscriminatedUnion<T extends Record<Discriminant, string>,
         },
     };
 }
+
+export default visitDiscriminatedUnion;

packages/fdr-sdk/package.json

@@ -31,6 +31,7 @@
     }
   },
   "dependencies": {
+    "@fern-ui/core-utils": "workspace:*",
     "dayjs": "^1.11.11",
     "fast-deep-equal": "^3.1.3",
     "form-data": "4.0.0",
@@ -43,6 +44,9 @@
     "ts-essentials": "^10.0.1",
     "url-join": "5.0.0"
   },
+  "peerDependencies": {
+    "lodash-es": ">=4"
+  },
   "devDependencies": {
     "@babel/core": "^7.18.9",
     "@babel/preset-env": "^7.24.3",

packages/fdr-sdk/src/navigation/__test__/fixtures.test.ts → packages/fdr-sdk/src/__test__/fixtures.test.ts

@@ -1,10 +1,12 @@
+import isPlainObject from "@fern-ui/core-utils/isPlainObject";
 import fs from "fs";
 import path from "path";
-import { FernNavigation } from "../..";
-import { isPlainObject } from "../../utils";
-import { NodeCollector } from "../NodeCollector";
-import { FernNavigationV1ToLatest } from "../migrators/v1ToV2";
-import { collectPageIds } from "../utils/collectPageIds";
+import { FernNavigation } from "..";
+import * as ApiDefinition from "../api-definition";
+import { ApiDefinitionV1ToLatest } from "../api-definition/migrators/v1ToV2";
+import { NodeCollector } from "../navigation/NodeCollector";
+import { FernNavigationV1ToLatest } from "../navigation/migrators/v1ToV2";
+import { collectPageIds } from "../navigation/utils/collectPageIds";
 import { readFixture } from "./readFixtures";
 
 const fixturesDir = path.join(__dirname, "fixtures");
@@ -14,11 +16,19 @@ function testNavigationConfigConverter(fixtureName: string): void {
     const v1 = FernNavigation.V1.toRootNode(fixture);
     const latest = FernNavigationV1ToLatest.create().root(v1);
 
+    const v2Apis = Object.values(fixture.definition.apis).map((api) =>
+        ApiDefinitionV1ToLatest.from(api, {
+            useJavaScriptAsTypeScript: false,
+            alwaysEnableJavaScriptFetch: false,
+            usesApplicationJsonInFormDataValue: false,
+        }).migrate(),
+    );
+
     // eslint-disable-next-line vitest/valid-title
     describe(fixtureName, () => {
         const collector = new NodeCollector(latest);
 
-        it("gets all urls from docs config", async () => {
+        it("gets all urls from docs config", () => {
             expect(JSON.stringify(sortObject(latest), undefined, 2)).toMatchFileSnapshot(
                 `output/${fixtureName}/node.json`,
             );
@@ -58,6 +68,27 @@ function testNavigationConfigConverter(fixtureName: string): void {
             expect(JSON.stringify(sortObject(collector.getVersionNodes()), undefined, 2)).toMatchFileSnapshot(
                 `output/${fixtureName}/versionNodes.json`,
             );
+
+            expect(JSON.stringify(v2Apis, undefined, 2)).toMatchFileSnapshot(
+                `output/${fixtureName}/apiDefinitions.json`,
+            );
+        });
+
+        describe("transformer", () => {
+            it("should generate unique keys", async () => {
+                for (const api of Object.values(v2Apis)) {
+                    const keys: string[] = [];
+
+                    await ApiDefinition.Transformer.keys((key) => keys.push(key)).apiDefinition(api);
+
+                    expect(JSON.stringify(keys, undefined, 2)).toMatchFileSnapshot(
+                        `output/${fixtureName}/apiDefinitionKeys-${api.id}.json`,
+                    );
+
+                    // all keys must be unique
+                    expect(Array.from(new Set(keys)).length).toBe(keys.length);
+                }
+            });
         });
 
         it("should have unique canonical urls for each page", () => {

packages/fdr-sdk/src/navigation/__test__/fixtures/assemblyai.json → packages/fdr-sdk/src/__test__/fixtures/assemblyai.json

@@ -15000,13 +15000,13 @@
       }
     },
     "files": {
-      "eaff1b77-0988-489d-b430-793cfad2bc54": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/ABCMonumentGrotesk-Regular.woff2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=c1b697fe1027be563d0167c5d2fde64197c22dd4f1d77f0028f0cb617396a3f8&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "36e34a20-ff6d-4cde-8b14-92d6ff2af084": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/ABCMonumentGrotesk-Medium.woff2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=5f774938d6aab931be53cb1532d88454e2fe14ccc2c8f3ac2981b49ab53c73a0&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "744e09e3-6bb2-4881-8e7b-c26ad4d8f752": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/ABCMonumentGrotesk-Bold.woff2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=c127e2555cfa09e3388c8dedf269655e9c2e5055ee8adcdd3446c7b23e17faf9&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "6c188f84-edb2-4fbd-93a7-d4a7374f5693": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/JetBrainsMono-Regular.woff2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=f81ea5b5e7150031fb01e149b59490fab593a26437cbf4876eac918be067784a&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "f981c76f-4e4b-4ef7-a199-4e6fc53b8c5f": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/JetBrainsMono-Medium.woff2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=37ce283e32014ed9af561a8fd1e5654931680f7f22742a75d9a134ba70851981&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "5351f437-2e66-4272-9488-e376c093233d": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/AssemblyAI_White.svg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=ff41d6c6343c6772357ccb7b16da3b04bccdd1b8e5384f9c7d22bc8e8dd030a9&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "e5c8b272-6062-415e-b5ec-8e446c129fc2": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/favicon.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240531%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240531T192111Z&X-Amz-Expires=604800&X-Amz-Signature=cffbdb9a08ff637e5b37fe2f402dbdeec9d764417ee49d3638265e3f19211c7d&X-Amz-SignedHeaders=host&x-id=GetObject"
+      "eaff1b77-0988-489d-b430-793cfad2bc54": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/ABCMonumentGrotesk-Regular.woff2",
+      "36e34a20-ff6d-4cde-8b14-92d6ff2af084": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/ABCMonumentGrotesk-Medium.woff2",
+      "744e09e3-6bb2-4881-8e7b-c26ad4d8f752": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/ABCMonumentGrotesk-Bold.woff2",
+      "6c188f84-edb2-4fbd-93a7-d4a7374f5693": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/JetBrainsMono-Regular.woff2",
+      "f981c76f-4e4b-4ef7-a199-4e6fc53b8c5f": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/JetBrainsMono-Medium.woff2",
+      "5351f437-2e66-4272-9488-e376c093233d": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/AssemblyAI_White.svg",
+      "e5c8b272-6062-415e-b5ec-8e446c129fc2": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/assemblyai.docs.buildwithfern.com/docs/2024-05-31T19%3A21%3A05.796Z/assets/favicon.png"
     },
     "filesV2": {
       "eaff1b77-0988-489d-b430-793cfad2bc54": {

packages/fdr-sdk/src/navigation/__test__/fixtures/astronomer.json → packages/fdr-sdk/src/__test__/fixtures/astronomer.json

@@ -30742,9 +30742,9 @@
       }
     },
     "files": {
-      "015057e2-bb14-4cd7-97dd-b2cca2a0b221": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/astronomer.docs.buildwithfern.com/docs/api/2024-05-24T18%3A11%3A19.885Z/docs/assets/logo-dark.svg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240524T181126Z&X-Amz-Expires=604800&X-Amz-Signature=6f1c9d734aadc87fe7616663d5488b60eeca4b38b7fc16220f49b7ab347ebc6a&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "49419424-29d4-4e84-983a-1d6d69cf072d": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/astronomer.docs.buildwithfern.com/docs/api/2024-05-24T18%3A11%3A19.885Z/docs/assets/logo-light.svg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240524T181126Z&X-Amz-Expires=604800&X-Amz-Signature=cc7645af242e4c0405dfa08807eabacffbd170bd8fb95eb89c441dd394184f32&X-Amz-SignedHeaders=host&x-id=GetObject",
-      "06193aca-2385-478c-8973-6c20d3994186": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/astronomer.docs.buildwithfern.com/docs/api/2024-05-24T18%3A11%3A19.885Z/docs/assets/favicon.svg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA6KXJSKKNE6LAYO7B%2F20240524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240524T181126Z&X-Amz-Expires=604800&X-Amz-Signature=6ac0c469761e7e63a44bd1220bd4d3f9866d53fd7df2e18c5c3dc9d18c6c24a7&X-Amz-SignedHeaders=host&x-id=GetObject"
+      "015057e2-bb14-4cd7-97dd-b2cca2a0b221": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/astronomer.docs.buildwithfern.com/docs/api/2024-05-24T18%3A11%3A19.885Z/docs/assets/logo-dark.svg",
+      "49419424-29d4-4e84-983a-1d6d69cf072d": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/astronomer.docs.buildwithfern.com/docs/api/2024-05-24T18%3A11%3A19.885Z/docs/assets/logo-light.svg",
+      "06193aca-2385-478c-8973-6c20d3994186": "https://fdr-prod-docs-files.s3.us-east-1.amazonaws.com/astronomer.docs.buildwithfern.com/docs/api/2024-05-24T18%3A11%3A19.885Z/docs/assets/favicon.svg"
     },
     "filesV2": {
       "015057e2-bb14-4cd7-97dd-b2cca2a0b221": {