feat: FDR Latest pulling at the edge (S3 upload/download)

fern/apis/fdr/definition/api/latest/__package__.yml

@@ -36,6 +36,12 @@ types:
       auths: map<auth.AuthSchemeId, auth.AuthScheme>
       globalHeaders: optional<list<type.ObjectProperty>>
 
+  LatestApiDefinition:
+    discriminated: false
+    union:
+      - string
+      - ApiDefinition
+
 errors:
   ApiDoesNotExistError:
     status-code: 404

fern/apis/fdr/definition/docs/v1/read/__package__.yml

@@ -68,7 +68,7 @@ types:
       algoliaSearchIndex: optional<algolia.AlgoliaSearchIndex>
       pages: map<rootCommons.PageId, PageContent>
       apis: map<rootCommons.ApiDefinitionId, apiReadV1.ApiDefinition>
-      apisV2: map<rootCommons.ApiDefinitionId, apiReadLatest.ApiDefinition>
+      apisV2: map<rootCommons.ApiDefinitionId, apiReadLatest.LatestApiDefinition>
       files: map<rootCommons.FileId, rootCommons.Url>
       filesV2: map<rootCommons.FileId, File>
       jsFiles:

packages/fdr-sdk/package.json

@@ -49,6 +49,8 @@
     "test": "vitest --run"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.726.1",
+    "@aws-sdk/s3-request-presigner": "^3.726.1",
     "@fern-api/ui-core-utils": "workspace:*",
     "@ungap/structured-clone": "^1.2.0",
     "dayjs": "^1.11.11",

packages/fdr-sdk/src/api-definition/index.ts

@@ -6,6 +6,7 @@ export * from "./lang";
 export * from "./latest";
 export * from "./migrators/v1ToV2";
 export * from "./prune";
+export * from "./s3Loader";
 export * from "./snippets/curl";
 export * from "./snippets/SnippetHttpRequest";
 export * from "./status-message";

packages/fdr-sdk/src/api-definition/s3Loader.ts

@@ -0,0 +1,51 @@
+import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import { FdrAPI } from "../client";
+import { LatestApiDefinition } from "./latest";
+
+export class S3Loader {
+  private s3Client: S3Client | undefined;
+  private bucketName: string | undefined;
+
+  constructor() {
+    if (
+      process.env.AWS_ACCESS_KEY_ID != null &&
+      process.env.AWS_SECRET_ACCESS_KEY != null &&
+      process.env.AWS_REGION != null &&
+      process.env.AWS_S3_BUCKET_NAME != null
+    ) {
+      this.s3Client = new S3Client({
+        region: process.env.AWS_REGION,
+        credentials: {
+          accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+        },
+      });
+      this.bucketName = process.env.AWS_S3_BUCKET_NAME;
+    }
+  }
+
+  async loadApiDefinition(
+    apiDefinitionOrKey: LatestApiDefinition
+  ): Promise<FdrAPI.api.latest.ApiDefinition> {
+    if (this.s3Client == null) {
+      throw new Error("S3 client not initialized");
+    }
+
+    let resolvedApi: FdrAPI.api.latest.ApiDefinition;
+    if (typeof apiDefinitionOrKey === "string") {
+      const command = new GetObjectCommand({
+        Bucket: this.bucketName,
+        Key: apiDefinitionOrKey,
+      });
+      const url = await getSignedUrl(this.s3Client, command, {
+        expiresIn: 604800,
+      });
+      resolvedApi = await (await fetch(url)).json();
+    } else {
+      resolvedApi = apiDefinitionOrKey;
+    }
+
+    return resolvedApi;
+  }
+}

packages/fdr-sdk/src/converters/read/convertDocsDefinitionToRead.ts

@@ -1,5 +1,5 @@
 import { mapValues } from "es-toolkit/object";
-import { APIV1Db, APIV1Read, DocsV1Db, DocsV1Read } from "../../client";
+import { APIV1Db, APIV1Read, DocsV1Db, DocsV1Read, FdrAPI } from "../../client";
 import { SearchInfo } from "../../client/FdrAPI";
 import { FernRegistry } from "../../client/generated";
 import { convertDbDocsConfigToRead } from "./convertDbDocsConfigToRead";
@@ -19,7 +19,7 @@ export function convertDocsDefinitionToRead({
   apis: Record<DocsV1Db.ApiDefinitionId, APIV1Read.ApiDefinition>;
   apisV2: Record<
     FernRegistry.ApiDefinitionId,
-    FernRegistry.api.latest.ApiDefinition
+    FdrAPI.api.latest.LatestApiDefinition
   >;
   id: APIV1Db.DocsConfigId | undefined;
   search: SearchInfo;

packages/fdr-sdk/src/navigation/utils/toApis.ts

@@ -2,7 +2,16 @@ import { mapValues } from "es-toolkit/object";
 import { ApiDefinition } from "../..";
 import { DocsV2Read } from "../../client";
 
-export function toApis(docs: DocsV2Read.LoadDocsForUrlResponse) {
+export async function toApis(
+  docs: DocsV2Read.LoadDocsForUrlResponse,
+  getPresignedDocsAssetsDownloadUrl: ({
+    key,
+    isPrivate,
+  }: {
+    key: string;
+    isPrivate: boolean;
+  }) => Promise<string>
+) {
   return {
     ...mapValues(docs.definition.apis, (api) =>
       ApiDefinition.ApiDefinitionV1ToLatest.from(api, {
@@ -11,6 +20,22 @@ export function toApis(docs: DocsV2Read.LoadDocsForUrlResponse) {
         usesApplicationJsonInFormDataValue: false,
       }).migrate()
     ),
-    ...docs.definition.apisV2,
+    ...Object.fromEntries(
+      await Promise.all(
+        Object.entries(docs.definition.apisV2 ?? {}).map(async ([key, def]) => {
+          if (typeof def === "string") {
+            const url = await getPresignedDocsAssetsDownloadUrl({
+              key: def,
+              isPrivate: true,
+            });
+            const response = await fetch(url.toString());
+            const apiDefinition = await response.json();
+            return [key, apiDefinition as ApiDefinition.ApiDefinition];
+          } else {
+            return [key, def];
+          }
+        })
+      )
+    ),
   };
 }

packages/fern-docs/bundle/src/server/DocsLoader.ts

@@ -2,6 +2,7 @@ import type { DocsV1Read, DocsV2Read } from "@fern-api/fdr-sdk";
 import {
   ApiDefinition,
   ApiDefinitionV1ToLatest,
+  S3Loader,
 } from "@fern-api/fdr-sdk/api-definition";
 import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
 import type { AuthEdgeConfig } from "@fern-docs/auth";
@@ -103,11 +104,13 @@ export class DocsLoader {
       return undefined;
     }
     const v1 = res.definition.apis[key];
+    const s3Loader = new S3Loader();
     const latest =
-      res.definition.apisV2?.[key] ??
-      (v1 != null
-        ? ApiDefinitionV1ToLatest.from(v1, this.edgeFlags).migrate()
-        : undefined);
+      res.definition.apisV2?.[key] != null
+        ? await s3Loader.loadApiDefinition(res.definition.apisV2[key])
+        : v1 != null
+          ? ApiDefinitionV1ToLatest.from(v1, this.edgeFlags).migrate()
+          : undefined;
     if (!latest) {
       return undefined;
     }

packages/fern-docs/search-server/src/algolia/__test__/generation.test.ts

@@ -50,20 +50,26 @@ for (const fixtureName of [
   describe(fixtureName, () => {
     it("should work", async () => {
       const fixture = readFixture(fixtureName);
+      const mockGetPresignedDocsAssetsDownloadUrl = vi.fn();
       const root = FernNavigation.utils.toRootNode(fixture);
-      const apis = FernNavigation.utils.toApis(fixture);
+      console.log(`${fixtureName} - a`);
+      const apis = await FernNavigation.utils.toApis(
+        fixture,
+        mockGetPresignedDocsAssetsDownloadUrl
+      );
+      console.log(`${fixtureName} - b`);
       const pages = FernNavigation.utils.toPages(fixture);
-
+      console.log(`${fixtureName} - c`);
       const { records, tooLarge } = createAlgoliaRecords({
         root,
         domain: "test.com",
         org_id: "test",
         pages,
         apis,
       });
-
+      console.log(`${fixtureName} - d`);
       expect(tooLarge.length).toBe(0);
-
+      console.log(`${fixtureName} - e`);
       records.forEach((record) => {
         if (record.description != null) {
           expect(record.description.length).toBeLessThanOrEqual(50_000);
@@ -73,9 +79,11 @@ for (const fixtureName of [
           expect(record.content.length).toBeLessThanOrEqual(50_000);
         }
       });
+      console.log(`${fixtureName} - f`);
       await expect(JSON.stringify(records, null, 2)).toMatchFileSnapshot(
         path.join("__snapshots__", `${fixtureName}.test.ts.json`)
       );
-    });
+      console.log(`${fixtureName} - g`);
+    }, 100_000);
   });
 }

packages/fern-docs/search-server/src/algolia/__test__/test-utils.ts

@@ -42,7 +42,11 @@ export function readFixtureToRootNode(
         ];
       })
     ),
-    ...fixture.definition.apisV2,
+    ...(Object.fromEntries(
+      Object.entries(fixture.definition.apisV2 ?? {}).filter(
+        ([_, api]) => typeof api !== "string"
+      )
+    ) as Record<string, ApiDefinition.ApiDefinition>),
   };
   const pages = mapValues(fixture.definition.pages, (page) => page.markdown);
   return { root, apis, pages };

packages/fern-docs/search-server/src/fdr/load-docs-with-url.ts

@@ -1,4 +1,5 @@
 import { ApiDefinition, FdrClient, FernNavigation } from "@fern-api/fdr-sdk";
+import { S3Loader } from "@fern-api/fdr-sdk/api-definition";
 import { withDefaultProtocol } from "@fern-api/ui-core-utils";
 import { mapValues } from "es-toolkit/object";
 
@@ -72,6 +73,7 @@ export async function loadDocsWithUrl(
   // migrate pages
   const pages = mapValues(docs.body.definition.pages, (page) => page.markdown);
 
+  const s3Loader = new S3Loader();
   // migrate apis
   const apis = {
     ...mapValues(docs.body.definition.apis, (api) =>
@@ -83,7 +85,16 @@ export async function loadDocsWithUrl(
           payload.usesApplicationJsonInFormDataValue ?? false,
       }).migrate()
     ),
-    ...docs.body.definition.apisV2,
+    ...Object.fromEntries(
+      await Promise.all(
+        Object.entries(docs.body.definition.apisV2 ?? {}).map(
+          async ([apiId, api]) => {
+            const resolvedApi = await s3Loader.loadApiDefinition(api);
+            return [apiId, resolvedApi];
+          }
+        )
+      )
+    ),
   };
 
   return { org_id: org.body, root, pages, apis, domain: domain.host };

packages/fern-docs/ui/src/resolver/resolveDocsContent.ts

@@ -1,4 +1,7 @@
-import { ApiDefinitionV1ToLatest } from "@fern-api/fdr-sdk/api-definition";
+import {
+  ApiDefinitionV1ToLatest,
+  S3Loader,
+} from "@fern-api/fdr-sdk/api-definition";
 import type {
   APIV1Read,
   DocsV1Read,
@@ -31,7 +34,7 @@ interface ResolveDocsContentArgs {
   prev: FernNavigation.NavigationNodeNeighbor | undefined;
   next: FernNavigation.NavigationNodeNeighbor | undefined;
   apis: Record<string, APIV1Read.ApiDefinition>;
-  apisV2: Record<string, FdrAPI.api.latest.ApiDefinition>;
+  apisV2: Record<string, FdrAPI.api.latest.LatestApiDefinition>;
   pages: Record<string, DocsV1Read.PageContent>;
   mdxOptions?: FernSerializeMdxOptions;
   edgeFlags: EdgeFlags;
@@ -69,6 +72,7 @@ export async function resolveDocsContent({
       engine
     );
 
+  const s3Loader = new S3Loader();
   // TODO: remove legacy when done
   const apiLoaders = {
     ...mapValues(apis, (api) => {
@@ -81,14 +85,22 @@ export async function resolveDocsContent({
         .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN)
         .withResolveDescriptions();
     }),
-    ...mapValues(apisV2 ?? {}, (api) => {
-      return ApiDefinitionLoader.create(domain, api.id)
-        .withMdxBundler(serializeMdx, engine)
-        .withEdgeFlags(edgeFlags)
-        .withApiDefinition(api)
-        .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN)
-        .withResolveDescriptions();
-    }),
+    ...Object.fromEntries(
+      await Promise.all(
+        Object.entries(apisV2 ?? {}).map(async ([apiId, api]) => {
+          const resolvedApi = await s3Loader.loadApiDefinition(api);
+          return [
+            apiId,
+            ApiDefinitionLoader.create(domain, resolvedApi.id)
+              .withMdxBundler(serializeMdx, engine)
+              .withEdgeFlags(edgeFlags)
+              .withApiDefinition(resolvedApi)
+              .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN)
+              .withResolveDescriptions(),
+          ];
+        })
+      )
+    ),
   };
 
   let result: DocsContent | undefined;