From c7efecf29a79f827d774559782218963fc4d838e Mon Sep 17 00:00:00 2001
From: Andrew Jiang <andrew@buildwithfern.com>
Date: Tue, 2 Jul 2024 18:14:06 -0400
Subject: [PATCH] fix: algolia records should index endpoint body shape (#1103)

Co-authored-by: dsinghvi <deep@usebirch.com>
---
 .../commons/search-utils/src/SearchConfig.ts  |  71 +++---
 packages/ui/app/src/search/SearchDialog.tsx   |  18 +-
 .../search/algolia/useAlgoliaSearchClient.ts  |  15 +-
 .../src/search/inkeep/InkeepCustomTrigger.tsx |  10 +-
 .../src/search/inkeep/useInkeepSettings.ts    |  10 +-
 .../ui/app/src/sidebar/SidebarSearchBar.tsx   |  14 +-
 .../algolia/AlgoliaSearchRecordGenerator.ts   | 230 +++++++++++++++++-
 .../services/algolia/getAllReferencedTypes.ts | 135 ++++++++++
 8 files changed, 429 insertions(+), 74 deletions(-)
 create mode 100644 servers/fdr/src/services/algolia/getAllReferencedTypes.ts

diff --git a/packages/commons/search-utils/src/SearchConfig.ts b/packages/commons/search-utils/src/SearchConfig.ts
index cc0e4ea1f8..eb69cd86f0 100644
--- a/packages/commons/search-utils/src/SearchConfig.ts
+++ b/packages/commons/search-utils/src/SearchConfig.ts
@@ -16,6 +16,7 @@ export const REGISTRY_SERVICE = new FdrClient({
 const FEATURE_FLAGS = ["inkeep-enabled" as const];
 
 export type InkeepSharedSettings = DeepReadonly<{
+    replaceSearch: boolean;
     baseSettings: InkeepWidgetBaseSettings;
     aiChatSettings?: Partial<InkeepAIChatSettings>;
     searchSettings?: Partial<InkeepSearchSettings>;
@@ -39,48 +40,35 @@ export declare namespace SearchConfig {
     }
 
     interface Inkeep extends InkeepSharedSettings {
-        isAvailable: true;
-        type: "inkeep";
+        replaceSearch: boolean;
     }
 
     interface Algolia {
-        isAvailable: true;
-        type: "algolia";
         appId: string;
         searchApiKey: Unversioned | Versioned;
         index: string;
     }
 
+    interface Available {
+        isAvailable: true;
+        inkeep?: Inkeep;
+        algolia: Algolia;
+    }
+
     interface Unavailable {
         isAvailable: false;
     }
 }
 
-export type SearchConfig = SearchConfig.Inkeep | SearchConfig.Algolia | SearchConfig.Unavailable;
+export type SearchConfig = SearchConfig.Available | SearchConfig.Unavailable;
 
 export interface SearchRequest {
     searchInfo: Algolia.SearchInfo | undefined;
 }
 
-export async function getSearchConfig(domain: string, { searchInfo }: SearchRequest): Promise<SearchConfig> {
-    try {
-        const config = await getAll<EdgeConfigResponse>(FEATURE_FLAGS);
-        const maybeInkeep = config["inkeep-enabled"]?.[domain];
-
-        if (maybeInkeep?.baseSettings.integrationId != null) {
-            return {
-                isAvailable: true,
-                type: "inkeep",
-                ...maybeInkeep,
-            };
-        }
-    } catch (e) {
-        // eslint-disable-next-line no-console
-        console.error("Error fetching edge config", e);
-    }
-
+async function getAlgoliaSearchConfig({ searchInfo }: SearchRequest): Promise<SearchConfig.Algolia | undefined> {
     if (typeof searchInfo !== "object" || searchInfo.type === "legacyMultiAlgoliaIndex") {
-        return { isAvailable: false };
+        return undefined;
     }
 
     const algoliaAppId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID;
@@ -95,12 +83,10 @@ export async function getSearchConfig(domain: string, { searchInfo }: SearchRequ
         });
 
         if (!resp.ok) {
-            return { isAvailable: false };
+            return undefined;
         }
 
         return {
-            isAvailable: true,
-            type: "algolia",
             appId: algoliaAppId,
             searchApiKey: {
                 type: "unversioned",
@@ -117,19 +103,17 @@ export async function getSearchConfig(domain: string, { searchInfo }: SearchRequ
             });
 
             if (!resp.ok) {
-                return { isAvailable: false };
+                return undefined;
             }
 
             values[versionId] = resp.body.searchApiKey;
         }
 
         if (Object.keys(values).length === 0) {
-            return { isAvailable: false };
+            return undefined;
         }
 
         return {
-            isAvailable: true,
-            type: "algolia",
             appId: algoliaAppId,
             searchApiKey: {
                 type: "versioned",
@@ -139,5 +123,30 @@ export async function getSearchConfig(domain: string, { searchInfo }: SearchRequ
         };
     }
 
-    return { isAvailable: false };
+    return undefined;
+}
+
+export async function getSearchConfig(domain: string, searchRequest: SearchRequest): Promise<SearchConfig> {
+    const algolia = await getAlgoliaSearchConfig(searchRequest);
+    if (algolia == null) {
+        return { isAvailable: false };
+    }
+
+    try {
+        const config = await getAll<EdgeConfigResponse>(FEATURE_FLAGS);
+        const maybeInkeep = config["inkeep-enabled"]?.[domain];
+
+        if (maybeInkeep?.baseSettings.integrationId != null) {
+            return {
+                isAvailable: true,
+                algolia,
+                inkeep: maybeInkeep,
+            };
+        }
+    } catch (e) {
+        // eslint-disable-next-line no-console
+        console.error("Error fetching edge config", e);
+    }
+
+    return { isAvailable: true, algolia };
 }
diff --git a/packages/ui/app/src/search/SearchDialog.tsx b/packages/ui/app/src/search/SearchDialog.tsx
index 3c01f79762..c41ce5e691 100644
--- a/packages/ui/app/src/search/SearchDialog.tsx
+++ b/packages/ui/app/src/search/SearchDialog.tsx
@@ -32,20 +32,20 @@ export const SearchDialog: React.FC<SearchDialog.Props> = ({ fromHeader }) => {
         return null;
     }
 
-    if (config.type === "algolia") {
+    if (config.inkeep == null) {
         return <AlgoliaSearchDialog fromHeader={fromHeader} />;
-    }
-
-    if (config.type === "inkeep") {
+    } else {
         return (
             <>
-                <InkeepCustomTrigger />
+                {config.inkeep.replaceSearch ? (
+                    <InkeepCustomTrigger />
+                ) : (
+                    <AlgoliaSearchDialog fromHeader={fromHeader} />
+                )}
                 <InkeepChatButton />
             </>
         );
     }
-
-    return null;
 };
 
 export declare namespace SearchSidebar {
@@ -69,7 +69,7 @@ export const SearchSidebar: React.FC<PropsWithChildren<SearchSidebar.Props>> = (
         return <>{children}</>;
     }
 
-    if (searchConfig.type === "algolia" && algoliaSearchClient != null) {
+    if (searchConfig.inkeep?.replaceSearch !== true && algoliaSearchClient != null) {
         const [searchClient, indexName] = algoliaSearchClient;
 
         return (
@@ -80,7 +80,7 @@ export const SearchSidebar: React.FC<PropsWithChildren<SearchSidebar.Props>> = (
         );
     }
 
-    if (searchConfig.type === "inkeep") {
+    if (searchConfig.inkeep != null) {
         return (
             <>
                 <div className="p-4 pb-0">
diff --git a/packages/ui/app/src/search/algolia/useAlgoliaSearchClient.ts b/packages/ui/app/src/search/algolia/useAlgoliaSearchClient.ts
index 0eaeecd14d..b5cd5b5ef3 100644
--- a/packages/ui/app/src/search/algolia/useAlgoliaSearchClient.ts
+++ b/packages/ui/app/src/search/algolia/useAlgoliaSearchClient.ts
@@ -9,25 +9,28 @@ export function useAlgoliaSearchClient(): [SearchClient, index: string] | undefi
     const [searchConfig] = useSearchConfig();
 
     return useMemo(() => {
-        if (!searchConfig.isAvailable || searchConfig.type !== "algolia") {
+        if (!searchConfig.isAvailable) {
             return;
         }
 
-        if (searchConfig.searchApiKey.type === "unversioned") {
-            return [algolia(searchConfig.appId, searchConfig.searchApiKey.value), searchConfig.index];
+        if (searchConfig.algolia.searchApiKey.type === "unversioned") {
+            return [
+                algolia(searchConfig.algolia.appId, searchConfig.algolia.searchApiKey.value),
+                searchConfig.algolia.index,
+            ];
         }
 
-        if (searchConfig.searchApiKey.type === "versioned") {
+        if (searchConfig.algolia.searchApiKey.type === "versioned") {
             assertNonNullish(
                 currentVersionId,
                 "Inconsistent State: Received search info is versioned but docs are unversioned.",
             );
-            const searchApiKey = searchConfig.searchApiKey.values[currentVersionId];
+            const searchApiKey = searchConfig.algolia.searchApiKey.values[currentVersionId];
             assertNonNullish(
                 searchApiKey,
                 `Inconsistent State: Did not receive index segment for version "${currentVersionId}". This may indicate a backend bug.`,
             );
-            return [algolia(searchConfig.appId, searchApiKey), searchConfig.index];
+            return [algolia(searchConfig.algolia.appId, searchApiKey), searchConfig.algolia.index];
         }
         return;
     }, [currentVersionId, searchConfig]);
diff --git a/packages/ui/app/src/search/inkeep/InkeepCustomTrigger.tsx b/packages/ui/app/src/search/inkeep/InkeepCustomTrigger.tsx
index d6906ee7c5..571f0a9f46 100644
--- a/packages/ui/app/src/search/inkeep/InkeepCustomTrigger.tsx
+++ b/packages/ui/app/src/search/inkeep/InkeepCustomTrigger.tsx
@@ -1,26 +1,22 @@
 import { useEventCallback } from "@fern-ui/react-commons";
-import { atom, useAtom } from "jotai";
+import { useAtom } from "jotai";
 import dynamic from "next/dynamic";
 import { ReactElement } from "react";
-import { useSearchTrigger } from "../useSearchTrigger";
+import { SEARCH_DIALOG_OPEN_ATOM } from "../../atoms/sidebar";
 import useInkeepSettings from "./useInkeepSettings";
 
 const CustomTrigger = dynamic(() => import("@inkeep/widgets").then((mod) => mod.InkeepCustomTrigger), {
     ssr: false,
 });
 
-export const INKEEP_TRIGGER = atom(false);
-
 export function InkeepCustomTrigger(): ReactElement | null {
     const settings = useInkeepSettings();
-    const [isOpen, setIsOpen] = useAtom(INKEEP_TRIGGER);
+    const [isOpen, setIsOpen] = useAtom(SEARCH_DIALOG_OPEN_ATOM);
 
     const handleClose = useEventCallback(() => {
         setIsOpen(false);
     });
 
-    useSearchTrigger(setIsOpen);
-
     if (settings == null) {
         return null;
     }
diff --git a/packages/ui/app/src/search/inkeep/useInkeepSettings.ts b/packages/ui/app/src/search/inkeep/useInkeepSettings.ts
index 683e52d2a4..d22d08c066 100644
--- a/packages/ui/app/src/search/inkeep/useInkeepSettings.ts
+++ b/packages/ui/app/src/search/inkeep/useInkeepSettings.ts
@@ -19,7 +19,7 @@ const useInkeepSettings = ():
     const { resolvedTheme: theme } = useTheme();
     const [searchConfig] = useSearchConfig();
 
-    if (!searchConfig.isAvailable || searchConfig.type !== "inkeep") {
+    if (!searchConfig.isAvailable || searchConfig.inkeep == null) {
         return;
     }
 
@@ -37,15 +37,15 @@ const useInkeepSettings = ():
                 },
             },
         },
-        ...searchConfig.baseSettings,
+        ...searchConfig.inkeep.baseSettings,
     };
 
     return {
         // cast readonly -> mutable types, since inkeep widget expects mutable types
         baseSettings: baseSettings as InkeepWidgetBaseSettings,
-        aiChatSettings: searchConfig.aiChatSettings as InkeepAIChatSettings | undefined,
-        searchSettings: searchConfig.searchSettings as InkeepSearchSettings | undefined,
-        modalSettings: searchConfig.modalSettings as InkeepModalSettings | undefined,
+        aiChatSettings: searchConfig.inkeep.aiChatSettings as InkeepAIChatSettings | undefined,
+        searchSettings: searchConfig.inkeep.searchSettings as InkeepSearchSettings | undefined,
+        modalSettings: searchConfig.inkeep.modalSettings as InkeepModalSettings | undefined,
     };
 };
 
diff --git a/packages/ui/app/src/sidebar/SidebarSearchBar.tsx b/packages/ui/app/src/sidebar/SidebarSearchBar.tsx
index 622b94e8d2..e1c850ce6b 100644
--- a/packages/ui/app/src/sidebar/SidebarSearchBar.tsx
+++ b/packages/ui/app/src/sidebar/SidebarSearchBar.tsx
@@ -1,10 +1,7 @@
-import { useEventCallback } from "@fern-ui/react-commons";
 import { MagnifyingGlassIcon } from "@radix-ui/react-icons";
 import cn from "clsx";
-import { useSetAtom } from "jotai";
 import { memo } from "react";
 import { useOpenSearchDialog } from "../atoms/sidebar";
-import { INKEEP_TRIGGER } from "../search/inkeep/InkeepCustomTrigger";
 import { useSearchConfig } from "../services/useSearchService";
 
 export declare namespace SidebarSearchBar {
@@ -19,20 +16,11 @@ export const SidebarSearchBar: React.FC<SidebarSearchBar.Props> = memo(function
     hideKeyboardShortcutHint,
 }) {
     const openSearchDialog = useOpenSearchDialog();
-    const openInkeepCustomTrigger = useSetAtom(INKEEP_TRIGGER);
     const [searchService] = useSearchConfig();
 
-    const handleClick = useEventCallback(() => {
-        if (searchService.isAvailable && searchService.type === "inkeep") {
-            openInkeepCustomTrigger(true);
-        } else {
-            openSearchDialog();
-        }
-    });
-
     return (
         <button
-            onClick={handleClick}
+            onClick={openSearchDialog}
             className={cn("fern-search-bar", className)}
             disabled={!searchService.isAvailable}
         >
diff --git a/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts b/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts
index 2d081a1b53..72f5d51ebe 100644
--- a/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts
+++ b/servers/fdr/src/services/algolia/AlgoliaSearchRecordGenerator.ts
@@ -11,6 +11,7 @@ import { APIV1Db, APIV1Read, DocsV1Db } from "../../api";
 import { LOGGER } from "../../app/FdrApplication";
 import { assertNever, convertMarkdownToText, truncateToBytes } from "../../util";
 import { compact } from "../../util/object";
+import { ReferencedTypes, getAllReferencedTypes } from "./getAllReferencedTypes";
 import type { AlgoliaSearchRecord, IndexSegment } from "./types";
 
 class NavigationContext {
@@ -323,12 +324,113 @@ export class AlgoliaSearchRecordGenerator {
                             LOGGER.error("Failed to find endpoint for API reference node.", node);
                             return;
                         }
+
+                        // this is a hack to include the endpoint request/response json in the search index
+                        // and potentially use it for conversational AI in the future.
+                        // this needs to be rewritten as a template, with proper markdown formatting + snapshot testing.
+                        // also, the content is potentially trimmed to 10kb.
+                        const contents = [endpoint.description ?? ""];
+
+                        const typeReferences: APIV1Read.TypeReference[] = [];
+
+                        if (endpoint.headers != null && endpoint.headers.length > 0) {
+                            contents.push("## Headers\n");
+                            contents.push("```json");
+                            contents.push(JSON.stringify(endpoint.headers));
+                            contents.push("```\n");
+                            endpoint.headers.forEach((header) => {
+                                typeReferences.push(header.type);
+                            });
+                        }
+
+                        if (endpoint.path.pathParameters.length > 0) {
+                            contents.push("## Path Parameters\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(endpoint.path.pathParameters));
+                            contents.push("\n```\n");
+                            endpoint.path.pathParameters.forEach((param) => {
+                                typeReferences.push(param.type);
+                            });
+                        }
+
+                        if (endpoint.queryParameters.length > 0) {
+                            contents.push("## Query Parameters\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(endpoint.queryParameters));
+                            contents.push("\n```\n");
+                            endpoint.queryParameters.forEach((param) => {
+                                typeReferences.push(param.type);
+                            });
+                        }
+
+                        if (endpoint.request != null) {
+                            contents.push("## Request\n");
+                            if (endpoint.request.description != null) {
+                                contents.push(`${endpoint.request.description}\n`);
+                            }
+
+                            contents.push("### Body\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(endpoint.request.type));
+                            contents.push("\n```\n");
+
+                            if (endpoint.request.type.type === "reference") {
+                                typeReferences.push(endpoint.request.type.value);
+                            } else if (endpoint.request.type.type === "formData") {
+                                endpoint.request.type.properties.forEach((property) => {
+                                    if (property.type === "bodyProperty") {
+                                        typeReferences.push(property.valueType);
+                                    }
+                                });
+                            } else if (endpoint.request.type.type === "object") {
+                                endpoint.request.type.properties.forEach((property) => {
+                                    typeReferences.push(property.valueType);
+                                });
+                            }
+                        }
+
+                        if (endpoint.response != null) {
+                            contents.push("## Response\n");
+                            if (endpoint.response.description != null) {
+                                contents.push(`${endpoint.response.description}\n`);
+                            }
+
+                            contents.push("### Body\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(endpoint.response.type));
+                            contents.push("\n```\n");
+
+                            if (endpoint.response.type.type === "reference") {
+                                typeReferences.push(endpoint.response.type.value);
+                            } else if (endpoint.response.type.type === "object") {
+                                endpoint.response.type.properties.forEach((property) => {
+                                    typeReferences.push(property.valueType);
+                                });
+                            }
+                        }
+
+                        let referencedTypes: ReferencedTypes = {};
+
+                        typeReferences.forEach((typeReference) => {
+                            referencedTypes = {
+                                ...referencedTypes,
+                                ...getAllReferencedTypes({ reference: typeReference, types: holder?.api.types ?? {} }),
+                            };
+                        });
+
+                        if (Object.keys(referencedTypes).length > 0) {
+                            contents.push("## Referenced Types\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(referencedTypes));
+                            contents.push("\n```\n");
+                        }
+
                         records.push(
                             compact({
                                 type: "endpoint-v3",
                                 objectID: uuid(),
                                 title: node.title,
-                                content: endpoint.description,
+                                content: truncateToBytes(contents.join("\n"), 10_000 - 1),
                                 breadcrumbs: toBreadcrumbs(parents),
                                 slug: node.slug,
                                 version,
@@ -345,12 +447,84 @@ export class AlgoliaSearchRecordGenerator {
                             LOGGER.error("Failed to find websocket for API reference node.", node);
                             return;
                         }
+
+                        const contents = [ws.description ?? ""];
+                        const typeReferences: APIV1Read.TypeReference[] = [];
+
+                        if (ws.headers.length > 0) {
+                            contents.push("## Headers\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(ws.headers));
+                            contents.push("\n```\n");
+                            ws.headers.forEach((header) => {
+                                typeReferences.push(header.type);
+                            });
+                        }
+
+                        if (ws.path.pathParameters.length > 0) {
+                            contents.push("## Path Parameters\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(ws.path.pathParameters));
+                            contents.push("\n```\n");
+                            ws.path.pathParameters.forEach((param) => {
+                                typeReferences.push(param.type);
+                            });
+                        }
+
+                        if (ws.queryParameters.length > 0) {
+                            contents.push("## Query Parameters\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(ws.queryParameters));
+                            contents.push("\n```\n");
+                            ws.queryParameters.forEach((param) => {
+                                typeReferences.push(param.type);
+                            });
+                        }
+
+                        if (ws.messages.length > 0) {
+                            contents.push("## Messages\n");
+                            ws.messages.forEach((message) => {
+                                contents.push(`### ${message.displayName} (${message.type}) - ${message.origin}\n`);
+                                if (message.description != null) {
+                                    contents.push(message.description);
+                                }
+                                contents.push("```json\n");
+                                contents.push(JSON.stringify(message.body));
+                                contents.push("\n```\n");
+                                if (message.body.type === "reference") {
+                                    typeReferences.push(message.body.value);
+                                } else if (message.body.type === "object") {
+                                    message.body.properties.forEach((property) => {
+                                        typeReferences.push(property.valueType);
+                                    });
+                                } else {
+                                    assertNever(message.body);
+                                }
+                            });
+                        }
+
+                        let referencedTypes: ReferencedTypes = {};
+
+                        typeReferences.forEach((typeReference) => {
+                            referencedTypes = {
+                                ...referencedTypes,
+                                ...getAllReferencedTypes({ reference: typeReference, types: holder?.api.types ?? {} }),
+                            };
+                        });
+
+                        if (Object.keys(referencedTypes).length > 0) {
+                            contents.push("## Referenced Types\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(referencedTypes));
+                            contents.push("\n```\n");
+                        }
+
                         records.push(
                             compact({
                                 type: "websocket-v3",
                                 objectID: uuid(),
                                 title: node.title,
-                                content: ws.description,
+                                content: truncateToBytes(contents.join("\n"), 10_000 - 1),
                                 breadcrumbs: toBreadcrumbs(parents),
                                 slug: node.slug,
                                 version,
@@ -365,6 +539,56 @@ export class AlgoliaSearchRecordGenerator {
                             LOGGER.error("Failed to find webhook for API reference node.", node);
                             return;
                         }
+
+                        const contents = [webhook.description ?? ""];
+                        const typeReferences: APIV1Read.TypeReference[] = [];
+
+                        if (webhook.headers.length > 0) {
+                            contents.push("## Headers\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(webhook.headers));
+                            contents.push("\n```\n");
+                            webhook.headers.forEach((header) => {
+                                typeReferences.push(header.type);
+                            });
+                        }
+
+                        contents.push("## Payload\n");
+
+                        if (webhook.payload.description != null) {
+                            contents.push(webhook.payload.description);
+                        }
+
+                        contents.push("```json\n");
+                        contents.push(JSON.stringify(webhook.payload.type));
+                        contents.push("\n```\n");
+
+                        if (webhook.payload.type.type === "reference") {
+                            typeReferences.push(webhook.payload.type.value);
+                        } else if (webhook.payload.type.type === "object") {
+                            webhook.payload.type.properties.forEach((property) => {
+                                typeReferences.push(property.valueType);
+                            });
+                        } else {
+                            assertNever(webhook.payload.type);
+                        }
+
+                        let referencedTypes: ReferencedTypes = {};
+
+                        typeReferences.forEach((typeReference) => {
+                            referencedTypes = {
+                                ...referencedTypes,
+                                ...getAllReferencedTypes({ reference: typeReference, types: holder?.api.types ?? {} }),
+                            };
+                        });
+
+                        if (Object.keys(referencedTypes).length > 0) {
+                            contents.push("## Referenced Types\n");
+                            contents.push("```json\n");
+                            contents.push(JSON.stringify(referencedTypes));
+                            contents.push("\n```\n");
+                        }
+
                         records.push(
                             compact({
                                 type: "webhook-v3",
@@ -398,7 +622,7 @@ export class AlgoliaSearchRecordGenerator {
                         type: "page-v3",
                         objectID: uuid(),
                         title: node.title,
-                        content: md,
+                        content: truncateToBytes(md, 10_000 - 1),
                         breadcrumbs: toBreadcrumbs(parents),
                         slug: node.slug,
                         version,
diff --git a/servers/fdr/src/services/algolia/getAllReferencedTypes.ts b/servers/fdr/src/services/algolia/getAllReferencedTypes.ts
new file mode 100644
index 0000000000..6159412e9e
--- /dev/null
+++ b/servers/fdr/src/services/algolia/getAllReferencedTypes.ts
@@ -0,0 +1,135 @@
+import { APIV1Read } from "@fern-api/fdr-sdk";
+import { assertNever } from "../../util";
+
+export type ReferencedTypes = Record<APIV1Read.TypeId, APIV1Read.TypeDefinition>;
+
+export function getAllReferencedTypes({
+    reference,
+    types,
+}: {
+    reference: APIV1Read.TypeReference;
+    types: Record<APIV1Read.TypeId, APIV1Read.TypeDefinition>;
+}): ReferencedTypes {
+    const visitedTypes = new Set<APIV1Read.TypeId>();
+    function getAllReferencedTypesFromReference({
+        reference,
+        results,
+    }: {
+        reference: APIV1Read.TypeReference;
+        results: ReferencedTypes;
+    }): ReferencedTypes {
+        switch (reference.type) {
+            case "id": {
+                if (visitedTypes.has(reference.value)) {
+                    break;
+                } else {
+                    visitedTypes.add(reference.value);
+                }
+                const type = types[reference.value];
+                if (type != null) {
+                    return {
+                        ...results,
+                        [reference.value]: type,
+                        ...getAllReferencedTypesFromDefinition({ definition: type, results }),
+                    };
+                }
+                break;
+            }
+            case "map":
+                return {
+                    ...results,
+                    ...getAllReferencedTypesFromReference({ reference: reference.keyType, results }),
+                    ...getAllReferencedTypesFromReference({ reference: reference.valueType, results }),
+                };
+            case "list":
+                return {
+                    ...results,
+                    ...getAllReferencedTypesFromReference({ reference: reference.itemType, results }),
+                };
+            case "set":
+                return {
+                    ...results,
+                    ...getAllReferencedTypesFromReference({ reference: reference.itemType, results }),
+                };
+            case "optional":
+                return {
+                    ...results,
+                    ...getAllReferencedTypesFromReference({ reference: reference.itemType, results }),
+                };
+            case "literal":
+            case "primitive":
+            case "unknown":
+                return results;
+            default:
+                assertNever(reference);
+        }
+        return results;
+    }
+
+    function getAllReferencedTypesFromDefinition({
+        definition,
+        results,
+    }: {
+        definition: APIV1Read.TypeDefinition;
+        results: ReferencedTypes;
+    }): ReferencedTypes {
+        switch (definition.shape.type) {
+            case "object": {
+                return {
+                    ...results,
+                    ...definition.shape.extends.reduce((base, value) => {
+                        return {
+                            ...base,
+                            ...getAllReferencedTypesFromReference({
+                                reference: { type: "id", value },
+                                results,
+                            }),
+                        };
+                    }, {} as ReferencedTypes),
+                    ...definition.shape.properties.reduce((base, value) => {
+                        return {
+                            ...base,
+                            ...getAllReferencedTypesFromReference({ reference: value.valueType, results }),
+                        };
+                    }, {} as ReferencedTypes),
+                };
+            }
+            case "alias":
+                return results;
+            case "discriminatedUnion":
+                return {
+                    ...results,
+                    ...definition.shape.variants.reduce((base, value) => {
+                        return {
+                            ...base,
+                            ...getAllReferencedTypesFromDefinition({
+                                definition: {
+                                    name: value.discriminantValue,
+                                    shape: { type: "object", ...value.additionalProperties },
+                                },
+                                results,
+                            }),
+                        };
+                    }, {} as ReferencedTypes),
+                };
+            case "enum":
+                return results;
+            case "undiscriminatedUnion":
+                return {
+                    ...results,
+                    ...definition.shape.variants.reduce((base, value) => {
+                        return {
+                            ...base,
+                            ...getAllReferencedTypesFromReference({
+                                reference: value.type,
+                                results,
+                            }),
+                        };
+                    }, {} as ReferencedTypes),
+                };
+            default:
+                assertNever(definition.shape);
+        }
+    }
+    return getAllReferencedTypesFromReference({ reference, results: {} });
+}