refactor: replace next-mdx-remote with mdx-bundler (#1116)

clis/generator-cli/package.json

@@ -32,7 +32,7 @@
     "@types/yargs": "^17.0.32",
     "@yarnpkg/esbuild-plugin-pnp": "^3.0.0-rc.15",
     "depcheck": "^1.4.3",
-    "esbuild": "^0.15.7",
+    "esbuild": "0.20.2",
     "eslint": "^8.56.0",
     "execa": "^5.1.1",
     "lodash-es": "^4.17.12",

fern/apis/fdr/definition/docs/v1/db/__package__.yml

@@ -39,6 +39,11 @@ types:
       referencedApis: set<rootCommons.ApiDefinitionId>
       files: map<commons.FileId, DbFileInfoV2>
       config: DocsDbConfig
+      jsFiles:
+        type: optional<map<string, string>>
+        docs: |
+          A map of file names to their contents. 
+          The key is the absolute path file name and the value is the file contents.
 
   DocsDbConfig:
     properties:

fern/apis/fdr/definition/docs/v1/read/__package__.yml

@@ -68,6 +68,11 @@ types:
       apis: map<rootCommons.ApiDefinitionId, apiReadV1.ApiDefinition>
       files: map<commons.FileId, commons.Url>
       filesV2: map<commons.FileId, File>
+      jsFiles:
+        type: optional<map<string, string>>
+        docs: |
+          A map of file names to their contents. 
+          The key is the absolute path file name and the value is the file contents.
       id: optional<rootCommons.DocsConfigId>
       config: DocsConfig
       search: algolia.SearchInfo

fern/apis/fdr/definition/docs/v1/write/__package__.yml

@@ -68,6 +68,11 @@ types:
     properties:
       pages: map<commons.PageId, PageContent>
       config: DocsConfig
+      jsFiles:
+        type: optional<map<string, string>>
+        docs: |
+          A map of file names to their contents. 
+          The key is the absolute path file name and the value is the file contents.
 
   PageContent:
     properties:

packages/fdr-sdk/src/converters/db/convertDocsDefinitionToDb.ts

@@ -109,6 +109,7 @@ export function convertDocsDefinitionToDb({
             integrations: writeShape.config.integrations,
         },
         pages: writeShape.pages,
+        jsFiles: writeShape.jsFiles,
     };
 }
 

packages/fdr-sdk/src/converters/db/upgrade/migrateDocsDbDefinition.ts

@@ -1,9 +1,10 @@
 import { DocsV1Db } from "../../../client";
 import { isPlainObject } from "../../../utils";
+import { WithoutQuestionMarks } from "../../utils/WithoutQuestionMarks";
 import { upgradeV1ToV2 } from "./upgradeV1ToV2";
 import { upgradeV2ToV3 } from "./upgradeV2ToV3";
 
-export function migrateDocsDbDefinition(buffer: unknown): DocsV1Db.DocsDefinitionDb.V3 {
+export function migrateDocsDbDefinition(buffer: unknown): WithoutQuestionMarks<DocsV1Db.DocsDefinitionDb.V3> {
     if (!isPlainObject(buffer)) {
         throw new Error("Invalid buffer");
     }
@@ -20,5 +21,5 @@ export function migrateDocsDbDefinition(buffer: unknown): DocsV1Db.DocsDefinitio
         return upgradeV2ToV3(dbValue);
     }
 
-    return dbValue as DocsV1Db.DocsDefinitionDb.V3;
+    return dbValue as WithoutQuestionMarks<DocsV1Db.DocsDefinitionDb.V3>;
 }

packages/fdr-sdk/src/converters/db/upgrade/upgradeV2ToV3.ts

@@ -1,7 +1,8 @@
 import { DocsV1Db } from "../../../client";
 import { mapValues } from "../../../utils";
+import { WithoutQuestionMarks } from "../../utils/WithoutQuestionMarks";
 
-export function upgradeV2ToV3(v2: DocsV1Db.DocsDefinitionDb.V2): DocsV1Db.DocsDefinitionDb.V3 {
+export function upgradeV2ToV3(v2: DocsV1Db.DocsDefinitionDb.V2): WithoutQuestionMarks<DocsV1Db.DocsDefinitionDb.V3> {
     return {
         type: "v3",
         pages: v2.pages,
@@ -14,5 +15,6 @@ export function upgradeV2ToV3(v2: DocsV1Db.DocsDefinitionDb.V2): DocsV1Db.DocsDe
             }),
         ),
         config: v2.config,
+        jsFiles: undefined,
     };
 }

packages/healthchecks/package.json

@@ -37,11 +37,10 @@
     "@fern-ui/core-utils": "workspace:*",
     "@types/jest": "^29.5.11",
     "@types/node": "^18.7.18",
-    "@types/node-emoji": "^2.1.0",
     "@types/yargs": "^17.0.32",
     "@yarnpkg/esbuild-plugin-pnp": "^3.0.0-rc.15",
     "depcheck": "^1.4.3",
-    "esbuild": "^0.15.7",
+    "esbuild": "0.20.2",
     "eslint": "^8.56.0",
     "organize-imports-cli": "^0.10.0",
     "prettier": "^3.3.2",

packages/ui/app/.depcheckrc.json

@@ -23,7 +23,8 @@
     "@storybook/addon-onboarding",
     "@storybook/blocks",
     "@storybook/test",
-    "@emotion/is-prop-valid"
+    "@emotion/is-prop-valid",
+    "mdx"
   ],
   "ignore-patterns": ["dist"]
 }

packages/ui/app/package.json

@@ -14,6 +14,14 @@
   "source": "src/index.ts",
   "module": "src/index.ts",
   "main": "src/index.ts",
+  "exports": {
+    ".": {
+      "default": "./src/index.ts"
+    },
+    "./bundlers": {
+      "default": "./src/mdx/bundlers/index.ts"
+    }
+  },
   "sideEffects": [
     "*.css",
     "*.scss"
@@ -65,6 +73,7 @@
     "bezier-easing": "^2.1.0",
     "clsx": "^2.1.0",
     "colorjs.io": "^0.5.0",
+    "esbuild": "0.20.2",
     "estree-util-visit": "^2.0.0",
     "fastdom": "^1.0.12",
     "framer-motion": "^11.2.4",
@@ -86,6 +95,7 @@
     "mdast-util-from-markdown": "^2.0.0",
     "mdast-util-mdx-jsx": "^3.1.0",
     "mdast-util-to-hast": "^13.1.0",
+    "mdx-bundler": "^10.0.2",
     "moment": "^2.30.1",
     "next": "^14.2.3",
     "next-mdx-remote": "^5.0.0",
@@ -122,6 +132,7 @@
   "devDependencies": {
     "@chromatic-com/storybook": "^1.3.5",
     "@fern-platform/configs": "workspace:*",
+    "@mdx-js/esbuild": "^3.0.1",
     "@storybook/addon-essentials": "8.1.0-alpha.6",
     "@storybook/addon-interactions": "8.1.0-alpha.6",
     "@storybook/addon-links": "8.1.0-alpha.6",
@@ -141,6 +152,7 @@
     "@types/jsonpath": "^0.2.4",
     "@types/lodash-es": "^4.17.12",
     "@types/mdast": "^4.0.3",
+    "@types/mdx": "^2.0.13",
     "@types/node": "^18.7.18",
     "@types/numeral": "^2.0.5",
     "@types/react": "^18.0.20",

packages/ui/app/src/api-page/ApiPageDescription.tsx

@@ -1,6 +1,6 @@
 import cn from "clsx";
 import dynamic from "next/dynamic";
-import { SerializedMdxContent } from "../mdx/mdx";
+import type { BundledMDX } from "../mdx/types";
 
 const Markdown = dynamic(() => import("../mdx/Markdown").then(({ Markdown }) => Markdown), {
     ssr: true,
@@ -9,7 +9,7 @@ const Markdown = dynamic(() => import("../mdx/Markdown").then(({ Markdown }) =>
 export declare namespace ApiPageDescription {
     export interface Props {
         className?: string;
-        description: string | SerializedMdxContent | undefined;
+        description: BundledMDX | undefined;
         isMarkdown: boolean;
     }
 }

packages/ui/app/src/api-page/endpoints/EndpointParameter.tsx

@@ -1,9 +1,9 @@
 import { APIV1Read } from "@fern-api/fdr-sdk";
 import cn from "clsx";
-import type { MDXRemoteSerializeResult } from "next-mdx-remote";
 import { FC, PropsWithChildren, ReactNode, memo, useRef, useState } from "react";
 import { AbsolutelyPositionedAnchor } from "../../commons/AbsolutelyPositionedAnchor";
 import { useRouteListener } from "../../contexts/useRouteListener";
+import type { BundledMDX } from "../../mdx/types";
 import { ResolvedTypeDefinition, ResolvedTypeShape } from "../../resolver/types";
 import { getAnchorId } from "../../util/anchor";
 import { ApiPageDescription } from "../ApiPageDescription";
@@ -14,7 +14,7 @@ import { EndpointAvailabilityTag } from "./EndpointAvailabilityTag";
 export declare namespace EndpointParameter {
     export interface Props {
         name: string;
-        description: MDXRemoteSerializeResult | string | undefined;
+        description: BundledMDX | undefined;
         shape: ResolvedTypeShape;
         anchorIdParts: readonly string[];
         route: string;
@@ -24,7 +24,7 @@ export declare namespace EndpointParameter {
 
     export interface ContentProps {
         name: string;
-        description: MDXRemoteSerializeResult | string | undefined;
+        description: BundledMDX | undefined;
         typeShorthand: ReactNode;
         anchorIdParts: readonly string[];
         route: string;

packages/ui/app/src/api-page/endpoints/EndpointRequestSection.tsx

@@ -1,8 +1,8 @@
 import { APIV1Read } from "@fern-api/fdr-sdk";
 import { visitDiscriminatedUnion } from "@fern-ui/core-utils";
 import cn from "clsx";
-import { MDXRemoteSerializeResult } from "next-mdx-remote";
 import { Fragment, ReactNode } from "react";
+import type { BundledMDX } from "../../mdx/types";
 import {
     ResolvedFormDataRequestProperty,
     ResolvedRequestBody,
@@ -128,7 +128,7 @@ export const EndpointRequestSection: React.FC<EndpointRequestSection.Props> = ({
 function getDescription(
     bodyProperty: ResolvedFormDataRequestProperty.BodyProperty,
     types: Record<string, ResolvedTypeDefinition>,
-): string | MDXRemoteSerializeResult | undefined {
+): BundledMDX | undefined {
     if (bodyProperty.description != null) {
         return bodyProperty.description;
     }

packages/ui/app/src/api-page/endpoints/EndpointSection.tsx

@@ -1,10 +1,10 @@
 import { FernButton, FernButtonGroup } from "@fern-ui/components";
 import { MinusIcon, PlusIcon } from "@radix-ui/react-icons";
-import type { MDXRemoteSerializeResult } from "next-mdx-remote";
 import dynamic from "next/dynamic";
 import { ReactNode, createElement, useRef } from "react";
 import { AbsolutelyPositionedAnchor } from "../../commons/AbsolutelyPositionedAnchor";
 import { FernErrorBoundary } from "../../components/FernErrorBoundary";
+import type { BundledMDX } from "../../mdx/types";
 import { getAnchorId } from "../../util/anchor";
 
 const Markdown = dynamic(() => import("../../mdx/Markdown").then(({ Markdown }) => Markdown), {
@@ -15,7 +15,7 @@ export declare namespace EndpointSection {
     export type Props = React.PropsWithChildren<{
         headerType?: "h1" | "h2" | "h3" | "h4" | "h5" | "h6";
         title: ReactNode;
-        description?: MDXRemoteSerializeResult | string | undefined;
+        description?: BundledMDX | undefined;
         anchorIdParts: readonly string[];
         route: string;
         expandAll?: () => void;

packages/ui/app/src/atoms/flags.ts

@@ -16,6 +16,7 @@ export interface FeatureFlags {
     useJavaScriptAsTypeScript: boolean;
     alwaysEnableJavaScriptFetch: boolean;
     scrollInContainerEnabled: boolean;
+    useMdxBundler: boolean;
 }
 
 export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
@@ -33,6 +34,7 @@ export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
     useJavaScriptAsTypeScript: false,
     alwaysEnableJavaScriptFetch: false,
     scrollInContainerEnabled: false,
+    useMdxBundler: false,
 };
 
 export const FEATURE_FLAGS_ATOM = atom(DEFAULT_FEATURE_FLAGS);

packages/ui/app/src/atoms/navigation.ts

@@ -13,7 +13,7 @@ export const RESOLVED_PATH_ATOM = atomWithReducer<ResolvedPath, ResolvedPath>(
         type: "custom-markdown-page",
         fullSlug: "",
         title: "",
-        serializedMdxContent: "",
+        mdx: "",
         neighbors: { prev: null, next: null },
         apis: {},
     },

packages/ui/app/src/atoms/sidebar.ts

@@ -75,8 +75,8 @@ export const SIDEBAR_DISMISSABLE_ATOM = atom((get) => {
     }
 
     const resolvedPath = get(RESOLVED_PATH_ATOM);
-    if (resolvedPath.type === "custom-markdown-page" && typeof resolvedPath.serializedMdxContent !== "string") {
-        const layout = resolvedPath.serializedMdxContent.frontmatter.layout;
+    if (resolvedPath.type === "custom-markdown-page" && typeof resolvedPath.mdx !== "string") {
+        const layout = resolvedPath.mdx.frontmatter.layout;
 
         if (layout === "page" || layout === "custom") {
             return true;

packages/ui/app/src/components/Chip.tsx

@@ -1,13 +1,13 @@
 import { FernTooltip } from "@fern-ui/components";
 import { useCopyToClipboard } from "@fern-ui/react-commons";
 import cn from "clsx";
-import type { MDXRemoteSerializeResult } from "next-mdx-remote";
 import { ReactElement } from "react";
 import { Markdown } from "../mdx/Markdown";
+import type { BundledMDX } from "../mdx/types";
 
 type ChipProps = {
     name: string;
-    description?: MDXRemoteSerializeResult | string | undefined;
+    description?: BundledMDX | undefined;
     small?: boolean;
 };