Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: llms-txt with code snippets #1835

Merged
merged 2 commits into from
Nov 19, 2024
Merged

fix: llms-txt with code snippets #1835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5c281dd
fix: llms-txt with code snippets
abvthecity Nov 19, 2024
992fa1d
fix references
abvthecity Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 19 additions & 39 deletions packages/ui/docs-bundle/src/pages/api/fern-docs/llms-full.txt.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import { DocsLoader } from "@/server/DocsLoader";
import { getMarkdownForPath } from "@/server/getMarkdownForPath";
import { getSectionRoot } from "@/server/getSectionRoot";
import { getStringParam } from "@/server/getStringParam";
import { convertToLlmTxtMarkdown } from "@/server/llm-txt-md";
import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node";
import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers";
import { isNonNullish } from "@fern-api/ui-core-utils";
import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config";
import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils";
import { uniqWith } from "es-toolkit/array";
import { uniqBy } from "es-toolkit/array";
import { NextApiRequest, NextApiResponse } from "next";

/**
Expand All @@ -32,19 +33,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const domain = getDocsDomainNode(req);
const host = getHostNode(req) ?? domain;
const fern_token = req.cookies[COOKIE_FERN_TOKEN];
const loader = DocsLoader.for(domain, host, fern_token);
const featureFlags = await getFeatureFlags(domain);
const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags);

const root = getSectionRoot(await loader.root(), path);
const pages = await loader.pages();

if (root == null) {
return res.status(404).end();
}

const pageInfos: {
pageId: FernNavigation.PageId;
nodeTitle: string;
}[] = [];
const nodes: FernNavigation.NavigationNodePage[] = [];

// traverse the tree in a depth-first manner to collect all the nodes that have markdown content
// in the order that they appear in the sidebar
Expand All @@ -57,42 +55,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
}

if (FernNavigation.hasMarkdown(node)) {
// if the node is noindexed, don't include it in the list
// TODO: include "noindexed" nodes in `llms-full.txt`
if (node.noindex) {
return SKIP;
}

const pageId = FernNavigation.getPageId(node);
if (pageId != null) {
pageInfos.push({
pageId,
nodeTitle: node.title,
});
}
}

if (FernNavigation.isApiLeaf(node)) {
// TODO: construct a markdown-compatible page for the API reference
if (FernNavigation.isPage(node)) {
nodes.push(node);
}

return CONTINUE;
});

const markdowns = uniqWith(pageInfos, (a, b) => a.pageId === b.pageId)
.map((pageInfo) => {
const page = pages[pageInfo.pageId];
if (page == null) {
return undefined;
}
return convertToLlmTxtMarkdown(
page.markdown,
pageInfo.nodeTitle,
pageInfo.pageId.endsWith(".mdx") ? "mdx" : "md",
);
})
.filter(isNonNullish);
const markdowns = (
await Promise.all(
uniqBy(nodes, (a) => FernNavigation.getPageId(a) ?? a.canonicalSlug ?? a.slug).map(async (node) => {
const markdown = await getMarkdownForPath(node, loader, featureFlags);
if (markdown == null) {
return undefined;
}
return markdown.content;
}),
)
).filter(isNonNullish);

if (markdowns.length === 0) {
return res.status(404).end();
Expand Down
31 changes: 15 additions & 16 deletions packages/ui/docs-bundle/src/pages/api/fern-docs/llms.txt.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import { DocsLoader } from "@/server/DocsLoader";
import { addLeadingSlash } from "@/server/addLeadingSlash";
import { getMarkdownForPath } from "@/server/getMarkdownForPath";
import { getSectionRoot } from "@/server/getSectionRoot";
import { getStringParam } from "@/server/getStringParam";
import { convertToLlmTxtMarkdown, getLlmTxtMetadata } from "@/server/llm-txt-md";
import { getLlmTxtMetadata } from "@/server/llm-txt-md";
import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node";
import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers";
import { isNonNullish, withDefaultProtocol } from "@fern-api/ui-core-utils";
import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config";
import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils";
import { NextApiRequest, NextApiResponse } from "next";

Expand Down Expand Up @@ -44,7 +46,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const domain = getDocsDomainNode(req);
const host = getHostNode(req) ?? domain;
const fern_token = req.cookies[COOKIE_FERN_TOKEN];
const loader = DocsLoader.for(domain, host, fern_token);
const featureFlags = await getFeatureFlags(domain);
const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags);

const root = getSectionRoot(await loader.root(), path);
const pages = await loader.pages();
Expand All @@ -70,16 +73,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}[] = [];

const landingPage = getLandingPage(root);
const landingPageId = landingPage != null ? FernNavigation.getPageId(landingPage) : undefined;
const landingPageRawMarkdown = landingPageId != null ? pages[landingPageId]?.markdown : undefined;
const landingPageLlmTxtMarkdown =
landingPageRawMarkdown != null
? convertToLlmTxtMarkdown(
landingPageRawMarkdown,
landingPage?.title ?? root.title,
landingPageId?.endsWith(".mdx") ? "mdx" : "md",
)
: undefined;
const markdown = landingPage != null ? await getMarkdownForPath(landingPage, loader, featureFlags) : undefined;

// traverse the tree in a depth-first manner to collect all the nodes that have markdown content
// in the order that they appear in the sidebar
Expand Down Expand Up @@ -173,7 +167,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
.map((endpointPageInfo) => {
return {
title: endpointPageInfo.nodeTitle,
href: String(new URL(addLeadingSlash(endpointPageInfo.slug), withDefaultProtocol(domain))),
href: String(
new URL(
addLeadingSlash(endpointPageInfo.slug) + (endpointPageInfo.endpointId != null ? ".mdx" : ""),
withDefaultProtocol(domain),
),
),
breadcrumb: endpointPageInfo.breadcrumb,
};
})
Expand All @@ -188,7 +187,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
.send(
[
// if there's a landing page, use the llm-friendly markdown version instead of the ${root.title}
landingPageLlmTxtMarkdown ?? `# ${root.title}`,
markdown?.content ?? `# ${root.title}`,
docs.length > 0 ? `## Docs\n\n${docs.join("\n")}` : undefined,
endpoints.length > 0 ? `## API Docs\n\n${endpoints.join("\n")}` : undefined,
]
Expand All @@ -201,7 +200,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)

function getLandingPage(
root: FernNavigation.NavigationNodeWithMetadata,
): FernNavigation.LandingPageNode | FernNavigation.NavigationNodeWithMarkdown | undefined {
): FernNavigation.LandingPageNode | FernNavigation.NavigationNodePage | undefined {
if (root.type === "version") {
return root.landingPage;
} else if (root.type === "root") {
Expand All @@ -213,7 +212,7 @@ function getLandingPage(
}
}

if (FernNavigation.hasMarkdown(root)) {
if (FernNavigation.isPage(root)) {
return root;
}

Expand Down
52 changes: 10 additions & 42 deletions packages/ui/docs-bundle/src/pages/api/fern-docs/markdown.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import { DocsLoader } from "@/server/DocsLoader";
import { getMarkdownForPath, getPageNodeForPath } from "@/server/getMarkdownForPath";
import { getStringParam } from "@/server/getStringParam";
import { convertToLlmTxtMarkdown } from "@/server/llm-txt-md";
import { removeLeadingSlash } from "@/server/removeLeadingSlash";
import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node";
import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
import { getFeatureFlags } from "@fern-ui/fern-docs-edge-config";
import { COOKIE_FERN_TOKEN } from "@fern-ui/fern-docs-utils";
import { NextApiRequest, NextApiResponse } from "next";

Expand All @@ -29,57 +28,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const domain = getDocsDomainNode(req);
const host = getHostNode(req) ?? domain;
const fern_token = req.cookies[COOKIE_FERN_TOKEN];
const loader = DocsLoader.for(domain, host, fern_token);
const featureFlags = await getFeatureFlags(domain);
const loader = DocsLoader.for(domain, host, fern_token).withFeatureFlags(featureFlags);

const root = await loader.root();
const pages = await loader.pages();

const pageInfo = getPageInfo(root, FernNavigation.Slug(removeLeadingSlash(path)));

// TODO: add support for api reference endpoint pages
if (pageInfo == null) {
const node = getPageNodeForPath(await loader.root(), path);
if (node == null) {
return res.status(404).end();
}

const page = pages[pageInfo.pageId];

if (!page) {
const markdown = await getMarkdownForPath(node, loader, featureFlags);
if (markdown == null) {
return res.status(404).end();
}

res.status(200)
.setHeader("Content-Type", `text/${pageInfo.pageId.endsWith(".mdx") ? "mdx" : "markdown"}`)
.setHeader("Content-Type", `text/${markdown.contentType}`)
// prevent search engines from indexing this page
.setHeader("X-Robots-Tag", "noindex")
// cannot guarantee that the content won't change, so we only cache for 60 seconds
.setHeader("Cache-Control", "s-maxage=60")
.send(
convertToLlmTxtMarkdown(page.markdown, pageInfo.nodeTitle, pageInfo.pageId.endsWith(".mdx") ? "mdx" : "md"),
);
.send(markdown.content);

return;
}

function getPageInfo(
root: FernNavigation.RootNode | undefined,
slug: FernNavigation.Slug,
): { pageId: FernNavigation.PageId; nodeTitle: string } | undefined {
if (root == null) {
return undefined;
}

const foundNode = FernNavigation.utils.findNode(root, slug);
if (foundNode == null || foundNode.type !== "found" || !FernNavigation.hasMarkdown(foundNode.node)) {
return undefined;
}

const pageId = FernNavigation.getPageId(foundNode.node);
if (pageId == null) {
return undefined;
}

return {
pageId,
nodeTitle: foundNode.node.title,
};
}
28 changes: 28 additions & 0 deletions packages/ui/docs-bundle/src/server/DocsLoader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
import type { DocsV1Read, DocsV2Read } from "@fern-api/fdr-sdk";
import { ApiDefinition, ApiDefinitionV1ToLatest } from "@fern-api/fdr-sdk/api-definition";
import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
import type { AuthEdgeConfig } from "@fern-ui/fern-docs-auth";
import { getAuthEdgeConfig } from "@fern-ui/fern-docs-edge-config";
import { ApiDefinitionLoader } from "@fern-ui/fern-docs-server";
import { getAuthState, type AuthState } from "./auth/getAuthState";
import { loadWithUrl } from "./loadWithUrl";
import { pruneWithAuthState } from "./withRbac";

interface DocsLoaderFlags {
isBatchStreamToggleDisabled: boolean;
isApiScrollingDisabled: boolean;

// for api definition:
useJavaScriptAsTypeScript: boolean;
alwaysEnableJavaScriptFetch: boolean;
usesApplicationJsonInFormDataValue: boolean;
}

export class DocsLoader {
Expand All @@ -25,6 +32,9 @@ export class DocsLoader {
private featureFlags: DocsLoaderFlags = {
isBatchStreamToggleDisabled: false,
isApiScrollingDisabled: false,
useJavaScriptAsTypeScript: false,
alwaysEnableJavaScriptFetch: false,
usesApplicationJsonInFormDataValue: false,
};
public withFeatureFlags(featureFlags: DocsLoaderFlags): DocsLoader {
this.featureFlags = featureFlags;
Expand Down Expand Up @@ -66,6 +76,24 @@ export class DocsLoader {
return this;
}

public async getApiDefinition(key: FernNavigation.ApiDefinitionId): Promise<ApiDefinition | undefined> {
const res = await this.loadDocs();
if (!res) {
return undefined;
}
const v1 = res.definition.apis[key];
if (!v1) {
return undefined;
}
const latest = ApiDefinitionV1ToLatest.from(v1, this.featureFlags).migrate();
return ApiDefinitionLoader.create(this.domain, key)
.withApiDefinition(latest)
.withFlags(this.featureFlags)
.withResolveDescriptions(false)
.withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN)
.load();
}

private async loadDocs(): Promise<DocsV2Read.LoadDocsForUrlResponse | undefined> {
if (!this.#loadForDocsUrlResponse) {
const response = await loadWithUrl(this.domain);
Expand Down
Loading
Loading