fix: revert to prior behavior where hidden pages are indexable

packages/fdr-sdk/src/__test__/fixtures.test.ts

@@ -101,7 +101,6 @@ function testNavigationConfigConverter(fixtureName: string): void {
                     console.log(node);
                 }
                 expect(FernNavigation.isPage(node), `${slug} is a page`).toBe(true);
-                expect(node.hidden, `${slug} is not hidden`).not.toBe(true);
 
                 if (FernNavigation.hasMarkdown(node)) {
                     expect(node.noindex, `${slug} is indexable`).not.toBe(true);

packages/fdr-sdk/src/__test__/output/humanloop/slugs-sitemap.json

@@ -226,6 +226,7 @@
   "docs/v5/changelog/2023/2/20",
   "docs/v4/tutorials/create-your-first-gpt-4-app",
   "docs/v4/tutorials/chatgpt-clone-in-nextjs",
+  "docs/v4/tutorials/debug-a-langchain-agent",
   "docs/v4/guides/create-prompt",
   "docs/v4/guides/generate-and-log-with-the-sdk",
   "docs/v4/guides/completion-using-the-sdk",

packages/fdr-sdk/src/navigation/NodeCollector.ts

@@ -175,13 +175,15 @@ export class NodeCollector {
     /**
      * Returns a list of slugs for pages that should be indexed by search engines, and by algolia.
      *
-     * This excludes hidden pages and noindex pages, and uses the canonical slug if it exists.
+     * This excludes noindex pages, and uses the canonical slug if it exists.
+     *
+     * IMPORTANT: hidden pages will be included, unless explicitly marked as noindex.
      */
     #getIndexablePageSlugs = once((): string[] => {
         return Array.from(
             new Set(
                 [...this.slugToNode.values()]
-                    .filter(({ node }) => FernNavigation.isPage(node) && !node.hidden)
+                    .filter(({ node }) => FernNavigation.isPage(node))
                     .filter(({ node }) => (FernNavigation.hasMarkdown(node) ? !node.noindex : true))
                     .map(({ node }) => node.canonicalSlug ?? node.slug),
             ),

packages/ui/app/src/seo/__test__/getSeoProps.test.ts

@@ -1,38 +1,97 @@
-import * as FernNavigation from "@fern-api/fdr-sdk/navigation";
-import { DefinitionObjectFactory } from "@fern-ui/fdr-utils";
+import { FernNavigation } from "@fern-api/fdr-sdk";
 import { extractHeadline, getSeoProps, stripMarkdown } from "../getSeoProp";
 
 describe("getSeoProps", () => {
     it("seo disabled", () => {
-        const props = getSeoProps(
-            "host",
-            DefinitionObjectFactory.createDocsDefinition().config,
-            {},
-            {},
-            {},
-            {
-                node: {
-                    id: FernNavigation.NodeId("id"),
-                    type: "page",
-                    pageId: FernNavigation.PageId("pageId"),
-                    title: "page",
-                    slug: FernNavigation.Slug("slug"),
-                    canonicalSlug: undefined,
-                    icon: undefined,
-                    hidden: false,
-                    noindex: undefined,
-                },
-                parents: [],
-                currentVersion: undefined,
-                root: { slug: FernNavigation.Slug("") } as FernNavigation.RootNode,
+        const props = getSeoProps({
+            domain: "https://plantstore.dev",
+            siteName: "Plant Store",
+            metadata: undefined,
+            favicon: undefined,
+            typography: undefined,
+            pages: {},
+            files: {},
+            apis: {},
+            isSeoDisabled: true,
+            isTrailingSlashEnabled: false,
+            node: {
+                id: FernNavigation.NodeId("id"),
+                type: "page",
+                pageId: FernNavigation.PageId("pageId"),
+                title: "page",
+                slug: FernNavigation.Slug("slug"),
+                canonicalSlug: undefined,
+                icon: undefined,
+                hidden: false,
+                noindex: undefined,
             },
-            true,
-            false,
-        );
+            parents: [],
+            version: undefined,
+        });
         expect(props.noindex).toBe(true);
         expect(props.nofollow).toBe(true);
     });
 
+    it("seo enabled on hidden page", () => {
+        const props = getSeoProps({
+            domain: "https://plantstore.dev",
+            siteName: "Plant Store",
+            metadata: undefined,
+            favicon: undefined,
+            typography: undefined,
+            pages: {},
+            files: {},
+            apis: {},
+            isSeoDisabled: false,
+            isTrailingSlashEnabled: false,
+            node: {
+                id: FernNavigation.NodeId("id"),
+                type: "page",
+                pageId: FernNavigation.PageId("pageId"),
+                title: "page",
+                slug: FernNavigation.Slug("slug"),
+                canonicalSlug: undefined,
+                icon: undefined,
+                hidden: true,
+                noindex: undefined,
+            },
+            parents: [],
+            version: undefined,
+        });
+        expect(props.noindex).toBe(false);
+        expect(props.nofollow).toBe(false);
+    });
+
+    it("seo disabled on hidden page with noindex=true", () => {
+        const props = getSeoProps({
+            domain: "https://plantstore.dev",
+            siteName: "Plant Store",
+            metadata: undefined,
+            favicon: undefined,
+            typography: undefined,
+            pages: {},
+            files: {},
+            apis: {},
+            isSeoDisabled: false,
+            isTrailingSlashEnabled: false,
+            node: {
+                id: FernNavigation.NodeId("id"),
+                type: "page",
+                pageId: FernNavigation.PageId("pageId"),
+                title: "page",
+                slug: FernNavigation.Slug("slug"),
+                canonicalSlug: undefined,
+                icon: undefined,
+                hidden: true,
+                noindex: true,
+            },
+            parents: [],
+            version: undefined,
+        });
+        expect(props.noindex).toBe(true);
+        expect(props.nofollow).toBe(false);
+    });
+
     it("extracts SEO title properly", () => {
         expect(stripMarkdown(extractHeadline("#"))).toBe("");
         expect(stripMarkdown(extractHeadline("# goodcase"))).toBe("goodcase");

packages/ui/app/src/seo/getSeoProp.ts

@@ -43,20 +43,57 @@ function getFile(fileOrUrl: DocsV1Read.FileIdOrUrl, files: Record<string, DocsV1
     });
 }
 
-export function getSeoProps(
-    domain: string,
-    { metadata, title, favicon, typographyV2: typography }: DocsV1Read.DocsConfig,
-    pages: Record<string, DocsV1Read.PageContent>,
-    files: Record<string, DocsV1Read.File_>,
-    apis: Record<string, APIV1Read.ApiDefinition>,
-    {
-        node,
-        parents,
-        currentVersion,
-    }: Pick<FernNavigation.utils.Node.Found, "node" | "parents" | "currentVersion" | "root">,
-    isSeoDisabled: boolean,
-    isTrailingSlashEnabled: boolean,
-): NextSeoProps {
+interface GetSeoPropsOptions {
+    /**
+     * This comes from headers["x-fern-host"] ?? headers.host
+     */
+    domain: string;
+
+    /**
+     * If set, it will add `- ${siteName}` to the end of the title template.
+     */
+    siteName: string | undefined;
+
+    /**
+     * SEO metadata options
+     */
+    metadata: DocsV1Read.MetadataConfig | undefined;
+
+    favicon: DocsV1Read.FileId | undefined;
+    typography: DocsV1Read.DocsTypographyConfigV2 | undefined;
+    pages: Record<DocsV1Read.PageId, DocsV1Read.PageContent>;
+    files: Record<DocsV1Read.FileId, DocsV1Read.File_>;
+    apis: Record<DocsV1Read.ApiDefinitionId, APIV1Read.ApiDefinition>;
+
+    /**
+     * Typically true if *.docs.buildwithfern.com or *.docs.dev.buildwithfern.com
+     */
+    isSeoDisabled: boolean;
+    isTrailingSlashEnabled: boolean;
+
+    /**
+     * Node and parent information
+     */
+    node: FernNavigation.NavigationNodePage;
+    parents: FernNavigation.NavigationNode[];
+    version: FernNavigation.VersionNode | undefined;
+}
+
+export function getSeoProps({
+    domain,
+    siteName,
+    metadata,
+    favicon,
+    typography,
+    pages,
+    files,
+    apis,
+    isSeoDisabled,
+    isTrailingSlashEnabled,
+    node,
+    parents,
+    version,
+}: GetSeoPropsOptions): NextSeoProps {
     const additionalMetaTags: MetaTag[] = [];
     const additionalLinkTags: LinkTag[] = [];
     const openGraph: NextSeoProps["openGraph"] = {};
@@ -212,7 +249,7 @@ export function getSeoProps(
 
     // defaults
     seo.title ??= node.title;
-    openGraph.siteName ??= title;
+    openGraph.siteName ??= siteName;
 
     /**
      * Disambiguate the title via the version title, if it exists and is not the default version.
@@ -221,15 +258,15 @@ export function getSeoProps(
      * v1:       Get Plants (v1) - Plant Store
      * v2:       Get Plants (v2) - Plant Store
      */
-    if (title != null && currentVersion != null && !currentVersion.default) {
-        seo.titleTemplate ??= `%s (${currentVersion.title}) — ${title}`;
+    if (siteName != null && version != null && !version.default) {
+        seo.titleTemplate ??= `%s (${version.title}) — ${siteName}`;
     }
 
     /**
      * Fallback title template: "Page Title — Site Title"
      */
-    if (title != null) {
-        seo.titleTemplate ??= `%s — ${title}`;
+    if (siteName != null) {
+        seo.titleTemplate ??= `%s — ${siteName}`;
     }
 
     if (favicon != null && files[favicon] != null) {
@@ -265,8 +302,9 @@ export function getSeoProps(
     seo.noindex = ogMetadata.noindex;
     seo.nofollow = ogMetadata.nofollow;
 
-    // do not index the page if it is hidden, or has noindex set, or if SEO is disabled
-    if ((FernNavigation.hasMarkdown(node) && node.noindex) || node.hidden || isSeoDisabled) {
+    // do not index the page if noindex set, or if SEO is disabled
+    // note: we allow indexing hidden pages unless explicitly set to noindex
+    if ((FernNavigation.hasMarkdown(node) && node.noindex) || isSeoDisabled) {
         seo.noindex = true;
     }
 

packages/ui/docs-bundle/src/server/disabledSeo.ts

@@ -1,3 +1,4 @@
+import { captureException } from "@sentry/nextjs";
 import { get } from "@vercel/edge-config";
 
 export async function getSeoDisabled(host: string): Promise<boolean> {
@@ -8,6 +9,11 @@ export async function getSeoDisabled(host: string): Promise<boolean> {
     ) {
         return true;
     }
-    const config = (await get<Array<string>>("seo-disabled")) ?? [];
-    return config.includes(host);
+    try {
+        const config = (await get<Array<string>>("seo-disabled")) ?? [];
+        return config.includes(host);
+    } catch (e) {
+        captureException(e);
+        return false;
+    }
 }

packages/ui/docs-bundle/src/server/withInitialProps.ts

@@ -210,16 +210,21 @@ export async function withInitialProps({
         },
         featureFlags,
         apis: Object.keys(docs.definition.apis).map(FernNavigation.ApiDefinitionId),
-        seo: getSeoProps(
-            docs.baseUrl.domain,
-            docs.definition.config,
-            docs.definition.pages,
-            docs.definition.filesV2,
-            docs.definition.apis,
-            node,
-            await getSeoDisabled(xFernHost),
-            isTrailingSlashEnabled(),
-        ),
+        seo: getSeoProps({
+            domain: docs.baseUrl.domain,
+            siteName: docs.definition.config.title,
+            metadata: docs.definition.config.metadata,
+            favicon: docs.definition.config.favicon,
+            typography: docs.definition.config.typographyV2,
+            pages: docs.definition.pages,
+            files: docs.definition.filesV2,
+            apis: docs.definition.apis,
+            isSeoDisabled: await getSeoDisabled(xFernHost),
+            isTrailingSlashEnabled: isTrailingSlashEnabled(),
+            node: node.node,
+            parents: node.parents,
+            version: node.currentVersion,
+        }),
         user: auth?.user,
         fallback: {},
         // eslint-disable-next-line deprecation/deprecation

packages/ui/local-preview-bundle/src/utils/getDocsPageProps.ts

@@ -164,16 +164,21 @@ export async function getDocsPageProps(
         },
         featureFlags,
         apis: Object.keys(docs.definition.apis).map(FdrAPI.ApiDefinitionId),
-        seo: getSeoProps(
-            docs.baseUrl.domain,
-            docs.definition.config,
-            docs.definition.pages,
-            docs.definition.filesV2,
-            docs.definition.apis,
-            node,
-            true,
-            false,
-        ),
+        seo: getSeoProps({
+            domain: docs.baseUrl.domain,
+            siteName: docs.definition.config.title,
+            metadata: docs.definition.config.metadata,
+            favicon: docs.definition.config.favicon,
+            typography: docs.definition.config.typographyV2,
+            pages: docs.definition.pages,
+            files: docs.definition.filesV2,
+            apis: docs.definition.apis,
+            isSeoDisabled: true,
+            isTrailingSlashEnabled: false,
+            node: node.node,
+            parents: node.parents,
+            version: node.currentVersion,
+        }),
         fallback: {},
         analytics: undefined,
         analyticsConfig: docs.definition.config.analyticsConfig,