From cd9ad374e392bace88ceef7af4da165a332e2f85 Mon Sep 17 00:00:00 2001
From: Augusto Herrmann <augusto+github@herrmann.tech>
Date: Thu, 13 Feb 2025 17:37:50 -0300
Subject: [PATCH] Add robots.txt file

fix #172
---
 src/api.py              | 23 ++++++++++++++++++++---
 tests/endpoints_test.py | 15 +++++++++++++++
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/src/api.py b/src/api.py
index 30aae62..00167f6 100644
--- a/src/api.py
+++ b/src/api.py
@@ -5,6 +5,7 @@
 from datetime import timedelta
 import json
 import os
+from textwrap import dedent
 from typing import Annotated, Awaitable, Callable, Union
 
 from fastapi import Depends, FastAPI, HTTPException, status, Header, Request, Response
@@ -118,9 +119,7 @@ async def check_user_agent(
 async def docs_redirect(
     accept: Union[str, None] = Header(default="text/html")
 ) -> RedirectResponse:
-    """
-    Redireciona para a documentação da API.
-    """
+    """Redireciona para a documentação da API."""
 
     if accept == "application/json":
         location = "/openapi.json"
@@ -132,6 +131,24 @@ async def docs_redirect(
     )
 
 
+@app.get("/robots.txt", include_in_schema=False)
+async def robots_txt() -> Response:
+    """Retorna um arquivo robots.txt para orientar crawlers e permitir
+    indexação somente dos caminhos de documentação.
+    """
+    return Response(
+        dedent(
+            """
+        User-agent: *
+        Allow: /docs$
+        Allow: /redoc$
+        Disallow: /
+        """
+        ).lstrip(),
+        media_type="text/plain",
+    )
+
+
 # ## AUTH --------------------------------------------------
 
 
diff --git a/tests/endpoints_test.py b/tests/endpoints_test.py
index 1ea11c5..c852dd2 100644
--- a/tests/endpoints_test.py
+++ b/tests/endpoints_test.py
@@ -85,3 +85,18 @@ def test_docs_csp_header(client: Client, path: str):
 
     assert response.status_code == status.HTTP_200_OK
     assert response.headers.get("Content-Security-Policy", None) is not None
+
+
+def test_robots_txt(client: Client):
+    """Testa a presença do arquivo robots.txt
+
+    Args:
+        client (Client): fixture do cliente http.
+    """
+    response = client.get("/robots.txt", headers={"Accept": "text/plain"})
+    content = response.text
+
+    assert response.status_code == status.HTTP_200_OK
+    assert "Allow: /docs$" in content
+    assert "Allow: /redoc$" in content
+    assert "Disallow: /\n" in content